From d62a9e623c7025f168d515cc9a3dd79d4a13dde7 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 4 Sep 2018 11:27:47 -0600 Subject: [PATCH] Fix the NC_INMEMORY code to work in all cases with HDF5 1.10. re: github issue https://github.com/Unidata/netcdf-c/issues/1111 One of the less common use cases for the in-memory feature is apparently failing with HDF5-1.10.x. The fix is complicated and requires significant changes to libhdf5/nc4memcb.c. The current setup is detailed in the file docs/inmeminternal.dox. Additionally, it was discovered that the program nc_test/tst_inmemory.c, which is invoked by nc_test/run_inmemory.sh, actually was failing because of the above problem. But the failure is not detected since the script does not return non-zero value. Other Changes: 1. Fix nc_test_tst_inmemory to return errors correctly. 2. Make ncdap_tests/findtestserver.c and dap4_tests/findtestserver4.c be generated from ncdap_test/findtestserver.c.in. 3. Make LOG() print output to stderr instead of stdout to avoid contaminating e.g. ncdump output. 4. Modify the handling of NC_INMEMORY and NC_DISKLESS flags to properly handle that NC_DISKLESS => NC_INMEMORY. This affects a number of code pieces, especially memio.c. --- CMakeLists.txt | 6 + cf | 8 +- configure.ac | 12 +- dap4_test/Makefile.am | 7 +- dap4_test/d4test_common.sh | 4 +- dap4_test/test_raw.sh | 16 +- dap4_test/test_remote.sh | 92 +++---- docs/Doxyfile.in | 1 + docs/Makefile.am | 2 +- docs/inmeminternal.dox | 401 ++++++++++++++++++++++++++++ docs/inmemory.md | 13 +- include/hdf5internal.h | 3 + include/nc4internal.h | 16 +- include/ncdap.h | 4 +- include/nctestserver.h | 2 +- include/ncwinpath.h | 24 ++ include/netcdf_mem.h | 2 +- libdap2/test_vara.c | 5 +- libdap4/d4file.c | 2 +- libdap4/d4read.c | 70 +++-- libdap4/ncd4types.h | 4 +- libdispatch/dauth.c | 2 +- libdispatch/dfile.c | 70 +++-- libdispatch/dvar.c | 2 +- libdispatch/dwinpath.c | 33 +++ libhdf5/hdf5create.c | 12 +- libhdf5/hdf5file.c | 54 ++-- libhdf5/hdf5open.c | 42 +-- libhdf5/nc4mem.c | 29 +- libhdf5/nc4memcb.c | 467 +++++++++++++++++++++++---------- libnetcdf.settings.in | 4 +- libsrc/memio.c | 53 +++- libsrc/nc3internal.c | 2 +- libsrc4/error4.c | 12 +- nc_test/Make0 | 12 +- nc_test/run_diskless.sh | 2 +- nc_test/run_inmemory.sh | 17 +- nc_test/tst_diskless.c | 17 +- nc_test/tst_inmemory.c | 77 +++--- ncdap_test/CMakeLists.txt | 2 +- ncdap_test/Makefile.am | 4 +- ncdap_test/findtestserver.c | 1 + ncdap_test/findtestserver.c.in | 79 ++++++ ncdap_test/testurl.sh | 8 +- oc2/ochttp.c | 2 +- 45 files changed, 1273 insertions(+), 424 deletions(-) create mode 100644 docs/inmeminternal.dox create mode 100644 ncdap_test/findtestserver.c.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a54d8338..26e229117 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1972,6 +1972,12 @@ IF(ENABLE_EXAMPLES) configure_file(${CMAKE_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_BINARY_DIR}/examples/C/findplugin.sh @ONLY NEWLINE_STYLE LF) ENDIF() +##### +# Build ncdap_test|dap4_test/findtestserver[4].c +##### +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF) +configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF) + #### # Export files #### diff --git a/cf b/cf index 48edb83e3..a775b45bd 100644 --- a/cf +++ b/cf @@ -108,10 +108,10 @@ FLAGS="$FLAGS --enable-extreme-numbers" #FLAGS="$FLAGS --disable-testsets" #FLAGS="$FLAGS --disable-dap-remote-tests" #FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server -#FLAGS="$FLAGS --enable-doxygen" -#FLAGS="$FLAGS --enable-internal-docs" +FLAGS="$FLAGS --enable-doxygen" +FLAGS="$FLAGS --enable-internal-docs" FLAGS="$FLAGS --enable-logging" -FLAGS="$FLAGS --disable-diskless" +#FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-mmap" #FLAGS="$FLAGS --with-udunits" #FLAGS="$FLAGS --with-libcf" @@ -200,7 +200,7 @@ DISTCHECK_CONFIGURE_FLAGS="$FLAGS" export DISTCHECK_CONFIGURE_FLAGS if test "x$NB" != x -o "x$FAST" = x ; then -${MAKE} maintainer-clean >/dev/null 2>&1 +${MAKE} distclean >/dev/null 2>&1 fi if test -z "$NB" ; then if autoreconf -i --force ; then ok=1; else exit ; fi diff --git a/configure.ac b/configure.ac index 76b15a223..dc9c33454 100644 --- a/configure.ac +++ b/configure.ac @@ -172,8 +172,12 @@ AC_DEFINE([USE_FSYNC], [1], [if true, include experimental fsync code]) fi # Temporary until JNA bug is fixed (which is probably never). -# See Jira NCF-298 -AC_MSG_CHECKING([if jna bug workaround is enabledd]) +# The problem being solved is this: +# > On Windows using the microsoft runtime, it is an error +# > for one library to free memory allocated by a different library. +# This is probably only an issue when using the netcdf-c library +# via JNA under Java. +AC_MSG_CHECKING([if jna bug workaround is enabled]) AC_ARG_ENABLE([jna], [AS_HELP_STRING([--enable-jna], [enable jna bug workaround])], @@ -1482,6 +1486,10 @@ AC_SUBST([MSVC], []) AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in) AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in) +# DAP 2/4 findtestserver[4].c setup +AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in) +AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in) + ##### # End netcdf_meta.h definitions. ##### diff --git a/dap4_test/Makefile.am b/dap4_test/Makefile.am index 8e09e847d..19c9b6725 100644 --- a/dap4_test/Makefile.am +++ b/dap4_test/Makefile.am @@ -8,8 +8,11 @@ include $(top_srcdir)/lib_flags.am #TEST_EXTENSIONS = .sh +#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose +#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose - +#TESTS_ENVIRONMENT = export SETX=1; + LDADD = ${top_builddir}/liblib/libnetcdf.la AM_CPPFLAGS += -I$(top_srcdir)/libdap4 @@ -62,6 +65,8 @@ CLEANFILES = *.exe # This should only be left behind if using parallel io CLEANFILES += tmp_* +DISTCLEANFILES = findtestserver4.c + # One last thing BUILT_SOURCES = .daprc diff --git a/dap4_test/d4test_common.sh b/dap4_test/d4test_common.sh index 5be59e447..f9f7405ce 100755 --- a/dap4_test/d4test_common.sh +++ b/dap4_test/d4test_common.sh @@ -24,8 +24,8 @@ cd ${srcdir}/baselineremote; BASELINEREM=`pwd` ; cd ${WD} BASELINEH=${BASELINEREM} setresultdir() { -rm -fr ./$1 -mkdir -p ./$1 +rm -fr ${builddir}/$1 +mkdir -p ${builddir}/$1 } FAILURES= diff --git a/dap4_test/test_raw.sh b/dap4_test/test_raw.sh index f294e937c..ef46a1a6d 100755 --- a/dap4_test/test_raw.sh +++ b/dap4_test/test_raw.sh @@ -42,29 +42,29 @@ resultclean() { } setresultdir results_test_raw + if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi for f in $F ; do echo "testing: $f" - URL="[dap4]file://${DAPTESTFILES}/${f}" - if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_raw/${f}.dmp; then + URL="[log][dap4]file://${DAPTESTFILES}/${f}" + if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_raw/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then - if ! diff -wBb ${BASELINERAW}/${f}.dmp ./results_test_raw/${f}.dmp ; then + if ! diff -wBb ${BASELINERAW}/${f}.dmp ${builddir}/results_test_raw/${f}.dmp ; then failure "diff ${f}.dmp" fi elif test "x${RESET}" = x1 ; then echo "${f}:" - cp ./results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp + cp ${builddir}/results_test_raw/${f}.dmp ${BASELINERAW}/${f}.dmp elif test "x${DIFF}" = x1 ; then echo "hdrtest: ${f}" - rm -f ./tr1 ./tr2 ./tb1 ./tb2 baseclean - if ! diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump ; then - failure diff -wBb ./${BASELINERAW}/${f}.dmp ./${BASELINE}/${f}.ncdump + if ! diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump ; then + failure diff -wBb ${BASELINERAW}/${f}.dmp ${BASELINE}/${f}.ncdump fi fi done -rm -rf ./results_test_raw +rm -rf ${builddir}/results_test_raw finish diff --git a/dap4_test/test_remote.sh b/dap4_test/test_remote.sh index 5e9b6809a..5582ab237 100755 --- a/dap4_test/test_remote.sh +++ b/dap4_test/test_remote.sh @@ -13,47 +13,47 @@ echo "test_remote.sh:" #NOCSUM=1 F="\ -test_atomic_array.nc -test_atomic_types.nc -test_enum.nc -test_enum_2.nc -test_enum_array.nc -test_fill.nc -test_groups1.nc -test_misc1.nc -test_one_var.nc -test_one_vararray.nc -test_opaque.nc -test_opaque_array.nc -test_struct1.nc -test_struct_array.nc -test_struct_nested.nc -test_struct_nested3.nc -test_struct_type.nc -test_utf8.nc -test_vlen1.nc -test_vlen2.nc -test_vlen3.nc -test_vlen4.nc -test_vlen5.nc -test_vlen6.nc -test_vlen7.nc -test_vlen8.nc -test_vlen9.nc -test_vlen10.nc -test_vlen11.nc -tst_fills.nc -test_struct_nested.hdf5 -test_struct_nested3.hdf5 -test_vlen3.hdf5 -test_vlen4.hdf5 -test_vlen5.hdf5 -test_anon_dim.syn -test_atomic_array.syn -test_atomic_types.syn -test_sequence_1.syn -test_sequence_2.syn -test_struct_array.syn +test_atomic_array.nc \ +test_atomic_types.nc \ +test_enum.nc \ +test_enum_2.nc \ +test_enum_array.nc \ +test_fill.nc \ +test_groups1.nc \ +test_misc1.nc \ +test_one_var.nc \ +test_one_vararray.nc \ +test_opaque.nc \ +test_opaque_array.nc \ +test_struct1.nc \ +test_struct_array.nc \ +test_struct_nested.nc \ +test_struct_nested3.nc \ +test_struct_type.nc \ +test_utf8.nc \ +test_vlen1.nc \ +test_vlen2.nc \ +test_vlen3.nc \ +test_vlen4.nc \ +test_vlen5.nc \ +test_vlen6.nc \ +test_vlen7.nc \ +test_vlen8.nc \ +test_vlen9.nc \ +test_vlen10.nc \ +test_vlen11.nc \ +tst_fills.nc \ +test_struct_nested.hdf5 \ +test_struct_nested3.hdf5 \ +test_vlen3.hdf5 \ +test_vlen4.hdf5 \ +test_vlen5.hdf5 \ +test_anon_dim.syn \ +test_atomic_array.syn \ +test_atomic_types.syn \ +test_sequence_1.syn \ +test_sequence_2.syn \ +test_struct_array.syn \ " setresultdir results_test_remote @@ -66,27 +66,27 @@ fi if test "x${RESET}" = x1 ; then rm -fr ${BASELINER}/*.dmp ; fi for f in $F ; do - URL="[log][dap4]${TESTSERVER}/d4ts/testfiles/${f}" + URL="[log][show=fetch][dap4]${TESTSERVER}/testfiles/${f}" if test "x$BIG" = x1; then URL="[ucar.littleendian=0]${URL}" fi if test "x$NOCSUM" = x1; then URL="[ucar.checksummode=none]${URL}" fi - if ! ${VG} ${NCDUMP} "${URL}" > ./results_test_remote/${f}.dmp; then + if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then failure "${URL}" fi if test "x${TEST}" = x1 ; then - if ! diff -wBb ${BASELINEREM}/${f}.dmp ./results_test_remote/${f}.dmp ; then + if ! diff -wBb "${BASELINEREM}/${f}.dmp" "${builddir}/results_test_remote/${f}.dmp" ; then failure "diff ${f}.dmp" fi elif test "x${RESET}" = x1 ; then echo "${f}:" - cp ./results_test_remote/${f}.dmp ${BASELINEREM}/${f}.dmp + cp "${builddir}/results_test_remote/${f}.dmp" "${BASELINEREM}/${f}.dmp" fi done -rm -fr ./results_test_remote +rm -fr ${builddir}/results_test_remote finish diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index e335c78d0..020a28cd7 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -768,6 +768,7 @@ INPUT = \ @abs_top_srcdir@/docs/bestpractices.md \ @abs_top_srcdir@/docs/tutorial.dox \ @abs_top_srcdir@/docs/internal.dox \ + @abs_top_srcdir@/docs/inmeminternal.dox \ @abs_top_srcdir@/docs/indexing.dox \ @abs_top_srcdir@/docs/testserver.dox \ @abs_top_srcdir@/include/netcdf.h \ diff --git a/docs/Makefile.am b/docs/Makefile.am index 5c644435a..43016cd36 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -11,7 +11,7 @@ install-fortran.md all-error-codes.md credits.md auth.md \ obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \ inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \ file_format_specifications.md known_problems.md COPYRIGHT.dox \ -user_defined_formats.md +user_defined_formats.md inmeminternal.dox # Turn off parallel builds in this directory. .NOTPARALLEL: diff --git a/docs/inmeminternal.dox b/docs/inmeminternal.dox new file mode 100644 index 000000000..ba22ed595 --- /dev/null +++ b/docs/inmeminternal.dox @@ -0,0 +1,401 @@ +/** +@if INTERNAL + +@page inmemintern Internal Architecture for NC_INMEMORY Support + +\tableofcontents + + + + +# Introduction {#inmemintern_intro} + +This document describes the internal workings +of the inmemory features of the netcdf-c library. +The companion document to this -- inmemory.md -- +describes the "external" operation of the inmemory features. + +This document describes how the in-memory operation +is implemented both for netcdf-3 files and for netcdf-4 files. + +# Generic Capabilities {#inmemintern_general} + +Both the netcdf-3 and netcdf-4 implementations assume that +they are initially given a (pointer,size) pair representing +a chunk of allocated memory of specified size. + +If a file is being created instead of opened, then only the size +is needed and the netcdf-c library will internally allocate the +corresponding memory chunk. + +If NC_DISKLESS is being used, then a chunk of memory is allocated +whose size is the same as the length of the file, and the contents +of the file is then read into that chunk of memory. + +This information is in general represented by the following struct +(see include/netcdf_mem.h). +```` +typedef struct NC_memio { + size_t size; + void* memory; + int flags; +} NC_memio; + +```` +The flags field describes properties and constraints to be applied +to the given memory. At the moment, only this one flag is defined. +```` +#define NC_MEMIO_LOCKED 1 +```` +If this flag is set, then the netcdf library will ensure that +the original allocated memory is ```locked```, which means +that it will never be realloc'd nor free'd. +Note that this flag is ignored when creating a memory file: it is only +relevant when opening a pre-allocated chunk of memory via the +_nc_open_mem_ function. + +Note that this flag does not prevent the memory from being modified. +If there is room, then the memory may be modified in place. If the size +of the memory needs to be increased and the this flag is set, then +the operation will fail. + +When the _nc_close_memio_ function is called instead of +_nc_close_, then the currently allocated memory (and its size) +is returned. If the _NC_MEMIO_LOCKED_ flag is set, then it +should be the case that the chunk of memory returned is the same +as originally provided. However, the size may be different +because it represents the amount of memory that contains +meaningful data; this value may be less than the original provided size. +The actual allocated size for the memory chunk is the same as originally +provided, so it that value is needed, then the caller must save it somewhere. + +Note also that ownership of the memory chunk is given to the +caller, and it is the caller's responsibility to _free_ the memory. + +# NetCDF-4 Implementation {#inmemintern_nc4} + +The implementation of in-memory support for netcdf-4 files +is quite complicated. + +The netCDF-4 implemention relies on the HDF5 library. In order +to implement in-memory storage of data, the HDF5 core driver is +used to manage underlying storage of the netcdf-c file. + +An HDF5 driver is an abstract interface that allows different +underlying storage implementations. So there is a standard file +driver as well as a core driver, which uses memory as the +underlying storage. + +Generically, the memory is referred to as a file image [1]. + +## libhdf5/nc4mem + +The primary API for in-memory operations is in the file +libhdf5/nc4mem.c and the defined functions are described in the next sections + +### nc4mem.NC4_open_image_file + +The signature is: +```` +int NC4_open_image_file(NC_FILE_INFO_T* h5) +```` +Basically, this function sets up the necessary state information +to use the HDF5 core driver. +It obtains the memory chunk and size from the _h5->mem.memio_ field. + +Specifically, this function converts the +_NC_MEMIO_LOCKED_ flag into using the HDF5 image specific flags: +_H5LT_FILE_IMAGE_DONT_COPY_ and _H5LT_FILE_IMAGE_DONT_RELEASE_. +It then invokes the function _libhdf5/nc4memcb/NC4_image_init_ +function to do the necessary HDF5 specific setup. + +### nc4mem.NC4_create_image_file + +The signature is: +```` +int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsize) +```` + +This function sets up the necessary state information +to use the HDF5 core driver, but for a newly created file. +It initializes the memory chunk and size in the _h5->mem.memio_ field +from the _initialsize_ argument and it leaves the memory chunk pointer NULL. +It ignores the _NC_MEMIO_LOCKED_ flag. +It then invokes the function _libhdf5/nc4memcb/NC4_image_init_ +function to do the necessary HDF5 specific setup. + +### libhdf5/hdf5file.c/nc4_close-netcdf4_file + +When a file is closed, this function is invoked. As part of its operation, +and if the file is an in-memory file, it does one of two things. + +1. If the user provided an _NC_memio_ instance, then return the final image +in that instance; the user is then responsible for freeing it. +2. If no _NC_memio_ instance was provided, then just discard the final image. + +## libhdf5/nc4memcb + +The HDF5 core driver uses an abstract interface for managing the +allocation and free'ing of memory. This interface is defined +as a set of callback functions [2] that implement the functions +of this struct. + +```` +typedef struct { + void *(*_malloc)(size_t size, H5_file_image_op_t op, void *udata); + void *(*_memcpy)(void *dest, const void *src, size_t size, + H5_file_image_op_t op, void *udata); + void *(*_realloc)(void *ptr, size_t size, + H5_file_image_op_t op, void *udata); + herr_t (*_free)(void *ptr, H5_file_image_op_t op, void *udata); + void *(*udata_copy)(void *udata); + herr_t (*udata_free)(void *udata); + void *udata; +} H5_file_image_callbacks_t; +```` +The _udata_ field at the end defines any extra state needed by the functions. +Each function is passed the udata as its last argument. The structure of the +udata is arbitrary, and is passed as _void*_ to the functions. + +The _udata_ structure and callback functions used by the netcdf-c library +are defined in the file _libhdf5/nc4memcb.c_. Setup is defined by the +function _NC4_image_init_ in that same file. + +The _udata_ structure used by netcdf is as follows. +```` +typedef struct { + void *app_image_ptr; /* Pointer to application buffer */ + size_t app_image_size; /* Size of application buffer */ + void *fapl_image_ptr; /* Pointer to FAPL buffer */ + size_t fapl_image_size; /* Size of FAPL buffer */ + int fapl_ref_count; /* Reference counter for FAPL buffer */ + void *vfd_image_ptr; /* Pointer to VFD buffer (Note: VFD => used by core driver) */ + size_t vfd_image_size; /* Size of VFD buffer */ + int vfd_ref_count; /* Reference counter for VFD buffer */ + unsigned flags; /* Flags indicate how the file image will be opened */ + int ref_count; /* Reference counter on udata struct */ + NC_FILE_INFO_T* h5; /* Pointer to the netcdf parent structure */ +} H5LT_file_image_ud_t; +```` + +It is necessary to understand one more point about the callback functions. +The first four take an argument of type _H5_file_image_op_t_ -- the operator. +This is an enumeration that indicates additional context about the purpose for +which the callback is being invoked. For the purposes of the netcdf-4 +implementation, only the following operators are used. + +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET +- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +- H5FD_FILE_IMAGE_OP_FILE_OPEN +- H5FD_FILE_IMAGE_OP_FILE_RESIZE +- H5FD_FILE_IMAGE_OP_FILE_CLOSE + +As can be seen, basically the operators indicate if the operation is with respect to +an HDF5 property list, or with respect to a file (i.e. a core image in this case). +For each callback described below, the per-operator actions will be described. +Not all operators are used with all callbacks. + +Internally, the HDF5 core driver thinks it is doing the following: + +1. Allocate memory and copy the incoming memory chunk into that newly allocated memory + (call image_malloc followed by image_memcpy). +2. Periodically reallocate the memory to increase its size + (call image_realloc). +3. Free up the memory as no longer needed + (call image_free). + +It turns out that for propertly lists, realloc is never called. +However the HDF5 core driver follows all of the above steps. + +The following sections describe the callback function operation. + +### libhdf5/nc4memcb/local_image_malloc + +This function is called to allocated an internal chunk of memory so +the original provided memory is no longer needed. In order to implement +the netcdf-c semantics, we modify this behavior. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET +We assume that the property list image info will never need to be modified, +so we just copy the incoming buffer info (the app_image fields) into the fapl_image fields. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY +Basically just return the fapl_image_ptr field, so no actual copying. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY and H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET +Basically just return the fapl_image_ptr field, so no actual copying or malloc needed. + +#### Operator H5FD_FILE_IMAGE_OP_FILE_OPEN +Since we always start by using the original incoming image buffer, we just +need to store that pointer and size into the vfd_image fields (remember, vfd is that +used by the core driver). + +### libhdf5/nc4memcb/local_image_memcpy +This function is supposed to be used to copy the incoming buffer into an internally +malloc'd buffer. Since we use the original buffer, no memcpy is actually needed. +As a safety check, we do actually do a memcpy if, for some reason, the _src_ and _dest_ +arguments are different. In practice, this never happens. + +### libhdf5/nc4memcb/local_image_realloc +Since the property list image is never realloc'd this is only called with +_H5FD_FILE_IMAGE_OP_FILE_RESIZE_. + +If the memory is not locked (i.e. the _NC_MEMIO_LOCKED_ flag was not used), +then we are free to realloc the vfd_ptr. But if the memory is locked, +then we cannot realloc and we must fake it as follows: + +1. If the chunk is big enough, then pretend to do a realloc by + changing the vfd_image_size. +2. If the chunk is not big enough to accomodate the requested new size, + then fail. + +There is one important complication. It turns out that the image_realloc +callback is sometimes called with a ptr argument value of NULL. This assumes +that if realloc is called with a NULL buffer pointer, then it acts like _malloc_. +Since we have found that some systems to do not implement this, we implement it +in our local_image_realloc code and do a _malloc_ instead of _realloc_. + +### libhdf5/nc4memcb/local_image_free + +This function is, of course, invoked to deallocate memory. +It is only invoked with the +H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +and H5FD_FILE_IMAGE_OP_FILE_CLOSE +operators. + +#### Operator H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE +For the way the netcdf library uses it, it should still be the case that +the fapl pointer is same as original incoming app_ptr, so we do not need +to do anything for this operator. + +#### Operator H5FD_FILE_IMAGE_OP_FILE_CLOSE +Since in our implementation, we maintain control of the memory, this case +will never free any memory, but may save a pointer to the current vfd memory +so it can be returned to the original caller, if they want it. +Specifically the vfd_image_ptr and vfd_image_size are always +copied to the _udata->h5->mem.memio_ field so they can +be referenced by higher level code. + +### libhdf5/nc4memcb/local_udata_copy +Our version of this function only manipulates the reference count. + +### libhdf5/nc4memcb/local_udata_free +Our version of this function only manipulates the reference count. + +# NetCDF-3 Implementation {#inmemintern_nc3} + +The netcdf-3 code -- in libsrc -- has its own, internal storage +management API as defined in the file _libsrc/ncio.h_. It implements +the API in the form of a set of function pointers as defined in the +structure _struct_ _ncio_. These function have the following signatures +and semantics. + +- int ncio_relfunc(ncio*, off_t offset, int rflags) -- + Indicate that you are done with the region which begins at offset. +- int ncio_getfunc(ncio*, off_t offset, size_t extent, int rflags, void **const vpp) -- + Request that the region (offset, extent) be made available through *vpp. +- int ncio_movefunc(ncio*, off_t to, off_t from, size_t nbytes, int rflags) -- + Like memmove(), safely move possibly overlapping data. +- int ncio_syncfunc(ncio*) -- + Write out any dirty buffers to disk and ensure that next read will get data from disk. +- int ncio_pad_lengthfunc(ncio*, off_t length) -- + Sync any changes to disk, then truncate or extend file so its size is length. +- int ncio_closefunc(ncio*, int doUnlink) + -- Write out any dirty buffers and ensure that next read will not get cached data. + Then sync any changes, and then close the open file. + +The _NC_INMEMORY_ semantics are implemented by creating an implementation of the above functions +specific for handling in-memory support. This is implemented in the file _libsrc/memio.c_. + +## Open/Create/Close + +Open and close related functions exist in _memio.c_ that are not specifically part of the API. +These functions are defined in the following sections. + +### memio_create +Signature: +```` +int memio_create(const char* path, int ioflags, size_t initialsz, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters /*ignored*/, ncio* *nciopp, void** const mempp) +```` +Create a new file. Invoke _memio_new_ to create the _ncio_ +instance. If it is intended that the resulting file be +persisted to the file system, then verify that writing such a +file is possible. Also create an initial in-memory buffer to +hold the file data. Otherwise act like e.g. _posixio_create_. + +### memio_open +Signature: +```` +int memio_open(const char* path, int ioflags, off_t igeto, size_t igetsz, size_t* sizehintp, void* parameters, ncio* *nciopp, void** const mempp) +```` +Open an existing file. Invoke _memio_new_ to create the _ncio_ +instance. If it is intended that the resulting file be +persisted to the file system, then verify that writing such a +file is possible. Also create an initial in-memory buffer to +hold the file data. Read the contents of the existing file into +the allocated memory. +Otherwise act like e.g. _posixio_open_. + +### memio_extract +Signature: +```` +int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp) +```` +This function is called as part of the NC3_close function in the event that +the user wants the final in-memory chunk returned to them via _nc_close_mem_. +It captures the existing in-memory chunk and returns it. At this point, +memio will no longer have access to that memory. + +## API Semantics + +The semantic interaction of the above API and NC_INMEMORY are described in the following sections. + +### ncio_relfunc +Just unlock the in-memory chunk. + +### ncio_getfunc +First guarantee that the requested region exists, and if necessary, +realloc to make it exist. If realloc is needed, and the +file is locked, then fail. + +### ncio_movefunc +First guarantee that the requested destination region exists, and if necessary, +realloc to make it exist. If realloc is needed, and the +file is locked, then fail. + +### ncio_syncfunc +This is a no-op as far as memio is concerned. + +### ncio_pad_lengthfunc +This may realloc the allocated in-memory buffer to achieve padding +rounded up to the pagesize. + +### ncio_filesizefunc +This just returns the used size of the in-memory chunk. +Note that the allocated size might be larger. + +### ncio_closefunc +If the usere wants the contents persisted, then write out the used portion +of the in-memory chunk to the target file. +Then, if the in-memory chunk is not locked, or for some reason has +been modified, go ahead and free that memory. + +# References {#inmemintern_bib} + +1. https://support.hdfgroup.org/HDF5/doc1.8/Advanced/FileImageOperations/HDF5FileImageOperations.pdf +2. https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFileImageCallbacks + +# Point of Contact {#inmemintern_poc} + +__Author__: Dennis Heimbigner
+__Email__: dmh at ucar dot edu
+__Initial Version__: 8/28/2018
+__Last Revised__: 8/28/2018 + + + +@endif + +*/ diff --git a/docs/inmemory.md b/docs/inmemory.md index b3bc88ba0..db22b9ffc 100644 --- a/docs/inmemory.md +++ b/docs/inmemory.md @@ -129,16 +129,19 @@ Note that it is still possible to modify the in-memory file if the NC_WRITE mode flag was set. However, failures can occur if an operation cannot complete because the memory needs to be expanded. 2. If the *NC_MEMIO_LOCKED* flag is not set, then -the netcdf library will take control of the incoming memory -and will feel free to reallocate the provided +the netcdf library will take control of the incoming memory. +This means that the user should not make any attempt to free +or even read the incoming memory block in this case. +The newcdf library is free to reallocate the incomming memory block to obtain a larger block when an attempt to modify the in-memory file requires more space. Note that implicit in this is that the old block -- the one originally provided -- may be free'd as a side effect of re-allocating the memory using the *realloc()* function. -If the caller invokes the *nc_close_memio()* function to retrieve the -final memory block, the returned block must always be freed -by the caller and that the original block should not be freed. +The caller may invoke the *nc_close_memio()* function to retrieve the +final memory block, which may not be the same as the originally block +provided by the caller. In any case, the returned block must always be freed +by the caller and the original block should not be freed. ### The **nc_create_mem** Function diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 83957d37c..8652ef816 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -60,6 +60,9 @@ typedef struct NC_HDF5_FILE_INFO int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); +/* In-memory functions */ +extern hid_t NC4_image_init(NC_FILE_INFO_T* h5); +extern void NC4_image_finalize(void*); /* These functions are internal to the libhdf5 directory. */ int nc4_detect_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp); int hdf5_set_log_level(); diff --git a/include/nc4internal.h b/include/nc4internal.h index 5bf4df522..c29eb5ad0 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -301,15 +301,15 @@ typedef struct NC_FILE_INFO void *format_file_info; struct NCFILEINFO* fileinfo; struct NC4_Memio { - NC_memio memio; - int locked; /* do not copy and do not release */ + NC_memio memio; /* What we sent to image_init and what comes back*/ + int locked; /* do not copy and do not free */ int persist; /* Should file be persisted out on close? */ - int inmemory; - int diskless; - unsigned int flags; /* for H5LTopen_file_image */ - int fapl; - size_t initialsize; + int inmemory; /* NC_INMEMORY flag was set */ + int diskless; /* NC_DISKLESS flag was set => inmemory */ int created; /* 1 => create, 0 => open */ + unsigned int imageflags; /* for H5LTopen_file_image */ + size_t initialsize; + void* udata; /* extra memory allocated in NC4_image_init */ } mem; } NC_FILE_INFO_T; @@ -414,7 +414,7 @@ int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name); int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value); /* Close the file. */ -int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem); +int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio*); /* HDF5 initialization */ extern int nc4_hdf5_initialized; diff --git a/include/ncdap.h b/include/ncdap.h index 50ccb4492..e30d36395 100644 --- a/include/ncdap.h +++ b/include/ncdap.h @@ -17,8 +17,8 @@ affect the operation of the system. */ typedef unsigned int NCFLAGS; -# define SETFLAG(controls,flag) ((controls.flags) |= (flag)) -# define CLRFLAG(controls,flag) ((controls.flags) &= ~(flag)) +# define SETFLAG(controls,flag) (((controls).flags) |= (flag)) +# define CLRFLAG(controls,flag) (((controls).flags) &= ~(flag)) # define FLAGSET(controls,flag) (((controls.flags) & (flag)) != 0) /* Defined flags */ diff --git a/include/nctestserver.h b/include/nctestserver.h index 6c603602c..637b9ac93 100644 --- a/include/nctestserver.h +++ b/include/nctestserver.h @@ -55,7 +55,7 @@ This indicates that the server is up and running. Return the complete url for the server plus the path. */ -char* +static char* nc_findtestserver(const char* path, int isdap4, const char* serverlist) { char** svc; diff --git a/include/ncwinpath.h b/include/ncwinpath.h index 28e9058a4..e62368d97 100644 --- a/include/ncwinpath.h +++ b/include/ncwinpath.h @@ -8,6 +8,9 @@ #include "config.h" #include #include +#ifdef HAVE_UNISTD_H +#include +#endif #include "ncexternl.h" #ifndef WINPATH @@ -19,6 +22,19 @@ #endif #endif +/* Define wrapper constants for use with NCaccess */ +#ifdef _MSC_VER +#define ACCESS_MODE_EXISTS 0 +#define ACCESS_MODE_R 4 +#define ACCESS_MODE_W 2 +#define ACCESS_MODE_RW 6 +#else +#define ACCESS_MODE_EXISTS (F_OK) +#define ACCESS_MODE_R (R_OK) +#define ACCESS_MODE_W (W_OK) +#define ACCESS_MODE_RW (R_OK|W_OK) +#endif + /* Path Converter */ EXTERNL char* NCpathcvt(const char* path); @@ -27,10 +43,18 @@ EXTERNL char* NCpathcvt(const char* path); EXTERNL FILE* NCfopen(const char* path, const char* flags); EXTERNL int NCopen3(const char* path, int flags, int mode); EXTERNL int NCopen2(const char* path, int flags); +EXTERNL int NCaccess(const char* path, int mode); +EXTERNL int NCremove(const char* path); #else /*!WINPATH*/ #define NCfopen(path,flags) fopen((path),(flags)) #define NCopen3(path,flags,mode) open((path),(flags),(mode)) #define NCopen2(path,flags) open((path),(flags)) +#define NCremove(path) remove(path) +#ifdef _MSC_VER +#define NCaccess(path,mode) _access(path,mode) +#else +#define NCaccess(path,mode) access(path,mode) +#endif #endif /*WINPATH*/ #endif /* _NCWINIO_H_ */ diff --git a/include/netcdf_mem.h b/include/netcdf_mem.h index c8079896c..2592907b2 100644 --- a/include/netcdf_mem.h +++ b/include/netcdf_mem.h @@ -17,8 +17,8 @@ typedef struct NC_memio { size_t size; void* memory; -#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */ int flags; +#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */ } NC_memio; #if defined(__cplusplus) diff --git a/libdap2/test_vara.c b/libdap2/test_vara.c index c15cdbf65..a2c9e8715 100644 --- a/libdap2/test_vara.c +++ b/libdap2/test_vara.c @@ -2,9 +2,8 @@ #include #include #include -#include - -extern char* nc_findtestserver(const char* path, int isdap4); +#include "netcdf.h" +#include "nctestserver.h" #define DTSTEST "/dts/test.06" diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 13e61726d..b2577d607 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -304,7 +304,7 @@ freeInfo(NCD4INFO* d4info) for some reason, so we delete it ourselves. */ if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); +// unlink(d4info->substrate.filename); } } nullfree(d4info->substrate.filename); /* always reclaim */ diff --git a/libdap4/d4read.c b/libdap4/d4read.c index b3d87b5a1..434055faa 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -9,8 +9,24 @@ /*Forward*/ static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, long*); -static int readfile(const NCURI*, const char* suffix, NCbytes* packet); -static int readfiletofile(const NCURI*, const char* suffix, FILE* stream, d4size_t*); +static int readfile(NCD4INFO* state, const NCURI*, const char* suffix, NCbytes* packet); +static int readfiletofile(NCD4INFO* state, const NCURI*, const char* suffix, FILE* stream, d4size_t*); + +#ifdef HAVE_GETTIMEOFDAY +static struct timeval time0; +static struct timeval time1; + +static double +deltatime() +{ + double t0, t1; + t0 = ((double)time0.tv_sec); + t0 += ((double)time0.tv_usec) / 1000000.0; + t1 = ((double)time1.tv_sec); + t1 += ((double)time1.tv_usec) / 1000000.0; + return (t1 - t0); +} +#endif int NCD4_readDMR(NCD4INFO* state) @@ -38,7 +54,7 @@ NCD4_readDAP(NCD4INFO* state, int flags) NCURI* url = state->uri; int fileprotocol = (strcmp(url->protocol,"file")==0); if(fileprotocol) { - stat = readfiletofile(url, ".dap", state->data.ondiskfile, &state->data.datasize); + stat = readfiletofile(state, url, ".dap", state->data.ondiskfile, &state->data.datasize); } else { char* readurl = NULL; int flags = 0; @@ -51,15 +67,11 @@ NCD4_readDAP(NCD4INFO* state, int flags) readurl = ncuribuild(url,NULL,".dods",NCURISVC); if(readurl == NULL) return THROW(NC_ENOMEM); - if (state->debug > 0) - {fprintf(stderr, "fetch url=%s\n", readurl);fflush(stderr);} stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, &state->data.datasize, &lastmod); nullfree(readurl); if(stat == NC_NOERR) state->data.daplastmodified = lastmod; - if (state->debug > 0) - {fprintf(stderr,"fetch complete\n"); fflush(stderr);} } } return THROW(stat); @@ -89,7 +101,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las if(fileprotocol) { /* Short circuit file://... urls*/ /* We do this because the test code always needs to read files*/ - stat = readfile(url,suffix,packet); + stat = readfile(state, url,suffix,packet); } else { char* fetchurl = NULL; int flags = NCURIBASE; @@ -97,13 +109,23 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, long* las flags |= NCURIENCODE; fetchurl = ncuribuild(url,NULL,suffix,flags); MEMCHECK(fetchurl); - if(state->debug > 0) - {fprintf(stderr,"fetch url=%s\n",fetchurl); fflush(stderr);} + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + nclog(NCLOGDBG,"fetch url=%s",fetchurl); +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time0,NULL); +#endif + } stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified); nullfree(fetchurl); if(stat) goto fail; - if(state->debug > 0) - {fprintf(stderr,"fetch complete\n"); fflush(stderr);} + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + double secs = 0; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time1,NULL); + secs = deltatime(); +#endif + nclog(NCLOGDBG,"fetch complete: %0.3f",secs); + } } #ifdef D4DEBUG { @@ -116,12 +138,12 @@ fail: } static int -readfiletofile(const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep) +readfiletofile(NCD4INFO* state, const NCURI* uri, const char* suffix, FILE* stream, d4size_t* sizep) { int stat = NC_NOERR; NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(uri,suffix,packet); + stat = readfile(state, uri,suffix,packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", (unsigned long)ncbyteslength(packet)); @@ -147,7 +169,7 @@ unwind: } static int -readfile(const NCURI* uri, const char* suffix, NCbytes* packet) +readfile(NCD4INFO* state, const NCURI* uri, const char* suffix, NCbytes* packet) { int stat = NC_NOERR; NCbytes* tmp = ncbytesnew(); @@ -159,6 +181,24 @@ readfile(const NCURI* uri, const char* suffix, NCbytes* packet) filename = ncbytesextract(tmp); ncbytesfree(tmp); + state->fileproto.filename = strdup(filename); + + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + char* surl = NULL; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time0,NULL); +#endif + surl = ncuribuild((NCURI*)uri,NULL,NULL,NCURIALL); + nclog(NCLOGDBG,"fetch uri=%s file=%s",surl,filename); + } stat = NC_readfile(filename,packet); + if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { + double secs; +#ifdef HAVE_GETTIMEOFDAY + gettimeofday(&time1,NULL); + secs = deltatime(); +#endif + nclog(NCLOGDBG,"fetch complete: %0.3f",secs); + } return THROW(stat); } diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index fa74110f4..eb41e5e89 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -280,7 +280,6 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - int debug; char* rawurltext; /* as given to ncd4_open */ char* urltext; /* as modified by ncd4_open */ NCURI* uri; /* parse of rawuritext */ @@ -307,6 +306,9 @@ struct NCD4INFO { char substratename[NC_MAX_NAME]; } controls; NCauth auth; + struct { + char* filename; + } fileproto; }; #endif /*D4TYPES_H*/ diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c index e03b91baf..914ad6ab4 100644 --- a/libdispatch/dauth.c +++ b/libdispatch/dauth.c @@ -30,7 +30,7 @@ See LICENSE.txt for license information. /* Define the curl flag defaults in envv style */ static const char* AUTHDEFAULTS[] = { -"HTTP.TIMEOUT","10", /*seconds */ +"HTTP.TIMEOUT","1800", /*seconds */ /* Long but not infinite */ NULL }; diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 25914d389..c87713065 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -45,7 +45,8 @@ struct MagicFile { long long filelen; int use_parallel; int inmemory; - void* parameters; + int diskless; + void* parameters; /* !NULL if inmemory && !diskless */ FILE* fp; #ifdef USE_PARALLEL MPI_File fh; @@ -292,9 +293,10 @@ NC_check_file_type(const char *path, int flags, void *parameters, { char magic[MAGIC_NUMBER_LEN]; int status = NC_NOERR; - int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS); - int inmemory = (!diskless && ((flags & NC_INMEMORY) == NC_INMEMORY)); + int inmemory = ((flags & NC_INMEMORY) == NC_INMEMORY); + int mmap = ((flags & NC_MMAP) == NC_MMAP); + #ifdef USE_PARALLEL int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO); #endif /* USE_PARALLEL */ @@ -303,25 +305,20 @@ NC_check_file_type(const char *path, int flags, void *parameters, *model = 0; *version = 0; + assert(inmemory ? !mmap : 1); /* inmemory => !mmap */ + assert((diskless && inmemory) ? !mmap : 1);/*diskless & inmemory => !mmap*/ + memset((void*)&file,0,sizeof(file)); file.path = path; /* do not free */ file.parameters = parameters; - if(inmemory && parameters == NULL) - {status = NC_EINMEMORY; goto done;} - if(inmemory) { - file.inmemory = inmemory; - goto next; - } - /* presumably a real file */ + file.inmemory = inmemory; + file.diskless = diskless; #ifdef USE_PARALLEL + /* presumably a real file */ /* for parallel, use the MPI functions instead (why?) */ - if (use_parallel) { - file.use_parallel = use_parallel; - goto next; - } + file.use_parallel = use_parallel; #endif /* USE_PARALLEL */ -next: status = openmagic(&file); if(status != NC_NOERR) {goto done;} /* Verify we have a large enough file */ @@ -1954,6 +1951,8 @@ static int check_create_mode(int mode) { int mode_format; + int mmap = 0; + int inmemory = 0; /* This is a clever check to see if more than one format bit is * set. */ @@ -1967,9 +1966,14 @@ check_create_mode(int mode) if (mode & NC_MPIIO && mode & NC_MPIPOSIX) return NC_EINVAL; - /* Can't use both parallel and diskless. */ - if ((mode & NC_MPIIO && mode & NC_DISKLESS) || - (mode & NC_MPIPOSIX && mode & NC_DISKLESS)) + mmap = ((mode & NC_MMAP) == NC_MMAP); + inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); + if(mmap && inmemory) /* cannot have both */ + return NC_EINMEMORY; + + /* Can't use both parallel and diskless|inmemory. */ + if ((mode & NC_MPIIO && mode & (NC_DISKLESS|NC_INMEMORY)) + || (mode & NC_MPIPOSIX && mode & (NC_DISKLESS|NC_INMEMORY))) return NC_EINVAL; #ifndef USE_NETCDF4 @@ -2029,11 +2033,19 @@ NC_create(const char *path0, int cmode, size_t initialsz, int model = NC_FORMATX_UNDEFINED; /* one of the NC_FORMATX values */ int isurl = 0; /* dap or cdmremote or neither */ char* path = NULL; + int mmap = 0; + int diskless = 0; TRACE(nc_create); if(path0 == NULL) return NC_EINVAL; + /* Fix the inmemory related flags */ + mmap = ((cmode & NC_MMAP) == NC_MMAP); + diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); + /* diskless && !mmap => inmemory */ + if(diskless && !mmap) cmode |= NC_INMEMORY; + /* Check mode flag for sanity. */ if ((stat = check_create_mode(cmode))) return stat; @@ -2197,6 +2209,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, NC_Dispatch* dispatcher = NULL; int inmemory = 0; int diskless = 0; + int mmap = 0; /* Need pieces of information for now to decide model*/ int model = 0; int isurl = 0; @@ -2210,14 +2223,23 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, if(stat) return stat; } + /* Fix the inmemory related flags */ + mmap = ((cmode & NC_MMAP) == NC_MMAP); + diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); + + /* diskless && !mmap => inmemory */ + if(diskless && !mmap) cmode |= NC_INMEMORY; + + inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY); + + if(mmap && inmemory) /* cannot have both */ + return NC_EINMEMORY; + /* Attempt to do file path conversion: note that this will do nothing if path is a 'file:...' url, so it will need to be repeated in protocol code: libdap2 and libdap4 */ - inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY); - diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS); - #ifdef WINPATH path = NCpathcvt(path0); #else @@ -2272,6 +2294,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, if(useparallel) flags |= NC_MPIIO; if(inmemory) flags |= NC_INMEMORY; if(diskless) flags |= NC_DISKLESS; + if(mmap) flags |= NC_MMAP; stat = NC_check_file_type(path,flags,parameters,&model,&version); if(stat == NC_NOERR) { if(model == 0) { @@ -2476,7 +2499,8 @@ static int openmagic(struct MagicFile* file) { int status = NC_NOERR; - if(file->inmemory) { + assert((!file->diskless && file->inmemory) ? file->parameters != NULL : 1); + if(file->inmemory && !file->diskless) { /* Get its length */ NC_memio* meminfo = (NC_memio*)file->parameters; file->filelen = (long long)meminfo->size; @@ -2538,7 +2562,7 @@ readmagic(struct MagicFile* file, long pos, char* magic) { int status = NC_NOERR; memset(magic,0,MAGIC_NUMBER_LEN); - if(file->inmemory) { + if(file->inmemory && !file->diskless) { char* mempos; NC_memio* meminfo = (NC_memio*)file->parameters; if((pos + MAGIC_NUMBER_LEN) > meminfo->size) diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 8701204e0..fae9c884a 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -647,7 +647,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) * @return ::NC_EBADID Bad ncid. * @return ::NC_ENOTVAR Variable not found. * @return ::NC_ENOMEM Out of memory. - * @return ::NC_EINVALCOORS Missing start array. + * @return ::NC_EINVALCOORDS Missing start array. * @author Ed Hartnett */ int diff --git a/libdispatch/dwinpath.c b/libdispatch/dwinpath.c index acd620557..6f5e551c8 100644 --- a/libdispatch/dwinpath.c +++ b/libdispatch/dwinpath.c @@ -202,4 +202,37 @@ NCopen2(const char *path, int flags) return NCopen3(path,flags,0); } +/* +Provide wrappers for other file system functions +*/ + +/* Return access applied to path+mode */ +EXTERNL +int +NCaccess(const char* path, int mode) +{ + int status = 0; + char* cvtname = NCpathcvt(path); + if(cvtname == NULL) return -1; +#ifdef _MSC_VER + status = _access(cvtname,mode); +#else + status = access(cvtname,mode); +#endif + free(cvtname); + return status; +} + +EXTERNL +int +NCremove(const char* path) +{ + int status = 0; + char* cvtname = NCpathcvt(path); + if(cvtname == NULL) return ENOENT; + status = remove(cvtname); + free(cvtname); + return status; +} + #endif /*WINPATH*/ diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index cc5194f17..78abde950 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -155,10 +155,6 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, } } #else /* only set cache for non-parallel... */ - if(cmode & NC_DISKLESS) { - if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist)) - BAIL(NC_EDISKLESS); - } if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, nc4_chunk_cache_preemption) < 0) BAIL(NC_EHDFERR); @@ -244,11 +240,9 @@ exit: /*failure exit*/ if (comm_duped) MPI_Comm_free(&nc4_info->comm); if (info_duped) MPI_Info_free(&nc4_info->info); #endif - if (fapl_id != H5P_DEFAULT) - H5Pclose(fapl_id); - if (!nc4_info) - return retval; - nc4_close_netcdf4_file(nc4_info, 1, 0); /* treat like abort */ + if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); + if(!nc4_info) return retval; + nc4_close_netcdf4_file(nc4_info,1,NULL); /* treat like abort */ return retval; } diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index b68934150..1f534cf9f 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -16,9 +16,6 @@ extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); -/* From nc4mem.c */ -extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); - /** @internal When we have open objects at file close, should we log them or print to stdout. Default is to log. */ #define LOGOPEN 1 @@ -146,17 +143,18 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) /** * @internal This function will free all allocated metadata memory, * and close the HDF5 file. The group that is passed in must be the - * root group of the file. + * root group of the file. If inmemory is used, then save + * the final memory in mem.memio. * * @param h5 Pointer to HDF5 file info struct. * @param abort True if this is an abort. - * @param extractmem True if we need to extract and save final inmemory + * @param memio the place to return a core image if not NULL * * @return ::NC_NOERR No error. * @author Ed Hartnett */ int -nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) +nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio* memio) { NC_HDF5_FILE_INFO_T *hdf5_info; int retval = NC_NOERR; @@ -202,15 +200,6 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) if (h5->fileinfo) free(h5->fileinfo); - /* Check to see if this is an in-memory file and we want to get its - final content. */ - if(extractmem) { - /* File must be read/write */ - if(!h5->no_write) { - retval = NC4_extract_file_image(h5); - } - } - /* Close hdf file. It may not be open, since this function is also * called by NC_create() when a file opening is aborted. */ if (hdf5_info->hdfid && H5Fclose(hdf5_info->hdfid) < 0) @@ -219,6 +208,25 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) BAIL(NC_EHDFERR); } + /* If inmemory is used and user wants the final memory block, + then capture and return the final memory block else free it */ + if(h5->mem.inmemory) { + if(!abort && memio != NULL) { + *memio = h5->mem.memio; /* capture it */ + h5->mem.memio.memory = NULL; /* avoid duplicate free */ + } + /* If needed, reclaim extraneous memory */ + if(h5->mem.memio.memory != NULL) { + /* If the original block of memory is not resizeable, then + it belongs to the caller and we should not free it. */ + if(!h5->mem.locked) + free(h5->mem.memio.memory); + } + h5->mem.memio.memory = NULL; + h5->mem.memio.size = 0; + NC4_image_finalize(h5->mem.udata); + } + /* Free the HDF5-specific info. */ if (h5->format_file_info) free(h5->format_file_info); @@ -242,6 +250,9 @@ dumpopenobjects(NC_FILE_INFO_T* h5) assert(h5 && h5->format_file_info); hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info; + if(hdf5_info->hdfid <= 0) + return; /* File was never opened */ + nobjs = H5Fget_obj_count(hdf5_info->hdfid, H5F_OBJ_ALL); /* Apparently we can get an error even when nobjs == 0 */ @@ -589,7 +600,7 @@ NC4_abort(int ncid) /* Free any resources the netcdf-4 library has for this file's * metadata. */ - if ((retval = nc4_close_netcdf4_file(nc4_info, 1, 0))) + if ((retval = nc4_close_netcdf4_file(nc4_info, 1, NULL))) return retval; /* Delete the file, if we should. */ @@ -617,6 +628,7 @@ NC4_close(int ncid, void* params) NC_FILE_INFO_T *h5; int retval; int inmemory; + NC_memio* memio = NULL; LOG((1, "%s: ncid 0x%x", __func__, ncid)); @@ -632,14 +644,14 @@ NC4_close(int ncid, void* params) inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY); - /* Call the nc4 close. */ - if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0)))) - return retval; if(inmemory && params != NULL) { - NC_memio* memio = (NC_memio*)params; - *memio = h5->mem.memio; + memio = (NC_memio*)params; } + /* Call the nc4 close. */ + if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, memio))) + return retval; + return NC_NOERR; } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 6a0095ab6..8755ffb9f 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -380,23 +380,10 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); - if(nc4_info->mem.inmemory) { - NC_memio* memparams = NULL; - if(parameters == NULL) - BAIL(NC_EINMEMORY); - memparams = (NC_memio*)parameters; - nc4_info->mem.memio = *memparams; /* keep local copy */ - /* As a safeguard, if !locked and NC_WRITE is set, - then we must take control of the incoming memory */ - nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; - if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) { - memparams->memory = NULL; - } + #ifdef USE_PARALLEL4 - } else { - mpiinfo = (NC_MPI_INFO*)parameters; + mpiinfo = (NC_MPI_INFO*)parameters; /* assume, may be changed if inmemory is true */ #endif /* !USE_PARALLEL4 */ - } /* Need this access plist to control how HDF5 handles open objects * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to @@ -471,10 +458,29 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc) if ((mode & NC_WRITE) == 0) nc4_info->no_write = NC_TRUE; + /* Now process if NC_INMEMORY is set (recall NC_DISKLESS => NC_INMEMORY) */ if(nc4_info->mem.inmemory) { + NC_memio* memio; + /* validate */ - if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL) - BAIL(NC_INMEMORY); + if(parameters == NULL) + BAIL(NC_EINMEMORY); + memio = (NC_memio*)parameters; + if(memio->memory == NULL || memio->size == 0) + BAIL(NC_EINMEMORY); + + /* initialize h5->mem */ + nc4_info->mem.memio = *memio; + + /* Is the incoming memory locked? */ + nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; + + /* As a safeguard, if !locked and not read-only, + then we must take control of the incoming memory */ + if(!nc4_info->mem.locked && !nc4_info->no_write) { + memio->memory = NULL; /* take control */ + memio->size = 0; + } retval = NC4_open_image_file(nc4_info); if(retval) BAIL(NC_EHDFERR); @@ -531,7 +537,7 @@ exit: #endif if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); if (!nc4_info) return retval; - nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort*/ + nc4_close_netcdf4_file(nc4_info,1,NULL); /* treat like abort*/ return retval; } diff --git a/libhdf5/nc4mem.c b/libhdf5/nc4mem.c index 6b50689a3..c9771cd75 100644 --- a/libhdf5/nc4mem.c +++ b/libhdf5/nc4mem.c @@ -31,8 +31,6 @@ #define HDrealloc(M,Z) realloc(M,Z) #endif /* HDrealloc */ -extern hid_t NC4_image_init(NC_FILE_INFO_T* h5); - int NC4_open_image_file(NC_FILE_INFO_T* h5) { @@ -43,13 +41,13 @@ NC4_open_image_file(NC_FILE_INFO_T* h5) if(h5->mem.memio.memory == NULL || h5->mem.memio.size == 0) {stat = NC_EINVAL; goto done;} - /* Figure out the flags */ - h5->mem.flags = 0; + /* Figure out the image flags */ + h5->mem.imageflags = 0; if(h5->mem.locked) { - h5->mem.flags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE); + h5->mem.imageflags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE); } if(!h5->no_write) - h5->mem.flags |= H5LT_FILE_IMAGE_OPEN_RW; + h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW; /* Create the file but using our version of H5LTopen_file_image */ hdfid = NC4_image_init(h5); @@ -67,21 +65,22 @@ int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsz) { int stat = NC_NOERR; - int hdfid; + hid_t hdfid; /* Create the file but using our version of H5LTopen_file_image */ h5->mem.created = 1; h5->mem.initialsize = initialsz; + h5->mem.imageflags |= H5LT_FILE_IMAGE_OPEN_RW; hdfid = NC4_image_init(h5); if(hdfid < 0) {stat = NC_EHDFERR; goto done;} - /* Remember HDF5 file identifier. */ ((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid = hdfid; done: return stat; } +#if 0 int NC4_extract_file_image(NC_FILE_INFO_T* h5) { @@ -89,14 +88,19 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5) hid_t fapl; herr_t herr; NC_memio mem; + hid_t hdfid = -1; assert(h5 && h5->format_file_info && !h5->no_write); - /* Get the file access property list */ - fapl = h5->mem.fapl; - if(fapl < 0) + if((hdfid = ((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid) < 0) {stat = NC_EHDFERR; goto done;} + /* The h5->mem.memio struct should contain what we want */ + + + /* Get the file access property list */ + if((fapl = H5Fget_access_plist(hdfid)) < 0) + {stat = NC_EHDFERR; goto done;} memset(&mem,0,sizeof(mem)); herr = H5Pget_file_image(fapl, &mem.memory, &mem.size); if(herr < 0) @@ -106,7 +110,8 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5) /* Close FAPL */ if (H5Pclose(fapl) < 0) {stat = NC_EHDFERR; goto done;} - h5->mem.fapl = 0; + done: return stat; } +#endif diff --git a/libhdf5/nc4memcb.c b/libhdf5/nc4memcb.c index 84cb00f79..2af02e4af 100644 --- a/libhdf5/nc4memcb.c +++ b/libhdf5/nc4memcb.c @@ -19,7 +19,7 @@ /* * @internal Inmemory support * - * This code is derived from H5Lt.c#H5LTopen_file_image. + * This code is derived from H5LT.c#H5LTopen_file_image. * In order to make the netcdf inmemory code work, it is necessary * to modify some of the callback functions; specifically * image_malloc, image_realloc, and image_memcpy. @@ -96,7 +96,8 @@ #endif #undef TRACE -#define CATCH +#undef CATCH +#undef TRACE_UDATA #ifdef TRACE #define CATCH @@ -104,18 +105,23 @@ #ifdef TRACE #include -static void trace(const char* fcn, void* _udata, ...); -static void traceend(const char* fcn, void* _udata); +static void trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...); +static void traceend(const char* fcn, void* _udata, uintptr_t retval); +static const char* traceop(H5FD_file_image_op_t op); +static char* traceflags(int flags); + /* In case we do not have variadic macros */ -#define TRACE1(fcn,udata,x1) trace(fcn,udata,x1) -#define TRACE2(fcn,udata,x1,x2) trace(fcn,udata,x1,x2) -#define TRACE3(fcn,udata,x1,x2,x3) trace(fcn,udata,x1,x2,x3) -#define TRACEEND(fcn,udata) traceend(fcn,udata); +#define TRACE0(fcn,op,udata) trace(fcn,op,udata) +#define TRACE1(fcn,op,udata,x1) trace(fcn,op,udata,x1) +#define TRACE2(fcn,op,udata,x1,x2) trace(fcn,op,udata,x1,x2) +#define TRACE3(fcn,op,udata,x1,x2,x3) trace(fcn,op,udata,x1,x2,x3) +#define TRACEEND(fcn,udata,retval) traceend(fcn,udata,(uintptr_t)retval); #else /*!TRACE*/ -#define TRACE1(fcn,udata,x1) -#define TRACE2(fcn,udata,x1,x2) -#define TRACE3(fcn,udata,x1,x2,x3) -#define TRACEEND(fcn,udata) +#define TRACE0(fcn,op,udata) +#define TRACE1(fcn,op,udata,x1) +#define TRACE2(fcn,op,udata,x1,x2) +#define TRACE3(fcn,op,udata,x1,x2,x3) +#define TRACEEND(fcn,udata,retval) #endif #ifdef CATCH @@ -235,20 +241,26 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; void * return_value = NULL; - TRACE1("malloc",_udata, size); + TRACE1("malloc", file_image_op, _udata, size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch ( file_image_op ) { /* the app buffer is "copied" to only one FAPL. Afterwards, FAPLs can be "copied" */ case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: + /* It appears that the fapl memory is never created as we use it, so + we expect the udata ptr to be either null or same as the app buffer.*/ + assert(udata->fapl_image_ptr == NULL || udata->fapl_image_ptr == udata->app_image_ptr); + if (udata->app_image_ptr == NULL) goto out; if (udata->app_image_size != size) goto out; - if (udata->fapl_image_ptr != NULL) + if (udata->fapl_image_ptr != NULL) goto out; if (udata->fapl_image_size != 0) goto out; @@ -259,6 +271,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata udata->fapl_image_size = udata->app_image_size; return_value = udata->fapl_image_ptr; udata->fapl_ref_count++; + return_value = udata->fapl_image_ptr; break; case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -273,10 +286,10 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata udata->fapl_ref_count++; break; - case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: - if(!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) + case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET: + if (udata->fapl_image_ptr == NULL) goto out; - /* fake the malloc by returning the original memory */ + /* fake the malloc by returning the current memory */ return_value = udata->fapl_image_ptr; break; @@ -310,7 +323,7 @@ local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata goto out; } /* end switch */ - TRACEEND("malloc",_udata); + TRACEEND("malloc",_udata,return_value); return(return_value); @@ -343,11 +356,13 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - TRACE3("memcpy",_udata,dest,src,size); + TRACE3("memcpy", file_image_op, _udata,dest,src,size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch(file_image_op) { case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: @@ -361,6 +376,15 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ goto out; if (udata->fapl_ref_count == 0) goto out; + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { + if(src != dest) { + memcpy(dest,src,size); +#ifdef TRACE + fprintf(stderr,"\t>>>> memcpy(%p,%p,%ld)\n", + dest,src,(unsigned long)size); +#endif + } + } break; case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -406,10 +430,7 @@ local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_ goto out; } /* end switch */ -#ifdef TRACE - fprintf(stderr,"trace: memcpy: dest=%p\n",dest); -#endif - TRACEEND("memcpy",_udata); + TRACEEND("memcpy",_udata,dest); return(dest); out: @@ -433,61 +454,85 @@ out: * *------------------------------------------------------------------------- */ + +/* This warning is from H5FDcore.c" + Be careful of non-Posix realloc() that doesn't understand + what to do when the first argument is null. +*/ + +/* Modified: +1. If the realloc new size is <= existing size, + then pretend we did a realloc and return success. + This avoids unneccessary heap operations. +2. If the H5LT_FILE_IMAGE_DONT_COPY or + H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the + realloc new size is > existing size, then fail + because the realloc() call may change the address + of the buffer. The new address cannot be + communicated to the application to release it. +3. Otherwise, use realloc(). Note that this may have the + side effect of freeing the previous memory chunk. +*/ static void * local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; void * return_value = NULL; - TRACE2("realloc",_udata, ptr, size); + TRACE2("realloc", file_image_op, _udata, ptr, size); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif /* realloc() is not allowed if the image is open in read-only mode */ if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW)) goto out; + /* DONT_COPY => DONT_RELEASE */ + assert(((udata->flags & H5LT_FILE_IMAGE_DONT_COPY)?(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE):1)); + + /* Note that the fapl pointer is never realloc'd */ if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) { - if (udata->vfd_image_ptr != ptr) - goto out; - if (udata->vfd_ref_count != 1) - goto out; - - /* Modified: - 1. If the realloc new size is <= existing size, - then pretend we did a realloc and return success. - This avoids unneccessary heap operations. - 2. If the H5LT_FILE_IMAGE_DONT_COPY or - H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the - realloc new size is > existing size, then fail - because the realloc() call may change the address - of the buffer. The new address cannot be - communicated to the application to release it. - 3. Otherwise, use realloc(). Note that this may have the - side effect of freeing the previous memory chunk. - */ - if(size <= udata->vfd_image_size) { - /* Ok, pretend we did a realloc */ - } else if((udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE) - || (udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { - goto out; /* realloc MAY violate these flags */ - } else { - if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size))) - goto out; - udata->vfd_image_size = size; + if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) { /* buffer modification is allowed */ + /* Divide code based on whether ptr == NULL or not */ + if(ptr == NULL) { + /* From realloc man page: If ptr is NULL, then the call is equivalent to malloc(size), + for all values of size; if size is equal to zero, and ptr is not NULL, then the call + is equivalent to free(ptr). */ + udata->vfd_image_ptr = malloc(size); + udata->vfd_ref_count++; + } else { /* ptr != NULL */ + if(udata->vfd_image_ptr != ptr) + goto out; + if (udata->vfd_ref_count != 1) + goto out; + udata->vfd_image_ptr = realloc(ptr, size); + if(NULL == udata->vfd_image_ptr) { + LOG((0,"image_realloc: unable to allocate memory block of size: %lu bytes",(unsigned long)size)); + goto out; + } +#ifdef TRACE + fprintf(stderr,"\t>>>> realloc(%p,%ld)=>%p\n",ptr,(unsigned long)size,udata->vfd_image_ptr); +#endif + } + udata->vfd_image_size = size; + } else { /* Cannot realloc, so fake it */ + if(size <= udata->vfd_image_size) { + /* Ok, pretend we did a realloc but just change size*/ + udata->vfd_image_size = size; + } else + goto out; } return_value = udata->vfd_image_ptr; } /* end if */ else goto out; -#ifdef TRACE - fprintf(stderr,"trace: realloc: return=%p\n",return_value); -#endif - TRACEEND("realloc",_udata); + TRACEEND("realloc",_udata,return_value); return(return_value); out: @@ -515,11 +560,13 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - TRACE1("free",_udata,ptr); + TRACE1("free", file_image_op, _udata, ptr); +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif switch(file_image_op) { case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE: @@ -530,14 +577,15 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) udata->fapl_ref_count--; - /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ - if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && - !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { - free(udata->fapl_image_ptr); - udata->app_image_ptr = NULL; - udata->fapl_image_ptr = NULL; - udata->vfd_image_ptr = NULL; - } /* end if */ + /* For the way we use it, it should still be the case that + the fapl pointer is same as image_ptr, so we do not need + to do anything */ + assert(udata->fapl_image_ptr == udata->app_image_ptr); + /* clean up */ +#if 0 + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; +#endif break; case H5FD_FILE_IMAGE_OP_FILE_CLOSE: @@ -548,17 +596,17 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) udata->vfd_ref_count--; - /* release the shared buffer only if indicated by the respective flag and there are no outstanding references */ - if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 && - !(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) { - free(udata->vfd_image_ptr); - udata->app_image_ptr = NULL; - udata->fapl_image_ptr = NULL; - udata->vfd_image_ptr = NULL; - } /* end if */ + udata->h5->mem.memio.memory = udata->vfd_image_ptr; + udata->h5->mem.memio.size = udata->vfd_image_size; + /* clean up */ +#if 0 + udata->app_image_ptr = NULL; + udata->fapl_image_ptr = NULL; +#endif + udata->vfd_image_ptr = NULL; break; - /* added unused labels to keep the compiler quite */ + /* added unused labels to keep the compiler quiet */ case H5FD_FILE_IMAGE_OP_NO_OP: case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET: case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY: @@ -569,7 +617,7 @@ local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata) goto out; } /* end switch */ - TRACEEND("free",_udata); + TRACEEND("free",_udata,1); return(SUCCEED); out: @@ -598,18 +646,24 @@ local_udata_copy(void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + TRACE0("udata_copy", 0, _udata); + +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif - if (udata->ref_count == 0) - goto out; - + /* never copy */ + if (udata->ref_count == 0) + goto out; udata->ref_count++; + TRACEEND("udata_copy",udata,1); return(udata); out: + TRACEFAIL("udata_copy"); return NULL; } /* end udata_copy */ @@ -634,63 +688,75 @@ local_udata_free(void *_udata) { H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; + TRACE0("udata_free", 0, _udata); + +#if 0 /* callback is only used if the application buffer is not actually copied */ if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) goto out; +#endif if (udata->ref_count == 0) goto out; - udata->ref_count--; /* checks that there are no references outstanding before deallocating udata */ - if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && - udata->vfd_ref_count == 0) + if (udata->ref_count == 0 && udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0) { free(udata); - +#ifdef TRACE_UDATA + fprintf(stderr,"\t>>>> freed: udata=%p\n",udata); +#endif + udata = NULL; + } + TRACEEND("udata_free",udata,1); return(SUCCEED); out: + TRACEFAIL("udata_free"); return(FAIL); } /* end udata_free */ + /* End of callbacks definitions for file image operations */ hid_t NC4_image_init(NC_FILE_INFO_T* h5) { hid_t fapl = -1, file_id = -1; /* HDF5 identifiers */ - unsigned file_open_flags;/* Flags for image open */ + unsigned file_open_flags = 0;/* Flags for hdf5 open */ char file_name[64]; /* Filename buffer */ size_t alloc_incr; /* Buffer allocation increment */ size_t min_incr = 65536; /* Minimum buffer increment */ double buf_prcnt = 0.1f; /* Percentage of buffer size to set as increment */ size_t buf_size = 0; - void* buf_ptr = h5->mem.memio.memory; - unsigned flags = h5->mem.flags; + void* buf_ptr = NULL; + unsigned imageflags; + int create = 0; + H5LT_file_image_ud_t *udata = NULL; /* Pointer to udata structure */ - static long file_name_counter; H5FD_file_image_callbacks_t callbacks = {&local_image_malloc, &local_image_memcpy, &local_image_realloc, &local_image_free, &local_udata_copy, &local_udata_free, (void *)NULL}; + static long file_name_counter; + + imageflags = h5->mem.imageflags; + create = h5->mem.created; + /* check arguments */ - if (buf_ptr == NULL) { - if(h5->mem.created) { + if (h5->mem.memio.memory == NULL) { + if(create) { if(h5->mem.memio.size == 0) h5->mem.memio.size = DEFAULT_CREATE_MEMSIZE; h5->mem.memio.memory = malloc(h5->mem.memio.size); } else goto out; /* open requires an input buffer */ - } - // reset + } else if(h5->mem.memio.size == 0) + goto out; + + /* alias for convenience */ buf_size = h5->mem.memio.size; buf_ptr = h5->mem.memio.memory; - /* validate */ - if (buf_ptr == NULL || buf_size == 0) - goto out; - if (flags & (unsigned)~(H5LT_FILE_IMAGE_ALL)) - goto out; /* Create FAPL to transmit file image */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -708,14 +774,10 @@ NC4_image_init(NC_FILE_INFO_T* h5) if (H5Pset_fapl_core(fapl, alloc_incr, FALSE) < 0) goto out; - /* Set callbacks for file image ops ONLY if the file image is NOT copied */ - if (flags & H5LT_FILE_IMAGE_DONT_COPY) - // if (0) + /* Set callbacks for file image ops always */ { - H5LT_file_image_ud_t *udata; /* Pointer to udata structure */ - /* Allocate buffer to communicate user data to callbacks */ - if (NULL == (udata = (H5LT_file_image_ud_t *)malloc(sizeof(H5LT_file_image_ud_t)))) + if (NULL == (udata = (H5LT_file_image_ud_t *)calloc(1,sizeof(H5LT_file_image_ud_t)))) goto out; /* Initialize udata with info about app buffer containing file image and flags */ @@ -727,7 +789,7 @@ NC4_image_init(NC_FILE_INFO_T* h5) udata->vfd_image_ptr = NULL; udata->vfd_image_size = 0; udata->vfd_ref_count = 0; - udata->flags = flags; + udata->flags = imageflags; udata->ref_count = 1; /* corresponding to the first FAPL */ udata->h5 = h5; @@ -735,28 +797,27 @@ NC4_image_init(NC_FILE_INFO_T* h5) callbacks.udata = (void *)udata; /* Set file image callbacks */ - if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) { - free(udata); + if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) goto out; - } /* end if */ } /* Assign file image in user buffer to FAPL */ if (H5Pset_file_image(fapl, buf_ptr, buf_size) < 0) goto out; - /* set file open flags */ - if (flags & H5LT_FILE_IMAGE_OPEN_RW) + /* define a unique file name */ + snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++); + + /* set file open/create flags */ + if(create) + file_open_flags = H5F_ACC_TRUNC; /* H5Fcreate does not like H5F_ACC_RDWR */ + else if (imageflags & H5LT_FILE_IMAGE_OPEN_RW) file_open_flags = H5F_ACC_RDWR; else file_open_flags = H5F_ACC_RDONLY; - /* define a unique file name */ - snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++); - /* Assign file image in FAPL to the core file driver */ - if(h5->mem.created) { - file_open_flags |= H5F_ACC_TRUNC; + if(create) { if ((file_id = H5Fcreate(file_name, file_open_flags, H5P_DEFAULT, fapl)) < 0) goto out; } else { @@ -764,76 +825,133 @@ NC4_image_init(NC_FILE_INFO_T* h5) goto out; } - h5->mem.fapl = fapl; + h5->mem.udata = udata; - /* Return file identifier */ - return file_id; - -out: +done: + /* Reclaim the fapl object */ H5E_BEGIN_TRY { if(fapl >= 0) H5Pclose(fapl); } H5E_END_TRY; - return -1; + /* Reclaim udata */ + /* Return file identifier */ + return file_id; + +out: + if(udata != NULL) free(udata); + file_id = -1; + goto done; } /* end H5LTopen_file_image() */ +void +NC4_image_finalize(void* _udata) +{ + if(_udata != NULL) { + H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t*)_udata; +#if 0 + if(udata->app_image_ptr != NULL) free(udata->app_image_ptr); + if(udata->fapl_image_ptr != NULL) free(udata->fapl_image_ptr); + if(udata->vfd_image_ptr != NULL) free(udata->vfd_image_ptr); +#endif + free(udata); + } +} + #ifdef TRACE -static void +static char* printudata(H5LT_file_image_ud_t* udata) { - if(udata == NULL) return; - fprintf(stderr," flags=0x%0x",udata->flags); - fprintf(stderr," flags=0x%0x",udata->flags); - fprintf(stderr," ref_count=%d",udata->ref_count); - fprintf(stderr," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size); - fprintf(stderr," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count); - fprintf(stderr," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count); + char buf[8192]; + char tmp[8192]; + char* flags = ""; + + buf[0] = '\0'; + if(udata == NULL) return strdup(""); + strlcat(buf,"flags=",sizeof(buf)); + flags = traceflags(udata->flags); + strlcat(buf,flags,sizeof(buf)); + if(flags != NULL) free(flags); + snprintf(tmp,sizeof(tmp)," ref_count=%d",udata->ref_count); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," app=(%p,%lld)",udata->app_image_ptr,(long long)udata->app_image_size); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," fapl=(%p,%lld)[%d]",udata->fapl_image_ptr,(long long)udata->fapl_image_size,udata->fapl_ref_count); + strlcat(buf,tmp,sizeof(tmp)); + snprintf(tmp,sizeof(tmp)," vfd=(%p,%lld)[%d]",udata->vfd_image_ptr,(long long)udata->vfd_image_size,udata->vfd_ref_count); + strlcat(buf,tmp,sizeof(tmp)); + return strdup(buf); } -static void trace(const char* fcn, void* _udata, ...) +static void +trace(const char* fcn, H5FD_file_image_op_t op, void* _udata, ...) { H5LT_file_image_ud_t *udata = NULL; va_list ap; + char buf[16000]; + char tmp[8192]; + char* ud; va_start(ap, _udata); /* Requires the last fixed parameter (to get the address) */ udata = (H5LT_file_image_ud_t *)_udata; + buf[0] = '\0'; + snprintf(tmp,sizeof(tmp),"trace [ %s: op=%s: ",fcn,traceop(op)); + strlcat(buf,tmp,sizeof(tmp)); if(strcmp("malloc",fcn)==0) { size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: size=%lld udata=",fcn,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"size=%lld",(long long)size); } else if(strcmp("realloc",fcn)==0) { void* ptr = va_arg(ap,void*); size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: ptr=%p, size=%lld udata=",fcn,ptr,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"ptr=%p, size=%lld",ptr,(long long)size); } else if(strcmp("free",fcn)==0) { void* ptr = va_arg(ap,void*); - fprintf(stderr,"trace: %s: ptr=%p udata=",fcn,ptr); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"ptr=%p",ptr); } else if(strcmp("memcpy",fcn)==0) { void* dest = va_arg(ap,void*); void* src = va_arg(ap,void*); size_t size = va_arg(ap,size_t); - fprintf(stderr,"trace: %s: dest=%p src=%p size=%lld udata=",fcn,dest,src,(long long)size); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"dest=%p, src=%p, size=%lld",dest,src,(long long)size); + } else if(strcmp("udata_copy",fcn)==0) { +#ifdef TRACE_UDATA + snprintf(tmp,sizeof(tmp),"udata=%p",udata); +#endif + } else if(strcmp("udata_free",fcn)==0) { +#ifdef TRACE_UDATA + snprintf(tmp,sizeof(tmp),"udata=%p",udata); +#endif } else { - fprintf(stderr,"unknown fcn: %s\n",fcn); + snprintf(tmp,sizeof(tmp),"unknown fcn: %s",fcn); } + strlcat(buf,tmp,sizeof(buf)); + ud = printudata(udata); + strlcat(buf,"\n\tudata=",sizeof(buf)); + strlcat(buf,ud,sizeof(tmp)); + free(ud); + strlcat(buf,"\n",sizeof(buf)); va_end(ap); + fprintf(stderr,"%s",buf); fflush(stderr); } -static void traceend(const char* fcn, void* _udata) +static void +traceend(const char* fcn, void* _udata, uintptr_t retval) { + char buf[16000]; + char tmp[8192]; + char* ud; + const char* tab = " "; + + buf[0] = '\0'; H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata; - fprintf(stderr,"traceend: %s: udata=",fcn); - printudata(udata); - fprintf(stderr,"\n"); + snprintf(tmp,sizeof(tmp),"%s]: retval=%p",tab,(void*)retval); + strlcat(buf,tmp,sizeof(buf)); + strlcat(buf," udata=",sizeof(buf)); + ud = printudata(udata); + strlcat(buf,ud,sizeof(tmp)); + free(ud); + strlcat(buf,"\n",sizeof(buf)); + fprintf(stderr,"%s",buf); fflush(stderr); } @@ -843,7 +961,68 @@ static void traceend(const char* fcn, void* _udata) static void tracefail(const char* fcn) { - fprintf(stderr,"fail: %s",fcn); + fprintf(stderr,"fail: %s\n",fcn); fflush(stderr); } #endif /*CATCH*/ + +#ifdef TRACE +static char* +traceflags(int flags) +{ + int i; + char buf[8192]; + char tmp[8192]; + buf[0] = '\0'; + for(i=0;i<16;i++) { + tmp[0] = '\0'; + if((flags & 1< 0) strlcat(tmp,"|",sizeof(tmp)); + switch(1< #include #include -#define access(path,mode) _access(path,mode) #endif #include "ncdispatch.h" @@ -116,6 +115,8 @@ static int memio_close(ncio* nciop, int); static int readfile(const char* path, NC_memio*); static int writefile(const char* path, NCMEMIO*); static int fileiswriteable(const char* path); +static int fileisreadable(const char* path); +static int fileexists(const char* path); /* Mnemonic */ #define DOOPEN 1 @@ -197,7 +198,7 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM if(fIsSet(ioflags,NC_DISKLESS)) memio->diskless = 1; - if(fIsSet(ioflags,NC_INMEMORY) && !memio->diskless) + if(fIsSet(ioflags,NC_INMEMORY)) memio->inmemory = 1; if(fIsSet(ioflags,NC_WRITE) && !fIsSet(ioflags,NC_NOCLOBBER) && memio->diskless) memio->persist = 1; @@ -248,8 +249,8 @@ memio_create(const char* path, int ioflags, return status; if(memio->persist) { - /* Verify the file is writeable */ - if(!fileiswriteable(path)) + /* Verify the file is writeable or does not exist*/ + if(fileexists(path) && !fileiswriteable(path)) {status = EPERM; goto unwind_open;} } @@ -374,9 +375,11 @@ fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->mem #endif if(memio->persist) { - /* Verify the file is writeable */ + /* Verify the file is writeable and exists */ + if(!fileexists(path)) + {status = ENOENT; goto unwind_open;} if(!fileiswriteable(path)) - {status = EPERM; goto unwind_open;} + {status = EACCES; goto unwind_open;} } /* Use half the filesize as the blocksize ; why? */ @@ -645,11 +648,39 @@ memio_extract(ncio* const nciop, size_t* sizep, void** memoryp) return status; } +/* Return 1 if file exists, 0 otherwise */ +static int +fileexists(const char* path) +{ + int ok; + /* See if the file exists at all */ + ok = NCaccess(path,ACCESS_MODE_EXISTS); + if(ok < 0) /* file does not exist */ + return 0; + return 1; +} + +/* Return 1 if file is writeable, return 0 otherwise; + assumes fileexists has been checked already */ static int fileiswriteable(const char* path) { int ok; - ok = access(path,O_RDWR); + /* if W is ok */ + ok = NCaccess(path,ACCESS_MODE_W); + if(ok < 0) + return 0; + return 1; +} + +/* Return 1 if file is READABLE, return 0 otherwise; + assumes fileexists has been checked already */ +static int +fileisreadable(const char* path) +{ + int ok; + /* if RW is ok */ + ok = NCaccess(path,ACCESS_MODE_R); if(ok < 0) return 0; return 1; @@ -714,14 +745,14 @@ writefile(const char* path, NCMEMIO* memio) size_t count = 0; char* p = NULL; - /* Open the file for writing*/ + /* Open/create the file for writing*/ #ifdef _MSC_VER - f = NCfopen(path,"rwb"); + f = NCfopen(path,"wb"); #else - f = NCfopen(path,"rw"); + f = NCfopen(path,"w"); #endif if(f == NULL) - {status = errno; goto done;} + {status = errno; goto done;} rewind(f); count = memio->size; p = memio->memory; diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index 79c6fbb1e..4f689fd0b 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -1044,7 +1044,7 @@ NC3_create(const char *path, int ioflags, int use_parallel, void* parameters, NC_Dispatch* dispatch, NC* nc) { - int status; + int status = NC_NOERR; void *xp = NULL; int sizeof_off_t = 0; NC3_INFO* nc3 = NULL; diff --git a/libsrc4/error4.c b/libsrc4/error4.c index 30f3d94ea..bdd2345a5 100644 --- a/libsrc4/error4.c +++ b/libsrc4/error4.c @@ -32,7 +32,7 @@ extern int nc_log_level; nc_log(0, "this computer will explode in %d seconds", i); After the first arg (the severity), use the rest like a normal - printf statement. Output will appear on stdout. + printf statement. Output will appear on stderr. This function is heavily based on the function in section 15.5 of the C FAQ. */ @@ -50,18 +50,18 @@ nc_log(int severity, const char *fmt, ...) /* If the severity is zero, this is an error. Otherwise insert that many tabs before the message. */ if (!severity) - fprintf(stdout, "ERROR: "); + fprintf(stderr, "ERROR: "); for (t=0; t ${FILE3}.cdl ${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl -if ! diff -wb ${FILE3}.cdl ${CREATE3}.cdl ; then - echo "FAIL: ${FILE3}.cdl vs ${CREATE3}.cdl -fi +diff -wb ${FILE3}.cdl ${CREATE3}.cdl if test "x$HASNC4" = "xyes" ; then ${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl ${NCDUMP} ${CREATE4}.nc > ${CREATE4}.cdl -if diff -wb ${FILE4}.cdl ${CREATE4}.cdl ; then - echo "FAIL: ${FILE4}.cdl vs ${CREATE4}.cdl -fi +diff -wb ${FILE4}.cdl ${CREATE4}.cdl # cleanup rm -f ${FILE3}.nc ${FILE4}.nc ${CREATE3}.nc ${CREATE4}.nc diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index c5da2c0ae..321de11f6 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -7,12 +7,12 @@ redistribution conditions. #undef DDBG -#include -#include -#include "err_macros.h" +#include "config.h" #include #include -#include +#include "netcdf.h" +#include "nc_tests.h" +#include "err_macros.h" /* netcdf tst_diskless { @@ -47,17 +47,12 @@ void fail(int line) { /* Control flags */ static int flags, persist, usenetcdf4, mmap; +/* Remove a file; do not care if it does not exist */ static void removefile(int persist, char* filename) { if(persist) { - if(remove(filename) != 0) { - if(errno != ENOENT) { - fprintf(stderr,"Could not remove file: %s: %d\n",filename,errno); - perror(""); - exit(1); - } - } + remove(filename); } } diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index c91fd4458..76e0e199b 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -442,11 +442,13 @@ test_open(const char* path, NC_memio* filedata, int mode) { int stat = NC_NOERR; NC_memio duplicate; - NC_memio* finaldata = NULL; + NC_memio finaldata; int ncid; int xmode = mode; /* modified mode */ - finaldata = calloc(1,sizeof(NC_memio)); + finaldata.memory = NULL; + finaldata.size = 0; + finaldata.flags = 0; fprintf(stderr,"\n\t***Test open 1: nc_open_mem(): read-only\n"); CHECK(duplicatememory(filedata,&duplicate,0)); @@ -460,13 +462,13 @@ test_open(const char* path, NC_memio* filedata, int mode) duplicate.flags = NC_MEMIO_LOCKED; CHECK(nc_open_memio(path, xmode, &duplicate, &ncid)) CHECK(verify_file(ncid,!MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Verify that finaldata is same */ - if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); - if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); - free(finaldata->memory); + if(finaldata.size != duplicate.size) CHECK(NC_EINVAL); + if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata.memory); finaldata.memory = NULL; fprintf(stderr,"\n\t***Test open 3: nc_open_memio(): read-write, copy\n"); xmode |= NC_WRITE; /* allow file to be modified */ @@ -475,13 +477,13 @@ test_open(const char* path, NC_memio* filedata, int mode) /* modify file */ CHECK(modify_file(ncid)); CHECK(verify_file(ncid,MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Verify that finaldata is same */ - if(finaldata->size < filedata->size) CHECK(NC_EINVAL); + if(finaldata.size < filedata->size) CHECK(NC_EINVAL); /* As a safeguard, the memory in duplicate should have been set to NULL*/ - free(finaldata->memory); + free(finaldata.memory); finaldata.memory = NULL; fprintf(stderr,"\n\t***Test open 4: nc_open_memio(): read-write, locked, extra space\n"); /* Store the filedata in a memory chunk that leaves room for modification */ @@ -493,14 +495,15 @@ test_open(const char* path, NC_memio* filedata, int mode) /* modify file */ CHECK(modify_file(ncid)); CHECK(verify_file(ncid,MODIFIED)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); - /* Check returned finaldata */ - if(finaldata->size != duplicate.size) CHECK(NC_EINVAL); - if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL); - free(finaldata->memory); - + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); + /* Check returned finaldata: + should have same memory but + actual used final size should not exceed the original */ + if(finaldata.size > duplicate.size) CHECK(NC_EINVAL); + if(finaldata.memory != duplicate.memory) CHECK(NC_EINVAL); + free(finaldata.memory); finaldata.memory = NULL; return stat; } @@ -508,33 +511,36 @@ static int test_create(const char* path, int mode) { int stat = NC_NOERR; - NC_memio* finaldata = NULL; + NC_memio finaldata; int ncid; int xmode = mode; + finaldata.memory = NULL; + finaldata.size = 0; + finaldata.flags = 0; + fprintf(stderr,"\n\t***Test create 1: nc_create_memio(): no initialsize\n"); CHECK(nc_create_mem(path, xmode, 0, &ncid)) /* create file metadata */ CHECK(define_metadata(ncid)); CHECK(verify_file(ncid,!MODIFIED)); - finaldata = calloc(1,sizeof(NC_memio)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); - free(finaldata->memory); - + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); + free(finaldata.memory); fprintf(stderr,"\n\t***Test create 2: nc_create_memio(): initialsize; save file\n"); CHECK(nc_create_mem(path, xmode, LARGE_SPACE, &ncid)) /* create file metadata */ CHECK(define_metadata(ncid)); CHECK(verify_file(ncid,!MODIFIED)); - finaldata = calloc(1,sizeof(NC_memio)); - CHECK(nc_close_memio(ncid,finaldata)); + CHECK(nc_close_memio(ncid,&finaldata)); /* Published returned finaldata */ - fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory); + fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata.size,finaldata.memory); /* Write out the final data as a .nc file */ - CHECK(writefile(path,finaldata)); - free(finaldata->memory); + CHECK(writefile(path,&finaldata)); + if(finaldata.memory != NULL) + free(finaldata.memory); + finaldata.memory = NULL; return stat; } @@ -584,15 +590,17 @@ test_xfail(const char* path, int mode, NC_memio* filedata) /* With HDF5 1.8.20, and possibly other versions, this tests causes a seg fault in the HDF5 Library. So until it is fixed, just leave well enough alone */ - NC_memio* finaldata = NULL; - finaldata = calloc(1,sizeof(NC_memio)); + NC_memio finaldata; + memset(&finaldata,0,sizeof(finaldata)); CHECK(duplicatememory(filedata,&duplicate,0)); duplicate.flags = NC_MEMIO_LOCKED; xmode |= NC_WRITE; CHECK(nc_open_memio(XFAIL, xmode, &duplicate, &ncid)) XCHECK(modify_file(ncid)); CHECK(nc_abort(ncid)); - free(finaldata->memory); + if(finaldata.memory != NULL) + free(finaldata.memory); + finaldata.memory = NULL; } return stat; @@ -605,11 +613,6 @@ main(int argc, char **argv) NC_memio filedata3; NC_memio filedata4; -#ifdef USE_NETCDF4 - nc_set_log_level(0); - H5Eprint1(stderr); -#endif - fprintf(stderr,"\n*** Testing the inmemory API: netcdf-3.\n"); CHECK(create_reference_file(FILE3,NC_NETCDF3,&filedata3)); /* netcdf-3 */ CHECK(test_open(FILE3,&filedata3,NC_NETCDF3)); diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 1bb5a72a7..ae20d6cff 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -60,5 +60,5 @@ ADD_SUBDIRECTORY(expectremote3) ## Specify files to be distributed by 'make dist' FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) +SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} findtestserver.c.in CMakeLists.txt Makefile.am) ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 9f64f0ac1..8e85b59de 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -83,12 +83,14 @@ EXTRA_DIST = tst_ncdap3.sh \ tst_longremote3.sh \ tst_filelists.sh tst_urls.sh tst_utils.sh \ t_dap.c CMakeLists.txt tst_formatx.sh testauth.sh testurl.sh \ - t_ncf330.c tst_ber.sh + t_ncf330.c tst_ber.sh findtestserver.c.in CLEANFILES = test_varm3 test_cvt3 file_results/* remote_results/* datadds* t_dap3a test_nstride_cached *.exe # This should only be left behind if using parallel io CLEANFILES += tmp_* +DISTCLEANFILES = findtestserver4.c + # This rule are used if someone wants to rebuild t_dap3a.c # Otherwise never invoked, but records how to do it. t_dap3a.c: t_dap.c diff --git a/ncdap_test/findtestserver.c b/ncdap_test/findtestserver.c index 69a324cbb..9ad5e697e 100644 --- a/ncdap_test/findtestserver.c +++ b/ncdap_test/findtestserver.c @@ -8,6 +8,7 @@ #define XSTRINGIFY(s) #s #define STRINGIFY(s) XSTRINGIFY(s) + /** usage: findtestserver dap2|dap4 suffix [serverlist] diff --git a/ncdap_test/findtestserver.c.in b/ncdap_test/findtestserver.c.in new file mode 100644 index 000000000..9ad5e697e --- /dev/null +++ b/ncdap_test/findtestserver.c.in @@ -0,0 +1,79 @@ +#include "config.h" +#include +#include +#include +#include "nctestserver.h" + +/* Support stringification of -D macros */ +#define XSTRINGIFY(s) #s +#define STRINGIFY(s) XSTRINGIFY(s) + + +/** +usage: findtestserver dap2|dap4 suffix [serverlist] + +Given a partial suffix path, try to find a +server for which a request to server + suffix +returns some kind of result using the +specified protocol. This indicates that the +server is up and running. Return the complete +url for the server plus the path. +If serverlist is present, then is should be a comma +separated list of servers (host+port) to try. +It defaults to REMOTETESTSERVERS. +*/ + +static void +usage() +{ + fprintf(stderr,"usage: findtestserver dap2|dap4 suffix [serverlist]\n"); + exit(1); +} + + +int +main(int argc, char** argv) +{ + const char* url = NULL; + const char* servlet = NULL; + const char* proto = NULL; + const char* serverlist = NULL; + int isdap4 = 0; /* 1 => dap4 */ + + argc--; argv++; + if(argc < 2) + usage(); + proto = strdup(argv[0]); + servlet = strdup(argv[1]); + if(argc >= 3) + serverlist = strdup(argv[2]); + +#ifdef ENABLE_DAP + if(strcasecmp(proto,"dap2")==0) + isdap4 = 0; + else +#endif +#ifdef ENABLE_DAP4 + if(strcasecmp(proto,"dap4")==0) + isdap4 = 1; + else +#endif + usage(); + + if(serverlist == NULL) { +#ifdef REMOTETESTSERVERS + serverlist = strdup(REMOTETESTSERVERS); +#endif + } + if(serverlist == NULL || strlen(serverlist) == 0) + fprintf(stderr,"Cannot determine a server list"); + + url = nc_findtestserver(servlet,isdap4,serverlist); + if(url == NULL) { + url = ""; + fprintf(stderr,"not found: %s\n",servlet); + } + printf("%s",url); + fflush(stdout); + exit(0); +} diff --git a/ncdap_test/testurl.sh b/ncdap_test/testurl.sh index 95ba9eb30..b851e8c79 100755 --- a/ncdap_test/testurl.sh +++ b/ncdap_test/testurl.sh @@ -56,7 +56,7 @@ locreset if test "x$NOP" != x1 ; then echo "***Testing url prefix parameters" -buildurl $PREFIX "" +buildurl "$PREFIX" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" @@ -66,7 +66,7 @@ if test "x${SHOW}" = x1 ; then cat ./tmp ; fi # Test that maxstrlen works as alias for stringlength echo "***Testing maxstrlen=stringlength alias" -buildurl $STRLEN "" +buildurl "$STRLEN" "" # Invoke ncdump to extract the URL echo "command: ${NCDUMP} -h $url" ${NCDUMP} "$url" >./tmp_testurl 2> ./errtmp_testurl @@ -82,7 +82,7 @@ fi locreset if test "x$NOS" != x1 ; then echo "***Testing url suffix parameters" -buildurl "" $SUFFIX +buildurl "" "$SUFFIX" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi @@ -92,7 +92,7 @@ locreset if test "x$NOB" != x1 ; then echo "***Testing url prefix+suffix parameters" -buildurl $BOTHP $BOTHS +buildurl "$BOTHP" "$BOTHS" # Invoke ncdump to extract the URL ${NCDUMP} -h "$url" >./tmp_testurl 2> ./errtmp_testurl if test "x${SHOW}" = x1 ; then cat ./tmp_testurl ; fi diff --git a/oc2/ochttp.c b/oc2/ochttp.c index 3f0d772f7..cbb1b76cb 100644 --- a/oc2/ochttp.c +++ b/oc2/ochttp.c @@ -312,7 +312,7 @@ ocping(const char* url) if (cstat != CURLE_OK) goto done; - /* use a very short timeout: 10 seconds */ + /* use a very short timeout: 5 seconds */ cstat = CURLERR(curl_easy_setopt(curl, CURLOPT_TIMEOUT, (long)5)); if (cstat != CURLE_OK) goto done;