mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-25 17:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

re: esupport MQO-415619

Browse Source 
and https://github.com/Unidata/netcdf-c/issues/708

Expand the NC_INMEMORY capabilities to support writing and accessing
the final modified memory.

Three new functions have been added:
nc_open_memio, nc_create_mem, and nc_close_memio.

The following new capabilities were added.
1. nc_open_memio() allows the NC_WRITE mode flag
   so a chunk of memory can be passed in and be modified
2. nc_create_mem() allows the NC_INMEMORY flag to be set
   to cause the created file to be kept in memory.
3. nc_close_mem() allows the final in-memory contents to be
   retrieved at the time the file is closed.
4. A special flag, NC_MEMIO_LOCK, is provided to ensure that
   the provided memory will not be freed or reallocated.

Note the following.
1. If nc_open_memio() is called with NC_WRITE, and NC_MEMIO_LOCK is not set,
   then the netcdf-c library will take control of the incoming memory.
   This means that the original memory block should not be freed
   but the block returned by nc_close_mem() must be freed.
2. If nc_open_memio() is called with NC_WRITE, and NC_MEMIO_LOCK is set,
   then modifications to the original memory may fail if the space available
   is insufficient.

Documentation is provided in the file docs/inmemory.md.
A test case is provided: nc_test/tst_inmemory.c driven by
nc_test/run_inmemory.sh

WARNING: changes were made to the dispatch table for
the close entry. From int (*close)(int) to int (*close)(int,void*).
This commit is contained in:
Dennis Heimbigner 2018-02-25 21:45:31 -07:00
parent fc6ab9881f
commit ccc70d640b
42 changed files with 2755 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cf
View File

@ -105,7 +105,7 @@ FLAGS="$FLAGS --enable-extreme-numbers"
#FLAGS="$FLAGS --disable-testsets"
#FLAGS="$FLAGS --disable-dap-remote-tests"
#FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
#FLAGS="$FLAGS --enable-doxygen --enable-internal-docs"
FLAGS="$FLAGS --enable-doxygen --enable-internal-docs"
FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --enable-mmap"

2
configure.ac
View File

@ -799,7 +799,7 @@ fi
# Setup the diskless and mmap conditionals
if test "x$enable_diskless" = xyes ; then
AC_DEFINE([USE_DISKLESS], [1], [if true, include NC_DISKLESS code])
AC_DEFINE([USE_DISKLESS], [1], [if true, include NC_DISKLESS and NC_INMEMORY code])
if test "x$enable_mmap" = xyes; then
AC_DEFINE([USE_MMAP], [1], [if true, use mmap for in-memory files])
fi

11
docs/CMakeLists.txt
View File

@ -74,6 +74,15 @@ IF(ENABLE_DOXYGEN)
ENDIF(ENABLE_DOXYGEN)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am netcdf.m4 DoxygenLayout.xml Doxyfile.in Doxyfile.guide.in footer.html mainpage.dox tutorial.dox guide.dox types.dox notes.md cdl.dox architecture.dox internal.dox install-fortran.dox Doxyfile.in.cmake windows-binaries.md building-with-cmake.md install.md)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST}
netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html
mainpage.dox tutorial.dox guide.dox types.dox cdl.dox
architecture.dox internal.dox windows-binaries.md
building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md
install-fortran.md all-error-codes.md credits.md auth.md
obsolete/fan_utils.html bestpractices.md filters.md inmemory.md
DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md
file_format_specifications.md known_problems.md
COPYRIGHT.dox)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

6
docs/Doxyfile.in
View File

@ -750,23 +750,23 @@ INPUT = \
@abs_top_srcdir@/docs/install-fortran.md \
@abs_top_srcdir@/docs/types.dox \
@abs_top_srcdir@/docs/internal.dox \
@abs_top_srcdir@/docs/indexing.dox \
@abs_top_srcdir@/docs/windows-binaries.md \
@abs_top_srcdir@/docs/guide.dox \
@abs_top_srcdir@/docs/OPeNDAP.dox \
@abs_top_srcdir@/docs/DAP4.dox \
@abs_top_srcdir@/docs/attribute_conventions.md \
@abs_top_srcdir@/docs/file_format_specifications.md \
@abs_top_srcdir@/docs/tutorial.dox \
@abs_top_srcdir@/docs/notes.md \
@abs_top_srcdir@/docs/auth.md \
@abs_top_srcdir@/docs/filters.md \
@abs_top_srcdir@/docs/inmemory.md \
@abs_top_srcdir@/docs/notes.md \
@abs_top_srcdir@/docs/all-error-codes.md \
@abs_top_srcdir@/docs/FAQ.md \
@abs_top_srcdir@/docs/known_problems.md \
@abs_top_srcdir@/docs/COPYRIGHT.dox \
@abs_top_srcdir@/docs/credits.md \
@abs_top_srcdir@/docs/bestpractices.md \
@abs_top_srcdir@/docs/tutorial.dox \
@abs_top_srcdir@/include/netcdf.h \
@abs_top_srcdir@/include/netcdf_mem.h \
@abs_top_srcdir@/include/netcdf_par.h \

5
docs/Makefile.am
View File

@ -9,7 +9,10 @@ mainpage.dox tutorial.dox guide.dox types.dox cdl.dox \
architecture.dox internal.dox windows-binaries.md \
building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md \
install-fortran.md all-error-codes.md credits.md auth.md \
obsolete/fan_utils.html bestpractices.md filters.md indexing.dox
obsolete/fan_utils.html bestpractices.md filters.md inmemory.md \
DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md \
file_format_specifications.md known_problems.md \
COPYRIGHT.dox
# Turn off parallel builds in this directory.
.NOTPARALLEL:

12
docs/auth.md
View File

@ -4,10 +4,6 @@ netCDF Authorization Support
# netCDF Authorization Support {#Header}
__Author__: Dennis Heimbigner<br>
__Initial Version__: 11/21/2014<br>
__Last Revised__: 08/24/2017
[TOC]
## Introduction {#Introduction}
@ -515,3 +511,11 @@ what you changed to the author so this document can be updated.
yes | keytool -trustcacerts -storepass "$PWD" -v -keystore ./$TRUSTSTORE -alias $alias -importcert -file "${c}"
done
exit
## Point of Contact
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu
__Initial Version__: 11/21/2014<br>
__Last Revised__: 08/24/2017

15
docs/filters.md
View File

@ -1,8 +1,8 @@
Filter Support in netCDF-4 (Enhanced)
NetCDF-4 Filter Support
============================
<!-- double header is needed to workaround doxygen bug -->
Filter Support in netCDF-4 (Enhanced) {#compress}
NetCDF-4 Filter Support {#compress}
=================================
[TOC]
@ -411,10 +411,19 @@ static const unsigned char b[4] = {0x0,0x0,0x0,0x1}; /* value 1 in big-endian*/
int endianness = (1 == *(unsigned int*)b); /* 1=>big 0=>little endian
````
References {#References}
References {#Filter_References}
==========
1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf
3. https://support.hdfgroup.org/services/filters.html
4. https://support.hdfgroup.org/services/contributions.html#filters
Point of Contact
================
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu
__Initial Version__: 1/10/2018<br>
__Last Revised__: 2/5/2018

215
docs/inmemory.md Normal file
View File

@ -0,0 +1,215 @@
NetCDF In-Memory Support
====================================
<!-- double header is needed to workaround doxygen bug -->
NetCDF In-Memory Support {#inmemory}
====================================
[TOC]
Introduction {#inmemory_intro}
--------------
It can be convenient to operate on a netcdf file whose
content is held in memory instead of in a disk file.
The netcdf API has been modified in a number of ways
to support this capability.
Actually, three distinct but related capabilities are provided.
1. DISKLESS -- Read a file into memory, operate on it, and optionally
write it back out to disk when nc_close() is called.
2. INMEMORY -- Tell the netcdf-c library to treat a provided block
of memory as if it were a netcdf file. At close, it is possible to ask
for the final contents of the memory chunk. Be warned that there is
some complexity to this as described below.
4. MMAP -- Tell the netcdf-c library to use the *mmap()* operating
system functionality to access a file.
The first two capabilities are intertwined in the sense that the *diskless*
capability makes use internally of the *inmemory* capability. But, the
*inmemory* capability can be used independently of the *diskless* capability.
The *mmap()* capability provides a capability similar to *diskless* but
using special capabilities of the underlying operating system.
Note also that *diskless* and *inmemory* can be used for both
*netcdf-3* (classic) and *netcdf-4* (enhanced) data. The *mmap*
capability can only be used with *netcdf-3*.
Enabling Diskless File Access {#Enable_Diskless}
--------------
The *diskless* capability can be used relatively transparently
using the *NC_DISKLESS* mode flag.
Note that since the file is stored in memory, size limitations apply.
If you are on using a 32-bit pointer then the file size must be less than 2^32
bytes in length. On a 64-bit machine, the size must be less than 2^64 bytes.
### Diskless File Open
Calling *nc_open()* using the mode flag *NC_DISKLESS* will cause
the file being opened to be read into memory. When calling *nc_close()*,
the file will optionally be re-written (aka "persisted") to disk. This
persist capability will be invoked if and only if *NC_WRITE* is specified
in the mode flags at the call to *nc_open()*.
### Diskless File Create
Calling *nc_create()* using the mode flag *NC_DISKLESS* will cause
the file to initially be created and kept in memory.
When calling *nc_close()*, the file will be written
to disk.
Note that if it is desired to create the file in memory,
but not write to a disk file, then one can either set
the NC_NOCLOBBER mode flag or one can call *nc_abort()*
instead of *nc_close()*.
Enabling Inmemory File Access {#Enable_Inmemory}
--------------
The netcdf API has been extended to support the inmemory capability.
The relevant API is defined in the file `netcdf_mem.h`.
The important data structure to use is `NC_memio`.
````
typedef struct NC_memio {
size_t size;
void* memory;
int flags;
} NC_memio;
````
An instance of this data structure is used when providing or
retrieving a block of data. It specifies the memory and its size
and also some relevant flags that define how to manage the memory.
Current only one flag is defined -- *NC_MEMIO_LOCKED*.
This tells the netcdf library that it should never try to
*realloc()* the memory nor to *free()* the memory. Note
that this does not mean that the memory cannot be modified, but
only that the modifications will be within the confines of the provided
memory. If doing such modifications is impossible without
reallocating the memory, then the modification will fail.
### In-Memory API
The new API consists of the following functions.
````
int nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp);
int nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp);
int nc_open_memio(const char* path, int mode, NC_memio* info, int* ncidp);
int nc_close_memio(int ncid, NC_memio* info);
````
### The **nc_open_mem** Function
The *nc_open_mem()* function is actually a convenience
function that internally invokes *nc_open_memio()*.
It essentially provides simple read-only access to a chunk of memory
of some specified size.
### The **nc_open_memio** Function
This function provides a more general read/write capability with respect
to a chunk of memory. It has a number of constraints and its
semantics are somewhat complex. This is primarily due to limitations
imposed by the underlying HDF5 library.
The constraints are as follows.
1. If the *NC_MEMIO_LOCKED* flag is set, then the netcdf library will
make no attempt to reallocate or free the provided memory.
If the caller invokes the *nc_close_memio()* function to retrieve the
final memory block, it should be the same
memory block as was provided when *nc_open_memio* was called.
Note that it is still possible to modify the in-memory file if the NC_WRITE
mode flag was set. However, failures can occur if an operation
cannot complete because the memory needs to be expanded.
2. If the *NC_MEMIO_LOCKED* flag is <b>not</b> set, then
the netcdf library will take control of the incoming memory
and will feel free to reallocate the provided
memory block to obtain a larger block when an attempt to modify
the in-memory file requires more space. Note that implicit in this
is that the old block -- the one originally provided -- may be
free'd as a side effect of re-allocating the memory using the
*realloc()* function.
If the caller invokes the *nc_close_memio()* function to retrieve the
final memory block, the returned block must always be freed
by the caller and that the original block should not be freed.
### The **nc_create_mem** Function
This function allows a user to create an in-memory file, write to it,
and then retrieve the final memory using *nc_close_memio()*.
The *initialsize* argument to *nc_create_mem()* tells the library
how much initial memory to allocate. Technically, this is advisory only
because it may be ignored by the underlying HDF5 library.
It is used, however, for netcdf-3 files.
### The **nc_close_memio** Function
The ordinary *nc_close()* function can be called to close an in-memory file.
However, it is often desirable to obtain the final size and memory block
for the in-memory file when that file has been modified.
The *nc_close_memio()* function provides a means to do this.
Its second argument is a pointer to an *NC_memio* object
into which the final memory and size are stored. WARNING,
the returned memory is owned by the caller and so the caller
is responsible for calling *free()* on that returned memory.
### Support for Writing with *NC_MEMIO_LOCKED*
When the NC_MEMIO_LOCKED flag is set in the *NC_memio* object
passed to *nc_open_memio()*, it is still possible to modify
the opened in-memory file (using the NC_WRITE mode flag).
The big problem is that any changes must fit into the memory provided
by the caller via the *NC_memio* object. This problem can be
mitigated, however, by using the "trick" of overallocating
the caller supplied memory. That is, if the original file is, say, 300 bytes,
then it is possible to allocate, say, 65000 bytes and copy the original file
into the first 300 bytes of the larger memory block. This will allow
the netcdf-c library to add to the file up to that 65000 byte limit.
In this way, it is possible to avoid memory reallocation while still
allowing modifications to the file. You will still need to call
*nc_close_memio()* to obtain the size of the final, modified, file.
Enabling MMAP File Access {#Enable_MMAP}
--------------
Some operating systems provide a capability called MMAP.
This allows disk files to automatically be mapped to chunks of memory.
It operates in a fashion somewhat similar to operating system virtual
memory, except with respect to a file.
By setting mode flag NC_MMAP, it is possible to do the equivalent
of NC_DISKLESS but using the operating system's mmap capabilities.
Currently, MMAP support is only available when using netcdf-3 or cdf5
files.
Known Bugs {#Inmemory_Bugs}
--------------
1. If you are modifying a locked memory chunk (using
NC_MEMIO_LOCKED) and are accessing it as a netcdf-4 file, and
you overrun the available space, then the HDF5 library will
fail with a segmentation fault.
References {#Inmemory_References}
--------------
1. https://support.hdfgroup.org/HDF5/doc1.8/Advanced/FileImageOperations/HDF5FileImageOperations.pdf
Point of Contact
--------------
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu
__Initial Version__: 2/3/2018<br>
__Last Revised__: 2/5/2018

51
docs/static-pages/software.html
View File

@ -95,6 +95,9 @@
<li>
<a href="#Gfdnavi" >Gfdnavi (Geophysical fluid data navigator)</a>
</li>
<li>
<a href="#gliderscope" >Gliderscope</a>
</li>
<li>
<a href="#GMT">GMT (Generic Mapping Tools)</a>
</li>
@ -200,6 +203,9 @@
<li>
<a href="#ncvtk" >ncvtk</a>
</li>
<li>
<a href="#netcdf_ninja" >NetCDF Ninja</a>
</li>
<li>
<a href="#netcdf_tools" >netcdf tools</a>
</li>
@ -927,6 +933,36 @@ or using ECMWF reanalysis on a reduced grid
</p>
<p></p>
<h2><a id="gliderscope" name="gliderscope">Gliderccope</a></h2>
<p>
Dr L. Mun Woo <mun.woo@uwa.edu.au>
at ANFOG (Australian National Facility for Ocean Gliders)
has developed
<a href="http://imos.org.au/facilities/oceangliders/glider-data/gliderscope/">Gliderscope</a>.
Gliderscope is an IMOS (Integrated Marine Observing System)
oceanographic software package allow users quick easy visualisation
of ocean glider data, via a convenient graphical user interface.
Originally developed for use with ANFOG NetCDF data, it has now
been expanded to handle NetCDF data files from IOOS (U.S.
Integrated Ocean Observing System) and EGO (Everyone's Gliding
Observatories) also.
</p><p>
Being interactive, Gliderscope speaks to users via an onscreen dialogue
box, helping the user decide what to do. With a few simple clicks of
the mouse, users can choose and extract segments of data, filter out the
bad data, perform calculations (e.g. for water density, sound velocity
in water, light attenuation, 1% photic depth) and apply a variety of
high level graphical data visualisation techniques to produce elegant
three/four-dimensional plots of water properties, interpolated contour
charts, vertical profile plots, water properties comparison charts
etc. Additionally, users can also export their data to text or NetCDF
files for easy access in other applications. Gliderscope is available
on Windows and Macintosh platforms, as standalone executable software
as well as an App for use within Matlab.
</p>
<p></p>
<h2><a id="GMT" name="GMT">GMT</a></h2>
<p>
<a href="http://gmt.soest.hawaii.edu/">GMT</a> (Generic Mapping Tools) is
@ -1965,6 +2001,21 @@ or using ECMWF reanalysis on a reduced grid
<p></p>
<h2><a id="netcdf_ninja" name="netcdf_ninja">netCDF Ninja</a></h2>
<p>
Dr L. Mun Woo of University of Western Australia <mun.woo@uwa.edu.au>
has developed
<a href="http://imos.org.au/facilities/oceangliders/glider-data/netcdfninja/">NetCDF Ninja</a>,
a graphical user interface that allows users to browse all
the metadata contained in NetCDF files, scrutinise the data using an
interactive graphical plot and even make small alterations or export
the data in text format without having any knowledge of coding. NetCDF
Ninja is available on Windows and Macintosh platforms, as standalone
executable software as well as an App for use within Matlab.
</p>
<p></p>
<h2><a id="netcdf_tools" name="netcdf_tools">Ivan Shmakov's netcdf tools</a></h2>
<p>

4
examples/C/hdf5plugins/Makefile.am
View File

@ -24,5 +24,5 @@ libbzip2_la_SOURCES = ${DLLSRC}
libbzip2_la_LDFLAGS = -module -avoid-version -shared -export-dynamic -no-undefined
endif #ENABLE_FILTER_TESTING
EXTRA_DIST = CMakeLists.txt H5Zbzip2.c Makefile.am blocksort.c bzlib.c bzlib.h bzlib_private.h compress.c \
crctable.c decompress.c h5bzip2.h huffman.c randtable.c bzip2.nc
EXTRA_DIST = CMakeLists.txt

2
include/nc3dispatch.h
View File

@ -80,7 +80,7 @@ extern int
NC3_abort(int ncid);
extern int
NC3_close(int ncid);
NC3_close(int ncid,void*);
extern int
NC3_set_fill(int ncid, int fillmode, int *old_modep);

2
include/nc4dispatch.h
View File

@ -44,7 +44,7 @@ extern int
NC4_abort(int ncid);
extern int
NC4_close(int ncid);
NC4_close(int ncid,void*);
extern int
NC4_set_fill(int ncid, int fillmode, int *old_modep);

11
include/nc4internal.h
View File

@ -20,6 +20,7 @@
#include "ncdimscale.h"
#include "nc_logging.h"
#include "netcdf_mem.h"
#ifdef USE_PARALLEL
#include "netcdf_par.h"
@ -332,9 +333,17 @@ typedef struct NC_HDF5_FILE_INFO
int sdid;
#endif /* USE_HDF4 */
struct NCFILEINFO* fileinfo;
struct NC4_Memio {
NC_memio memio;
int locked; /* do not copy and do not release */
int persist; /* Should file be persisted out on close? */
int inmemory;
int diskless;
unsigned int flags; /* for H5LTopen_file_image */
int fapl;
} mem;
} NC_HDF5_FILE_INFO_T;
/* Defined in lookup3.c */
extern uint32_t hash_fast(const void *key, size_t length);

7
include/ncdispatch.h
View File

@ -105,11 +105,6 @@ typedef struct NC_MPI_INFO {
MPI_Info info;
} NC_MPI_INFO;
typedef struct NC_MEM_INFO {
size_t size;
void* memory;
} NC_MEM_INFO;
/* Define known dispatch tables and initializers */
/*Forward*/
@ -211,7 +206,7 @@ int (*redef)(int);
int (*_enddef)(int,size_t,size_t,size_t,size_t);
int (*sync)(int);
int (*abort)(int);
int (*close)(int);
int (*close)(int,void*);
int (*set_fill)(int,int,int*);
int (*inq_base_pe)(int,int*);
int (*set_base_pe)(int,int);

5
include/netcdf.h
View File

@ -154,7 +154,7 @@ Use this in mode flags for both nc_create() and nc_open(). */
Use this in mode flags for both nc_create() and nc_open(). */
#define NC_MPIPOSIX 0x4000 /**< \deprecated As of libhdf5 1.8.13. */
#define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create(). */
#define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() => NC_DISKLESS */
#define NC_PNETCDF (NC_MPIIO) /**< Use parallel-netcdf library; alias for NC_MPIIO. */
@ -459,7 +459,8 @@ by the desired type. */
#define NC_EFILTER (-132) /**< Filter operation failed. */
#define NC_ERCFILE (-133) /**< RC file failure */
#define NC_ENULLPAD (-134) /**< Header Bytes not Null-Byte padded */
#define NC4_LAST_ERROR (-135) /**< @internal All netCDF errors > this. */
#define NC_EINMEMORY (-135) /**< In-memory file error */
#define NC4_LAST_ERROR (-136) /**< @internal All netCDF errors > this. */
/** @internal This is used in netCDF-4 files for dimensions without
* coordinate vars. */

18
include/netcdf_mem.h
View File

@ -14,12 +14,30 @@
#include <netcdf.h>
typedef struct NC_memio {
size_t size;
void* memory;
#define NC_MEMIO_LOCKED 1 /* Do not try to realloc or free provided memory */
int flags;
} NC_memio;
#if defined(__cplusplus)
extern "C" {
#endif
/* Treate a memory block as a file; read-only */
EXTERNL int nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp);
EXTERNL int nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp);
/* Alternative to nc_open_mem with extended capabilites
See docs/inmemory.md
*/
EXTERNL int nc_open_memio(const char* path, int mode, NC_memio* info, int* ncidp);
/* Close memory file and return the final memory state */
EXTERNL int nc_close_memio(int ncid, NC_memio* info);
#if defined(__cplusplus)
}
#endif

6
libdap2/ncd2dispatch.c
View File

@ -225,7 +225,7 @@ NCD2_sync(int ncid)
static int
NCD2_abort(int ncid)
{
return NCD2_close(ncid);
return NCD2_close(ncid,NULL);
}
static int
@ -596,13 +596,13 @@ fprintf(stderr,"ncdap3: final constraint: %s\n",dapcomm->oc.url->query);
return ncstat;
done:
if(drno != NULL) NCD2_close(drno->ext_ncid);
if(drno != NULL) NCD2_close(drno->ext_ncid,NULL);
if(ocstat != OC_NOERR) ncstat = ocerrtoncerr(ocstat);
return THROW(ncstat);
}
int
NCD2_close(int ncid)
NCD2_close(int ncid, void* ignore)
{
NC* drno;
NCDAPCOMMON* dapcomm;

2
libdap2/ncd2dispatch.h
View File

@ -50,7 +50,7 @@ NCD2_open(const char *path, int mode,
struct NC_Dispatch* dispatch, NC* ncp);
extern int
NCD2_close(int ncid);
NCD2_close(int ncid,void*);
extern int
NCD2_inq_format_extended(int ncid, int* formatp, int* modep);

4
libdap4/d4file.c
View File

@ -245,7 +245,7 @@ done:
}
int
NCD4_close(int ncid)
NCD4_close(int ncid, void* ignore)
{
int ret = NC_NOERR;
NC* nc;
@ -278,7 +278,7 @@ done:
int
NCD4_abort(int ncid)
{
return NCD4_close(ncid);
return NCD4_close(ncid,NULL);
}
/**************************************************/

2
libdap4/ncd4dispatch.h
View File

@ -21,7 +21,7 @@ NCD4_open(const char *path, int mode,
struct NC_Dispatch* dispatch, NC* ncp);
extern int
NCD4_close(int ncid);
NCD4_close(int ncid,void*);
extern int
NCD4_abort(int ncid);

8
libdispatch/derror.c
View File

@ -262,9 +262,11 @@ const char *nc_strerror(int ncerr1)
case NC_EMPI: return "NetCDF: MPI operation failed.";
case NC_ERCFILE:
return "NetCDF: RC File Failure.";
case NC_ENULLPAD:
return "NetCDF: File fails strict Null-Byte Header check.";
default:
case NC_ENULLPAD:
return "NetCDF: File fails strict Null-Byte Header check.";
case NC_EINMEMORY:
return "NetCDF: In-memory File operation failed.";
default:
#ifdef USE_PNETCDF
/* The behavior of ncmpi_strerror here is to return
NULL, not a string. This causes problems in (at least)

219
libdispatch/dfile.c
View File

@ -176,10 +176,10 @@ NC_check_file_type(const char *path, int flags, void *parameters,
int status = NC_NOERR;
int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS);
int inmemory = (!diskless && ((flags & NC_INMEMORY) == NC_INMEMORY));
#ifdef USE_PARALLEL
int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO);
#endif /* USE_PARALLEL */
int inmemory = (diskless && ((flags & NC_INMEMORY) == NC_INMEMORY));
struct MagicFile file;
*model = 0;
@ -189,7 +189,7 @@ NC_check_file_type(const char *path, int flags, void *parameters,
file.path = path; /* do not free */
file.parameters = parameters;
if(inmemory && parameters == NULL)
{status = NC_EDISKLESS; goto done;}
{status = NC_EINMEMORY; goto done;}
if(inmemory) {
file.inmemory = inmemory;
goto next;
@ -258,8 +258,8 @@ and attributes.
NC_64BIT_DATA (Alias NC_CDF5) (create CDF-5 file),
NC_NETCDF4 (create netCDF-4/HDF5 file),
NC_CLASSIC_MODEL (enforce netCDF classic mode on netCDF-4/HDF5 files),
NC_DISKLESS (store data only in memory),
NC_MMAP (use MMAP for NC_DISKLESS),
NC_DISKLESS (store data in memory),
NC_MMAP (use MMAP for NC_DISKLESS instead of NC_INMEMORY),
and NC_WRITE.
See discussion below.
@ -273,7 +273,10 @@ aspects of how it may be used.
Setting NC_NOCLOBBER means you do not want to clobber (overwrite) an
existing dataset; an error (NC_EEXIST) is returned if the specified
dataset already exists.
dataset already exists. As a slight variation on this, if you
specify NC_DISKLESS and NC_NOCLOBBER, the file will be created
in-memory, but no attempt will be made to persiste the in-memory
data to a disk file.
The NC_SHARE flag is appropriate when one process may be writing the
dataset and one or more other processes reading the dataset
@ -523,6 +526,58 @@ nc__create(const char *path, int cmode, size_t initialsz,
chunksizehintp, 0, NULL, ncidp);
}
/** \ingroup datasets
Create a netCDF file with the contents stored in memory.
\param path Must be non-null, but otherwise only used to set the dataset name.
\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY.
\param initialsize (advisory) size to allocate for the created file
\param ncidp Pointer to location where returned netCDF ID is to be
stored.
\returns ::NC_NOERR No error.
\returns ::NC_ENOMEM Out of memory.
\returns ::NC_EDISKLESS diskless io is not enabled for fails.
\returns ::NC_EINVAL, etc. other errors also returned by nc_open.
<h1>Examples</h1>
In this example we use nc_create_mem() to create a classic netCDF dataset
named foo.nc. The initial size is set to 4096.
@code
#include <netcdf.h>
...
int status = NC_NOERR;
int ncid;
int mode = 0;
size_t initialsize = 4096;
...
status = nc_create_mem("foo.nc", mode, initialsize, &ncid);
if (status != NC_NOERR) handle_error(status);
@endcode
*/
int
nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp)
{
#ifdef USE_DISKLESS
if(mode & (NC_MPIIO|NC_MPIPOSIX|NC_MMAP))
return NC_EINVAL;
mode |= (NC_INMEMORY|NC_NOCLOBBER); /* Specifically, do not set NC_DISKLESS */
return NC_create(path, mode, initialsize, 0, NULL, 0, NULL, ncidp);
#else
return NC_EDISKLESS;
#endif
}
/**
* @internal Create a file with special (deprecated) Cray settings.
*
@ -735,7 +790,7 @@ Open a netCDF file with the contents taken from a block of memory.
\param path Must be non-null, but otherwise only used to set the dataset name.
\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_NOWRITE|NC_DISKLESS|NC_INMEMORY.
\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY.
\param size The length of the block of memory being passed.
@ -779,22 +834,89 @@ int
nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp)
{
#ifdef USE_DISKLESS
NC_MEM_INFO meminfo;
NC_memio meminfo;
/* Sanity checks */
if(memory == NULL || size < MAGIC_NUMBER_LEN || path == NULL)
return NC_EINVAL;
if(mode & (NC_WRITE|NC_MPIIO|NC_MPIPOSIX|NC_MMAP))
return NC_EINVAL;
mode |= (NC_INMEMORY|NC_DISKLESS);
mode |= (NC_INMEMORY); /* DO not set NC_DISKLESS */
meminfo.size = size;
meminfo.memory = memory;
meminfo.flags = NC_MEMIO_LOCKED;
return NC_open(path, mode, 0, NULL, 0, &meminfo, ncidp);
#else
return NC_EDISKLESS;
#endif
}
/** \ingroup datasets
Open a netCDF file with the contents taken from a block of memory.
Similar to nc_open_mem, but with parameters. Warning: if you do
specify that the provided memory is locked, then <b>never</b>
pass in non-heap allocated memory. Additionally, if not locked,
then do not assume that the memory returned by nc_close_mem
is the same as passed to nc_open_memio. You <b>must</b> check
before attempting to free the original memory.
\param path Must be non-null, but otherwise only used to set the dataset name.
\param mode the mode flags; Note that this procedure uses a limited set of flags because it forcibly sets NC_INMEMORY.
\param params controlling parameters
\param ncidp Pointer to location where returned netCDF ID is to be
stored.
\returns ::NC_NOERR No error.
\returns ::NC_ENOMEM Out of memory.
\returns ::NC_EDISKLESS diskless io is not enabled for fails.
\returns ::NC_EINVAL, etc. other errors also returned by nc_open.
<h1>Examples</h1>
Here is an example using nc_open_memio() to open an existing netCDF dataset
named foo.nc for read-only, non-shared access. It differs from the nc_open_mem()
example in that it uses a parameter block.
@code
#include <netcdf.h>
#include <netcdf_mem.h>
...
int status = NC_NOERR;
int ncid;
NC_memio params;
...
params.size = <compute file size of foo.nc in bytes>;
params.memory = malloc(size);
params.flags = <see netcdf_mem.h>
...
status = nc_open_memio("foo.nc", 0, &params, &ncid);
if (status != NC_NOERR) handle_error(status);
@endcode
*/
int
nc_open_memio(const char* path, int mode, NC_memio* params, int* ncidp)
{
#ifdef USE_DISKLESS
/* Sanity checks */
if(path == NULL || params == NULL)
return NC_EINVAL;
if(params->memory == NULL || params->size < MAGIC_NUMBER_LEN)
return NC_EINVAL;
if(mode & (NC_MPIIO|NC_MPIPOSIX|NC_MMAP))
return NC_EINVAL;
mode |= (NC_INMEMORY);
return NC_open(path, mode, 0, NULL, 0, params, ncidp);
#else
return NC_EINMEMORY;
#endif
}
/**
* @internal Open a netCDF file with extra parameters for Cray.
*
@ -1258,8 +1380,7 @@ nc_close(int ncid)
if(ncp->refcount <= 0)
#endif
{
stat = ncp->dispatch->close(ncid);
stat = ncp->dispatch->close(ncid,NULL);
/* Remove from the nc list */
if (!stat)
{
@ -1270,6 +1391,75 @@ nc_close(int ncid)
return stat;
}
/** \ingroup datasets
Do a normal close (see nc_close()) on an in-memory dataset,
then return a copy of the final memory contents of the dataset.
\param ncid NetCDF ID, from a previous call to nc_open() or nc_create().
\param memio a pointer to an NC_memio object into which the final valid memory
size and memory will be returned.
\returns ::NC_NOERR No error.
\returns ::NC_EBADID Invalid id passed.
\returns ::NC_ENOMEM Out of memory.
\returns ::NC_EDISKLESS if the file was not created as an inmemory file.
\returns ::NC_EBADGRPID ncid did not contain the root group id of this
file. (NetCDF-4 only).
<h1>Example</h1>
Here is an example using nc_close_mem to finish the definitions of a new
netCDF dataset named foo.nc, return the final memory,
and release its netCDF ID:
\code
#include <netcdf.h>
...
int status = NC_NOERR;
int ncid;
NC_memio finalmem;
size_t initialsize = 65000;
...
status = nc_create_mem("foo.nc", NC_NOCLOBBER, initialsize, &ncid);
if (status != NC_NOERR) handle_error(status);
... create dimensions, variables, attributes
status = nc_close_memio(ncid,&finalmem);
if (status != NC_NOERR) handle_error(status);
\endcode
*/
int
nc_close_memio(int ncid, NC_memio* memio)
{
#ifdef USE_DISKLESS
NC* ncp;
int stat = NC_check_id(ncid, &ncp);
if(stat != NC_NOERR) return stat;
#ifdef USE_REFCOUNT
ncp->refcount--;
if(ncp->refcount <= 0)
#endif
{
stat = ncp->dispatch->close(ncid,memio);
/* Remove from the nc list */
if (!stat)
{
del_from_NCList(ncp);
free_NC(ncp);
}
}
return stat;
#else
return NC_EINMEMORY;
#endif
}
/** \ingroup datasets
Change the fill-value mode to improve write performance.
@ -1769,7 +1959,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
}
#ifndef USE_DISKLESS
cmode &= (~ NC_DISKLESS); /* Force off */
cmode &= (~ (NC_DISKLESS|NC_INMEMORY)); /* Force off */
#endif
#ifdef WINPATH
@ -1959,13 +2149,12 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp,
#ifndef USE_DISKLESS
/* Clean up cmode */
cmode &= (~ NC_DISKLESS);
cmode &= (~ (NC_DISKLESS|NC_INMEMORY));
#endif
inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY);
diskless = ((cmode & NC_DISKLESS) == NC_DISKLESS);
#ifdef WINPATH
path = NCpathcvt(path0);
#else
@ -2190,7 +2379,7 @@ openmagic(struct MagicFile* file)
int status = NC_NOERR;
if(file->inmemory) {
/* Get its length */
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)file->parameters;
NC_memio* meminfo = (NC_memio*)file->parameters;
file->filelen = (long long)meminfo->size;
goto done;
}
@ -2255,7 +2444,7 @@ readmagic(struct MagicFile* file, long pos, char* magic)
memset(magic,0,MAGIC_NUMBER_LEN);
if(file->inmemory) {
char* mempos;
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)file->parameters;
NC_memio* meminfo = (NC_memio*)file->parameters;
if((pos + MAGIC_NUMBER_LEN) > meminfo->size)
{status = NC_EDISKLESS; goto done;}
mempos = ((char*)meminfo->memory) + pos;

4
libdispatch/dutil.c
View File

@ -169,9 +169,9 @@ NC_readfile(const char* filename, NCbytes* content)
char part[1024];
#ifdef _MSC_VER
stream = NCfopen(filename,"r");
#else
stream = NCfopen(filename,"rb");
#else
stream = NCfopen(filename,"r");
#endif
if(stream == NULL) {ret=errno; goto done;}
for(;;) {

2
libdispatch/dvar.c
View File

@ -426,7 +426,7 @@ NC_inq_recvar(int ncid, int varid, int* nrecdimsp, int *is_recdim)
* Find the length of a type. This is how much space is required by
* the in memory to hold one element of this type.
*
* @parm type A netCDF atomic type.
* @param type A netCDF atomic type.
*
* @return Length of the type in bytes, or -1 if type not found.
* @author Ed Hartnett

421
libsrc/memio.c
View File

@ -2,40 +2,35 @@
* Copyright 1996, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#if defined (_WIN32) || defined (_WIN64)
#include <windows.h>
#include <winbase.h>
#include <io.h>
#define lseek64 lseek
#endif
#if HAVE_CONFIG_H
#include <config.h>
#endif
#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#ifdef _MSC_VER /* Microsoft Compilers */
#include <io.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef _MSC_VER /* Microsoft Compilers */
#include <windows.h>
#include <winbase.h>
#include <io.h>
#define access(path,mode) _access(path,mode)
#endif
#include "ncdispatch.h"
#include "nc3internal.h"
#include "netcdf_mem.h"
#include "ncwinpath.h"
#undef DEBUG
#ifdef DEBUG
#include <stdio.h>
#endif
#ifndef HAVE_SSIZE_T
typedef int ssize_t;
#endif
@ -77,15 +72,33 @@ typedef int ssize_t;
#undef X_ALIGN
#endif
#undef REALLOCBUG
#ifdef REALLOCBUG
/* There is some kind of realloc bug that I cannot solve yet */
#define reallocx(m,new,old) realloc(m,new)
#else
static void*
reallocx(void* mem, size_t newsize, size_t oldsize)
{
void* m = malloc(newsize);
memcpy(m,mem,oldsize);
return m;
}
#endif
/* Private data for memio */
typedef struct NCMEMIO {
int locked; /* => we cannot realloc */
int locked; /* => we cannot realloc or free*/
int modified; /* => we realloc'd memory at least once */
int persist; /* => save to a file; triggered by NC_WRITE */
char* memory;
off_t alloc;
off_t size;
off_t pos;
size_t alloc;
size_t size;
size_t pos;
/* Convenience flags */
int diskless;
int inmemory; /* assert(inmemory iff !diskless */
} NCMEMIO;
/* Forward */
@ -96,25 +109,28 @@ static int memio_sync(ncio *const nciop);
static int memio_filesize(ncio* nciop, off_t* filesizep);
static int memio_pad_length(ncio* nciop, off_t length);
static int memio_close(ncio* nciop, int);
static int readfile(const char* path, NC_memio*);
static int writefile(const char* path, NCMEMIO*);
static int fileiswriteable(const char* path);
/* Mnemonic */
#define DOOPEN 1
static long pagesize = 0;
static size_t pagesize = 0;
/*! Create a new ncio struct to hold info about the file. */
static int memio_new(const char* path, int ioflags, off_t initialsize, void* memory, ncio** nciopp, NCMEMIO** memiop)
static int
memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEMIO** memiop)
{
int status = NC_NOERR;
ncio* nciop = NULL;
NCMEMIO* memio = NULL;
off_t minsize = initialsize;
int inmemory = (fIsSet(ioflags,NC_INMEMORY));
size_t minsize = (size_t)initialsize;
/* use asserts because this is an internal function */
assert(fIsSet(ioflags,NC_INMEMORY));
assert(memiop != NULL && nciopp != NULL);
assert(path != NULL || (memory != NULL && initialsize > 0));
assert(!inmemory || (memory != NULL && initialsize > 0));
assert(path != NULL);
if(pagesize == 0) {
#if defined (_WIN32) || defined(_WIN64)
@ -122,7 +138,7 @@ static int memio_new(const char* path, int ioflags, off_t initialsize, void* mem
GetSystemInfo (&info);
pagesize = info.dwPageSize;
#elif defined HAVE_SYSCONF
pagesize = sysconf(_SC_PAGE_SIZE);
pagesize = (size_t)sysconf(_SC_PAGE_SIZE);
#elif defined HAVE_GETPAGESIZE
pagesize = getpagesize();
#else
@ -165,24 +181,25 @@ static int memio_new(const char* path, int ioflags, off_t initialsize, void* mem
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
memio->alloc = initialsize;
memio->pos = 0;
memio->size = minsize;
memio->memory = NULL;
memio->persist = fIsSet(ioflags,NC_WRITE);
if(memiop && memio) *memiop = memio; else free(memio);
if(nciopp && nciop) *nciopp = nciop;
else {
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
}
if(inmemory) {
memio->memory = memory;
} else {
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto fail;}
}
memio->alloc = (size_t)initialsize;
memio->pos = 0;
memio->size = minsize;
memio->memory = NULL; /* filled in by caller */
if(fIsSet(ioflags,NC_DISKLESS))
memio->diskless = 1;
if(fIsSet(ioflags,NC_INMEMORY) && !memio->diskless)
memio->inmemory = 1;
if(fIsSet(ioflags,NC_WRITE) && !fIsSet(ioflags,NC_NOCLOBBER) && memio->diskless)
memio->persist = 1;
done:
return status;
@ -214,41 +231,31 @@ int
memio_create(const char* path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
void* parameters /*ignored*/,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;
int fd;
int status;
NCMEMIO* memio = NULL;
int persist = (ioflags & NC_WRITE?1:0);
int oflags;
if(path == NULL ||* path == 0)
return NC_EINVAL;
status = memio_new(path, ioflags, initialsz, NULL, &nciop, &memio);
status = memio_new(path, ioflags, initialsz, &nciop, &memio);
if(status != NC_NOERR)
return status;
if(persist) {
/* Open the file just tomake sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= (O_CREAT|O_TRUNC);
if(fIsSet(ioflags,NC_NOCLOBBER))
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
if(fd < 0) {status = errno; goto unwind_open;}
if(memio->persist) {
/* Verify the file is writeable */
if(!fileiswriteable(path))
{status = EPERM; goto unwind_open;}
}
(void)close(fd); /* will reopen at nc_close */
} /*!persist*/
/* Allocate the memory for this file */
memio->memory = (char*)malloc((size_t)memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
memio->locked = 0;
#ifdef DEBUG
fprintf(stderr,"memio_create: initial memory: %lu/%lu\n",(unsigned long)memio->memory,(unsigned long)memio->alloc);
@ -270,7 +277,7 @@ fprintf(stderr,"memio_create: initial memory: %lu/%lu\n",(unsigned long)memio->m
}
/* Pick a default sizehint */
if(sizehintp) *sizehintp = pagesize;
if(sizehintp) *sizehintp = (size_t)pagesize;
*nciopp = nciop;
return NC_NOERR;
@ -280,8 +287,7 @@ unwind_open:
return status;
}
/* This function opens the data file.
/* This function opens the data file or inmemory data
path - path of data file.
ioflags - flags passed into nc_open.
igeto - looks like this function can do an initial page get, and
@ -305,85 +311,75 @@ memio_open(const char* path,
ncio* nciop = NULL;
int fd = -1;
int status = NC_NOERR;
int persist = (fIsSet(ioflags,NC_WRITE)?1:0);
int inmemory = (fIsSet(ioflags,NC_INMEMORY));
int oflags = 0;
NCMEMIO* memio = NULL;
size_t sizehint = 0;
off_t filesize = 0;
off_t red = 0;
char* pos = NULL;
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
NC_memio meminfo; /* use struct to avoid worrying about free'ing it */
NCMEMIO* memio = NULL;
size_t initialsize;
/* Should be the case that diskless => inmemory but not converse */
int diskless = (fIsSet(ioflags,NC_DISKLESS));
int inmemory = (fIsSet(ioflags,NC_INMEMORY) && !diskless);
int locked = 0;
if(path == NULL || strlen(path) == 0)
return NC_EINVAL;
assert(sizehintp != NULL);
sizehint = *sizehintp;
if(inmemory) {
filesize = meminfo->size;
} else {
/* Open the file,and make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
#ifdef DEBUG
if(fd < 0) {
fprintf(stderr,"open failed: file=%s err=",path);
perror("");
}
#endif
if(fd < 0) {status = errno; goto unwind_open;}
/* get current filesize = max(|file|,initialize)*/
filesize = lseek(fd,0,SEEK_END);
if(filesize < 0) {status = errno; goto unwind_open;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
if(filesize < (off_t)sizehint)
filesize = (off_t)sizehint;
if(inmemory) { /* parameters provide the memory chunk */
NC_memio* memparams = (NC_memio*)parameters;
meminfo = *memparams;
locked = fIsSet(meminfo.flags,NC_MEMIO_LOCKED);
/* As a safeguard, if !locked and NC_WRITE is set,
then we must take control of the incoming memory */
if(!locked && fIsSet(ioflags,NC_WRITE)) {
memparams->memory = NULL;
}
} else { /* read the file into a chunk of memory*/
assert(diskless);
status = readfile(path,&meminfo);
if(status != NC_NOERR)
{goto unwind_open;}
}
if(inmemory)
status = memio_new(path, ioflags, filesize, meminfo->memory, &nciop, &memio);
else
status = memio_new(path, ioflags, filesize, NULL, &nciop, &memio);
if(status != NC_NOERR) {
if(fd >= 0)
close(fd);
return status;
/* Fix up initial size */
initialsize = meminfo.size;
/* create the NCMEMIO structure */
status = memio_new(path, ioflags, initialsize, &nciop, &memio);
if(status != NC_NOERR)
{goto unwind_open;}
memio->locked = locked;
/* Initialize the memio memory */
memio->memory = meminfo.memory;
/* memio_new may have modified the allocated size, in which case,
reallocate the memory unless the memory is locked. */
if(memio->alloc > meminfo.size) {
if(memio->locked)
memio->alloc = meminfo.size; /* force it back to what it was */
else {
void* oldmem = memio->memory;
memio->memory = reallocx(oldmem,memio->alloc,meminfo.size);
if(memio->memory == NULL)
{status = NC_ENOMEM; goto unwind_open;}
}
}
#ifdef DEBUG
fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->memory,(unsigned long)memio->alloc);
#endif
if(!inmemory) {
/* Read the file into the memio memory */
/* We need to do multiple reads because there is no
guarantee that the amount read will be the full amount */
red = memio->size;
pos = memio->memory;
while(red > 0) {
ssize_t count = read(fd, pos, red);
if(count < 0) {status = errno; goto unwind_open;}
if(count == 0) {status = NC_ENOTNC; goto unwind_open;}
red -= count;
pos += count;
}
(void)close(fd);
if(memio->persist) {
/* Verify the file is writeable */
if(!fileiswriteable(path))
{status = EPERM; goto unwind_open;}
}
/* Use half the filesize as the blocksize ; why? */
sizehint = filesize/2;
sizehint = (size_t)(memio->alloc/2);
/* sizehint must be multiple of 8 */
sizehint = (sizehint / 8) * 8;
@ -437,26 +433,34 @@ static int
memio_pad_length(ncio* nciop, off_t length)
{
NCMEMIO* memio;
size_t len = (size_t)length;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
if(!fIsSet(nciop->ioflags, NC_WRITE))
if(!memio->persist)
return EPERM; /* attempt to write readonly file*/
if(memio->locked > 0)
if(memio->locked)
return NC_EDISKLESS;
if(length > memio->alloc) {
if(len > memio->alloc) {
/* Realloc the allocated memory to a multiple of the pagesize*/
off_t newsize = length;
size_t newsize = (size_t)len;
void* newmem = NULL;
/* Round to a multiple of pagesize */
if((newsize % pagesize) != 0)
newsize += (pagesize - (newsize % pagesize));
newmem = (char*)realloc(memio->memory,newsize);
newmem = (char*)reallocx(memio->memory,newsize,memio->alloc);
if(newmem == NULL) return NC_ENOMEM;
/* If not copy is set, then fail if the newmem address is different
from old address */
if(newmem != memio->memory) {
memio->modified++;
if(memio->locked) {
free(newmem);
return NC_EINMEMORY;
}
}
/* zero out the extra memory */
memset((void*)((char*)newmem+memio->alloc),0,(newsize - memio->alloc));
@ -465,10 +469,13 @@ fprintf(stderr,"realloc: %lu/%lu -> %lu/%lu\n",
(unsigned long)memio->memory,(unsigned long)memio->alloc,
(unsigned long)newmem,(unsigned long)newsize);
#endif
if(memio->memory != NULL && (!memio->locked || memio->modified))
free(memio->memory);
memio->memory = newmem;
memio->alloc = newsize;
memio->modified = 1;
}
memio->size = length;
memio->size = len;
return NC_NOERR;
}
@ -486,46 +493,23 @@ memio_close(ncio* nciop, int doUnlink)
{
int status = NC_NOERR;
NCMEMIO* memio ;
int fd = -1;
int inmemory = 0;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
inmemory = (fIsSet(nciop->ioflags,NC_INMEMORY));
memio = (NCMEMIO*)nciop->pvt;
assert(memio != NULL);
/* See if the user wants the contents persisted to a file */
if(!inmemory && memio->persist) {
/* Try to open the file for writing */
int oflags = O_WRONLY|O_CREAT|O_TRUNC;
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
fd = open(nciop->path, oflags, OPENMODE);
if(fd >= 0) {
/* We need to do multiple writes because there is no
guarantee that the amount written will be the full amount */
off_t written = memio->size;
char* pos = memio->memory;
while(written > 0) {
ssize_t count = write(fd, pos, written);
if(count < 0)
{status = errno; goto done;}
if(count == 0)
{status = NC_ENOTNC; goto done;}
written -= count;
pos += count;
}
} else
status = errno;
}
if(memio->persist && memio->memory != NULL) {
status = writefile(nciop->path,memio);
}
done:
if(!inmemory && memio->memory != NULL)
/* We only free the memio memory if file is not locked or has been modified */
if(memio->memory != NULL && (!memio->locked || memio->modified)) {
free(memio->memory);
memio->memory = NULL;
}
/* do cleanup */
if(fd >= 0) (void)close(fd);
if(memio != NULL) free(memio);
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
@ -533,9 +517,10 @@ done:
}
static int
guarantee(ncio* nciop, off_t endpoint)
guarantee(ncio* nciop, off_t endpoint0)
{
NCMEMIO* memio = (NCMEMIO*)nciop->pvt;
size_t endpoint = (size_t)endpoint0;
if(endpoint > memio->alloc) {
/* extend the allocated memory and size */
int status = memio_pad_length(nciop,endpoint);
@ -637,3 +622,117 @@ memio_sync(ncio* const nciop)
{
return NC_NOERR; /* do nothing */
}
/* "Hidden" Internal function to extract a copy of
the size and/or contents of the memory
*/
int
memio_extract(ncio* const nciop, size_t* sizep, void** memoryp)
{
int status = NC_NOERR;
NCMEMIO* memio = NULL;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
memio = (NCMEMIO*)nciop->pvt;
assert(memio != NULL);
if(sizep) *sizep = memio->size;
if(memoryp && memio->memory != NULL) {
*memoryp = memio->memory;
memio->memory = NULL; /* make sure it does not get free'd */
}
return status;
}
static int
fileiswriteable(const char* path)
{
int ok;
ok = access(path,O_RDWR);
if(ok < 0)
return 0;
return 1;
}
/* Read contents of a disk file into a memory chunk */
static int
readfile(const char* path, NC_memio* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t filesize = 0;
size_t count = 0;
char* memory = NULL;
char* p = NULL;
/* Open the file for reading */
#ifdef _MSC_VER
f = NCfopen(path,"rb");
#else
f = NCfopen(path,"r");
#endif
if(f == NULL)
{status = errno; goto done;}
/* get current filesize */
if(fseek(f,0,SEEK_END) < 0)
{status = errno; goto done;}
filesize = (size_t)ftell(f);
/* allocate memory */
memory = malloc((size_t)filesize);
if(memory == NULL)
{status = NC_ENOMEM; goto done;}
/* move pointer back to beginning of file */
rewind(f);
count = filesize;
p = memory;
while(count > 0) {
size_t actual;
actual = fread(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
}
if(memio) {
memio->size = (size_t)filesize;
memio->memory = memory;
}
done:
if(status != NC_NOERR && memory != NULL)
free(memory);
if(f != NULL) fclose(f);
return status;
}
/* write contents of a memory chunk back into a disk file */
static int
writefile(const char* path, NCMEMIO* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t count = 0;
char* p = NULL;
/* Open the file for writing*/
#ifdef _MSC_VER
f = NCfopen(path,"rwb");
#else
f = NCfopen(path,"rw");
#endif
if(f == NULL)
{status = errno; goto done;}
rewind(f);
count = memio->size;
p = memio->memory;
while(count > 0) {
size_t actual;
actual = fwrite(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
}
done:
if(f != NULL) fclose(f);
return status;
}

13
libsrc/nc3internal.c
View File

@ -19,6 +19,7 @@
#endif
#include "nc3internal.h"
#include "netcdf_mem.h"
#include "rnd.h"
#include "ncx.h"
@ -35,6 +36,10 @@
/* For cdf5 */
#define NC_NUMRECS_EXTENT5 8
/* Internal function; breaks ncio abstraction */
extern int memio_extract(ncio* const nciop, size_t* sizep, void** memoryp);
static void
free_NC3INFO(NC3_INFO *nc3)
{
@ -1311,7 +1316,7 @@ NC3_abort(int ncid)
}
int
NC3_close(int ncid)
NC3_close(int ncid, void* params)
{
int status = NC_NOERR;
NC *nc;
@ -1359,6 +1364,12 @@ NC3_close(int ncid)
}
}
if(params != NULL && (nc->mode & NC_INMEMORY) != 0) {
NC_memio* memio = (NC_memio*)params;
/* Extract the final memory size &/or contents */
status = memio_extract(nc3->nciop,&memio->size,&memio->memory);
}
(void) ncio_close(nc3->nciop, 0);
nc3->nciop = NULL;

10
libsrc/ncio.c
View File

@ -46,9 +46,9 @@ ncio_create(const char *path, int ioflags, size_t initialsz,
ncio** iopp, void** const mempp)
{
#ifdef USE_DISKLESS
if(fIsSet(ioflags,NC_DISKLESS)) {
if(fIsSet(ioflags,NC_INMEMORY)) {
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP))
if(fIsSet(ioflags,NC_MMAP) && fIsSet(ioflags, NC_DISKLESS))
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
else
# endif /*USE_MMAP*/
@ -72,12 +72,12 @@ ncio_open(const char *path, int ioflags,
ncio** iopp, void** const mempp)
{
/* Diskless open has the following constraints:
1. file must be classic version 1 or 2
1. file must be classic version 1 or 2 or 5
*/
#ifdef USE_DISKLESS
if(fIsSet(ioflags,NC_DISKLESS)) {
if(fIsSet(ioflags,NC_INMEMORY)) {
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP))
if(fIsSet(ioflags,NC_MMAP) && fIsSet(ioflags, NC_DISKLESS))
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
else
# endif /*USE_MMAP*/

2
libsrc4/CMakeLists.txt
View File

@ -1,6 +1,6 @@
# Process these files with m4.
SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4file.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c nc4hdf.c nc4info.c)
SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4file.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c nc4hdf.c nc4info.c nc4mem.c nc4memcb.c)
IF(LOGGING)
SET(libsrc4_SOURCES ${libsrc4_SOURCES} error4.c)

2
libsrc4/Makefile.am
View File

@ -12,7 +12,7 @@ libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS}
noinst_LTLIBRARIES = libnetcdf4.la
libnetcdf4_la_SOURCES = nc4dispatch.c nc4attr.c nc4dim.c nc4file.c \
nc4grp.c nc4hdf.c nc4internal.c nc4type.c nc4var.c ncfunc.c error4.c \
nc4info.c nc4printer.c
nc4info.c nc4printer.c nc4mem.c nc4memcb.c
EXTRA_DIST = CMakeLists.txt

2
libsrc4/nc4attr.c
View File

@ -590,7 +590,7 @@ exit:
* @param file_type Type of the attribute data in file.
* @param len Number of elements in attribute array.
* @param data Attribute data.
* @param memtype Type of data in memory.
* @param mem_type Type of data in memory.
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid parameters.

252
libsrc4/nc4file.c
View File

@ -16,10 +16,21 @@
#include "nc.h"
#include "nc4internal.h"
#include "nc4dispatch.h"
#include <H5DSpublic.h> /* must be after nc4internal.h */
#include <H5Fpublic.h>
#include "netcdf_mem.h"
#ifdef USE_HDF4
#include <mfhdf.h>
#endif
#include <hdf5_hl.h>
#define DEFAULT_CREATE_MEMSIZE ((size_t)1<<16)
extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
/* From nc4mem.c */
extern int NC4_open_image_file(NC_HDF5_FILE_INFO_T* h5);
extern int NC4_create_image_file(NC_HDF5_FILE_INFO_T* h5);
extern int NC4_extract_file_image(NC_HDF5_FILE_INFO_T* h5);
/** @internal When we have open objects at file close, should
we log them or print to stdout. Default is to log. */
#define LOGOPEN 1
@ -457,7 +468,7 @@ exit:
static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP|NC_64BIT_OFFSET);
/** @internal These flags may not be set for create. */
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_INMEMORY|NC_64BIT_OFFSET|NC_CDF5);
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5);
extern void reportopenobjects(int log, hid_t);
@ -551,12 +562,13 @@ sync_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
*
* @param h5 Pointer to HDF5 file info struct.
* @param abort True if this is an abort.
* @param extractmem True if we need to extract and save final inmemory
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
static int
close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort)
close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort, int extractmem)
{
int retval = NC_NOERR;
@ -589,18 +601,28 @@ close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5, int abort)
MPI_Info_free(&h5->info);
}
#endif
if(h5->fileinfo) free(h5->fileinfo);
if (H5Fclose(h5->hdfid) < 0)
{
int nobjs;
nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL);
/* Apparently we can get an error even when nobjs == 0 */
if(nobjs < 0) {
BAIL_QUIET(NC_EHDFERR);
} else if(nobjs > 0) {
if(h5->fileinfo) free(h5->fileinfo);
/* Check to see if this is an in-memory file and we want to get its
final content
*/
if(extractmem) {
/* File must be read/write */
if(!h5->no_write) {
retval = NC4_extract_file_image(h5);
}
}
if (H5Fclose(h5->hdfid) < 0)
{
int nobjs;
nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL);
/* Apparently we can get an error even when nobjs == 0 */
if(nobjs < 0) {
BAIL_QUIET(NC_EHDFERR);
} else if(nobjs > 0) {
#ifdef LOGGING
char msg[1024];
int logit = 1;
@ -826,8 +848,7 @@ nc4typelen(nc_type type)
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param comm MPI communicator (parallel IO only).
* @param info MPI info (parallel IO only).
* @param parameters extra parameter info (like MPI communicator)
* @param nc Pointer to an instance of NC.
*
* @return ::NC_NOERR No error.
@ -838,25 +859,43 @@ nc4typelen(nc_type type)
* @author Ed Hartnett, Dennis Heimbigner
*/
static int
nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
NC *nc)
nc4_create_file(const char *path, int cmode, void* parameters, NC *nc)
{
hid_t fcpl_id, fapl_id = -1;
unsigned flags;
FILE *fp;
int retval = NC_NOERR;
NC_HDF5_FILE_INFO_T* nc4_info = NULL;
#ifdef USE_PARALLEL4
NC_MPI_INFO* mpiinfo = NULL;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#else /* !USE_PARALLEL4 */
int persist = 0; /* Should diskless try to persist its data into file?*/
#endif
#endif /* !USE_PARALLEL4 */
assert(nc && path);
LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode));
if(cmode & NC_DISKLESS)
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode))))
BAIL(retval);
nc4_info = NC4_DATA(nc);
assert(nc4_info && nc4_info->root_grp);
nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY;
nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS;
if(nc4_info->mem.inmemory && parameters)
nc4_info->mem.memio = *(NC_memio*)parameters;
#ifdef USE_PARALLEL4
else if(parameters) {
mpinfo = (NC_MPI_INFO *)parameters;
comm = mpiinfo->comm;
info = mpiinfo->info;
}
#endif
if(nc4_info->mem.diskless)
flags = H5F_ACC_TRUNC;
else if(cmode & NC_NOCLOBBER)
flags = H5F_ACC_EXCL;
@ -864,23 +903,17 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
flags = H5F_ACC_TRUNC;
/* If this file already exists, and NC_NOCLOBBER is specified,
return an error. */
if (cmode & NC_DISKLESS) {
#ifndef USE_PARALLEL4
if(cmode & NC_WRITE)
persist = 1;
#endif
return an error (unless diskless|inmemory) */
if (nc4_info->mem.diskless) {
if((cmode & NC_WRITE) && (cmode & NC_NOCLOBBER) == 0)
nc4_info->mem.persist = 1;
} else if (nc4_info->mem.inmemory) {
/* ok */
} else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) {
fclose(fp);
return NC_EEXIST;
}
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode))))
BAIL(retval);
nc4_info = NC4_DATA(nc);
assert(nc4_info && nc4_info->root_grp);
/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to
* fail if there are any open objects in the file. */
@ -892,8 +925,7 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
#ifdef USE_PARALLEL4
/* If this is a parallel file create, set up the file creation
property list. */
if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX))
{
if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) {
nc4_info->parallel = NC_TRUE;
if (cmode & NC_MPIIO) /* MPI/IO */
{
@ -935,7 +967,7 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
}
#else /* only set cache for non-parallel... */
if(cmode & NC_DISKLESS) {
if (H5Pset_fapl_core(fapl_id, 4096, persist))
if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist))
BAIL(NC_EDISKLESS);
}
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
@ -974,7 +1006,21 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
H5Pset_coll_metadata_write(fapl_id, 1);
#endif
if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
if(nc4_info->mem.inmemory) {
#if 0
if(nc4_info->mem.memio.size == 0)
nc4_info->memio.size = DEFAULT_CREATE_MEMSIZE; /* last ditch fix */
if(nc4_info->memio.memory == NULL) { /* last ditch fix */
nc4_info->memio.memory = malloc(nc4_info->memio.size);
if(nc4_info->memio.memory == NULL)
BAIL(NC_ENOMEM);
}
assert(nc4_info->memio.size > 0 && nc4_info->memio.memory != NULL);
#endif
retval = NC4_create_image_file(nc4_info);
if(retval)
BAIL(retval);
} else if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
/*Change the return error from NC_EFILEMETADATA to
System error EACCES because that is the more likely problem */
BAIL(EACCES);
@ -1003,7 +1049,7 @@ exit: /*failure exit*/
#endif
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if(!nc4_info) return retval;
close_netcdf4_file(nc4_info,1); /* treat like abort */
close_netcdf4_file(nc4_info,1,0); /* treat like abort */
return retval;
}
@ -1031,22 +1077,12 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
size_t *chunksizehintp, int use_parallel, void *parameters,
NC_Dispatch *dispatch, NC* nc_file)
{
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
int res;
assert(nc_file && path);
LOG((1, "%s: path %s cmode 0x%x comm %d info %d",
__func__, path, cmode, comm, info));
#ifdef USE_PARALLEL4
if (parameters)
{
comm = ((NC_MPI_INFO *)parameters)->comm;
info = ((NC_MPI_INFO *)parameters)->info;
}
#endif /* USE_PARALLEL4 */
LOG((1, "%s: path %s cmode 0x%x parameters %p",
__func__, path, cmode, parameters));
/* If this is our first file, turn off HDF5 error messages. */
if (!nc4_hdf5_initialized)
@ -1089,7 +1125,7 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
nc_file->int_ncid = nc_file->ext_ncid;
res = nc4_create_file(path, cmode, comm, info, nc_file);
res = nc4_create_file(path, cmode, parameters, nc_file);
done:
return res;
@ -2476,27 +2512,47 @@ static int
nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
{
hid_t fapl_id = H5P_DEFAULT;
unsigned flags = (mode & NC_WRITE) ?
H5F_ACC_RDWR : H5F_ACC_RDONLY;
int retval;
unsigned flags;
NC_HDF5_FILE_INFO_T* nc4_info = NULL;
int inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
#ifdef USE_PARALLEL4
NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#endif /* !USE_PARALLEL4 */
NC_MPI_INFO* mpiinfo = NULL;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#endif
LOG((3, "%s: path %s mode %d", __func__, path, mode));
assert(path && nc);
flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY;
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_nc4f_list_add(nc, path, mode)))
BAIL(retval);
nc4_info = NC4_DATA(nc);
assert(nc4_info && nc4_info->root_grp);
nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS);
if(nc4_info->mem.inmemory) {
NC_memio* memparams = NULL;
if(parameters == NULL)
BAIL(NC_EINMEMORY);
memparams = (NC_memio*)parameters;
nc4_info->mem.memio = *memparams; /* keep local copy */
/* As a safeguard, if !locked and NC_WRITE is set,
then we must take control of the incoming memory */
nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED;
if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) {
memparams->memory = NULL;
}
#ifdef USE_PARALLEL4
} else {
mpiinfo = (NC_MPI_INFO*)parameters;
#endif /* !USE_PARALLEL4 */
}
/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to
* fail if there are any open objects in the file. */
@ -2564,12 +2620,13 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
#ifdef HDF5_HAS_COLL_METADATA_OPS
H5Pset_all_coll_metadata_ops(fapl_id, 1 );
#endif
if(inmemory) {
if((nc4_info->hdfid = H5LTopen_file_image(meminfo->memory,meminfo->size,
H5LT_FILE_IMAGE_DONT_COPY|H5LT_FILE_IMAGE_DONT_RELEASE
)) < 0)
if(nc4_info->mem.inmemory) {
/* validate */
if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL)
BAIL(NC_INMEMORY);
retval = NC4_open_image_file(nc4_info);
if(retval)
BAIL(NC_EHDFERR);
nc4_info->no_write = NC_TRUE;
} else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
@ -2610,7 +2667,7 @@ exit:
#endif
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if (!nc4_info) return retval;
close_netcdf4_file(nc4_info,1); /* treat like abort*/
close_netcdf4_file(nc4_info,1,0); /* treat like abort*/
return retval;
}
@ -2898,7 +2955,7 @@ NC4_abort(int ncid)
/* Free any resources the netcdf-4 library has for this file's
* metadata. */
if ((retval = close_netcdf4_file(nc4_info, 1)))
if ((retval = close_netcdf4_file(nc4_info, 1, 0)))
return retval;
/* Delete the file, if we should. */
@ -2913,12 +2970,57 @@ NC4_abort(int ncid)
* @internal Close the netcdf file, writing any changes first.
*
* @param ncid File and group ID.
* @param params any extra parameters in/out of close
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
NC4_close(int ncid)
NC4_close(int ncid, void* params)
{
NC_GRP_INFO_T *grp;
NC *nc;
NC_HDF5_FILE_INFO_T *h5;
int retval;
int inmemory;
LOG((1, "%s: ncid 0x%x", __func__, ncid));
/* Find our metadata for this file. */
if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
return retval;
assert(nc && h5 && grp);
/* This must be the root group. */
if (grp->parent)
return NC_EBADGRPID;
inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY);
/* Call the nc4 close. */
if ((retval = close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0))))
return retval;
if(inmemory && params != NULL) {
NC_memio* memio = (NC_memio*)params;
*memio = h5->mem.memio;
}
return NC_NOERR;
}
/**
* @internal Close an in-memory netcdf file, writing any changes first.
*
* @param ncid File and group ID.
* @param sizep ptr into which the final size is stored
* @param memp ptr into which the final memory is stored
*
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
NC4_close_mem(int ncid, size_t* sizep, void** memp)
{
NC_GRP_INFO_T *grp;
NC *nc;
@ -2937,10 +3039,15 @@ NC4_close(int ncid)
if (grp->parent)
return NC_EBADGRPID;
/* Call the nc4 close. */
if ((retval = close_netcdf4_file(grp->nc4_info, 0)))
/* If the file is not an in-memory file, then treat like normal close */
if((h5->cmode & NC_INMEMORY) == 0)
return NC4_close(ncid,NULL);
/* Call the nc4 close and extract memory */
if ((retval = close_netcdf4_file(grp->nc4_info, 0, 1)))
return retval;
return NC_NOERR;
}
@ -3049,4 +3156,3 @@ nc4_enddef_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
return sync_netcdf4_file(h5);
}

1
libsrc4/nc4internal.c
View File

@ -1707,6 +1707,7 @@ nc4_break_coord_var(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *coord_var, NC_DIM_INFO_T
*
* @param grp The grp of the dimscale-only dataset to be deleted, or a
* higher group in the heirarchy (ex. root group).
* @param dimid dimension id
* @param dim Pointer to the dim with the dimscale-only dataset to be
* deleted.
*

317
libsrc4/nc4mem.c Normal file
View File

@ -0,0 +1,317 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <hdf5.h>
#include <H5DSpublic.h> /* must be after nc4internal.h */
#include <H5Fpublic.h>
#include <H5LTpublic.h>
#include "netcdf.h"
#include "nc4internal.h"
#undef DEBUG
#ifndef HDrealloc
#define HDrealloc(M,Z) realloc(M,Z)
#endif /* HDrealloc */
extern int NC4_open_image(NC_HDF5_FILE_INFO_T* h5);
#if 0
/* Forward */
/* not working */
static int NC4_set_realloc_callback(NC_HDF5_FILE_INFO_T*, hid_t);
#endif
int
NC4_open_image_file(NC_HDF5_FILE_INFO_T* h5)
{
int stat = NC_NOERR;
hid_t hdfid;
/* check arguments */
if(h5->mem.memio.memory == NULL || h5->mem.memio.size == 0)
{stat = NC_EINVAL; goto done;}
/* Figure out the flags */
h5->mem.flags = 0;
if(h5->mem.locked) {
h5->mem.flags |= (H5LT_FILE_IMAGE_DONT_COPY | H5LT_FILE_IMAGE_DONT_RELEASE);
}
if(!h5->no_write)
h5->mem.flags |= H5LT_FILE_IMAGE_OPEN_RW;
/* Create the file but using our version of H5LTopen_file_image */
hdfid = NC4_open_image(h5);
#if 0
hdfid = H5LTopen_file_image(h5->mem.memio.memory,h5->mem.memio.size,h5->mem.flags);
#endif
if(hdfid < 0)
{stat = NC_EHDFERR; goto done;}
#if 0
/* Override the realloc callback if necessary */
if(h5->mem.flags & H5LT_FILE_IMAGE_DONT_COPY) {
if((stat = NC4_set_realloc_callback(h5,hdfid)) != NC_NOERR)
goto done;
}
#endif
/* Return file identifier */
h5->hdfid = hdfid;
done:
return stat;
}
int
NC4_create_image_file(NC_HDF5_FILE_INFO_T* h5)
{
int stat = NC_NOERR;
NC* nc = h5->controller;
hid_t fcpl, fapl, file_id; /* HDF5 identifiers */
size_t alloc_incr; /* Buffer allocation increment */
size_t min_incr = 65536; /* Minimum buffer increment */
#if 0
double buf_prcnt = 0.1f; /* Percentage of buffer size to set
as increment */
#endif
#if 0
see if we can get by without creating an initial memory for the core driver
/* Assume that the default buffer size and memory were set by caller */
if(h5->mem.memio.memory == NULL || h5->memio.size == 0)
{stat = NC_EINVAL; goto done;}
/* set allocation increment to a percentage of the supplied buffer mem, or
* a pre-defined minimum increment value, whichever is larger
*/
if ((buf_prcnt * h5->memio.size) > min_incr)
alloc_incr = (size_t)(buf_prcnt * h5->memio.size);
else
alloc_incr = min_incr;
#else
alloc_incr = min_incr;
#endif
/* Create FAPL to establish core driver */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
{stat = NC_EHDFERR; goto done;}
/* Make fcpl be default for now */
fcpl = H5P_DEFAULT;
/* Configure FAPL to use the core file driver with no backing store */
if (H5Pset_fapl_core(fapl, alloc_incr, 0) < 0)
{stat = NC_EHDFERR; goto done;}
#if 0
/* Assign file image in user buffer to FAPL */
if(H5Pset_file_image(fapl, h5->memio.memory, h5->memio.size) < 0)
{stat = NC_EHDFERR; goto done;}
#endif
/* Create the file */
if((file_id = H5Fcreate(nc->path, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
{stat = NC_EHDFERR; goto done;}
/* Return file identifier */
h5->hdfid = file_id;
h5->mem.fapl = fapl;
done:
if(stat != NC_NOERR) {
/* Close FAPL */
H5Pclose(fapl);
}
return stat;
}
int
NC4_extract_file_image(NC_HDF5_FILE_INFO_T* h5)
{
int stat = NC_NOERR;
hid_t fapl;
herr_t herr;
NC_memio mem;
assert(h5 && h5->hdfid && !h5->no_write);
/* Get the file access property list */
#if 0
fapl = H5Fget_access_plist(h5->hdfid);
#else
fapl = h5->mem.fapl;
#endif
if(fapl < 0)
{stat = NC_EHDFERR; goto done;}
memset(&mem,0,sizeof(mem));
herr = H5Pget_file_image(fapl, &mem.memory, &mem.size);
if(herr < 0)
{stat = NC_EHDFERR; goto done;}
h5->mem.memio = mem;
/* Close FAPL */
if (H5Pclose(fapl) < 0)
{stat = NC_EHDFERR; goto done;}
h5->mem.fapl = 0;
#if 0
/* Pass 1: get the memory chunk size */
size = H5Fget_file_image(hdfid, NULL, 0);
if(size < 0)
{stat = NC_EHDFERR; goto done;}
/* Create the space to hold image */
h5->mem.memio.size = (size_t)size;
h5->mem.memio.memory = malloc(h5->mem.memio.size);
if(h5->mem.memio.memory == NULL)
{stat = NC_ENOMEM; goto done;}
/* Pass 2: get the memory chunk itself */
size = H5Fget_file_image(hdfid, h5->mem.memio.memory, h5->mem.memio.size);
if(size < 0)
{stat = NC_EHDFERR; goto done;}
#endif
done:
return stat;
}
/*-------------------------------------------------------------------------
* Function: image_realloc
*
* Purpose: Reallocates the shared application image buffer and updates data
* structures that manage buffer "copying".
*
* Return: Address of reallocated buffer, if successful. Otherwise, it returns
* NULL.
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
* Modified from code in H5LT.c.
*/
/**
Type H5LT_file_image_ud_t is taken from H5LT.c.
*/
#if 0
/* Data structure to pass application data to callbacks. */
typedef struct {
void *app_image_ptr; /* Pointer to application buffer */
size_t app_image_size; /* Size of application buffer */
void *fapl_image_ptr; /* Pointer to FAPL buffer */
size_t fapl_image_size; /* Size of FAPL buffer */
int fapl_ref_count; /* Reference counter for FAPL buffer */
void *vfd_image_ptr; /* Pointer to VFD buffer */
size_t vfd_image_size; /* Size of VFD buffer */
int vfd_ref_count; /* Reference counter for VFD buffer */
unsigned flags; /* Flags indicate how the file image will */
/* be open */
int ref_count; /* Reference counter on udata struct */
} H5LT_file_image_ud_t;
static void *
local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
void * return_value = NULL;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
/* realloc() is not allowed if the image is open in read-only mode */
if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW))
goto out;
if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) {
if (udata->vfd_image_ptr != ptr)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
/* Modified:
If H5LT_FILE_IMAGE_DONT_RELEASE flag is set: two cases
1. If the realloc new size is <= existing size,
then pretend we did a realloc and return success.
2. Otherwise, realloc() is not allowed because reallocation
may change the address of the buffer. The
new address cannot be communicated to the application
to release it.
*/
if (udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE) {
if(size > udata->vfd_image_size)
goto out;
} else {
if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size)))
goto out;
udata->vfd_image_size = size;
}
return_value = udata->vfd_image_ptr;
} /* end if */
else
goto out;
return(return_value);
out:
return NULL;
} /* end local_image_realloc() */
static int
NC4_set_realloc_callback(NC_HDF5_FILE_INFO_T* h5, hid_t hdfid)
{
int stat = NC_NOERR;
hid_t fapl;
herr_t hstat;
H5FD_file_image_callbacks_t callbacks;
/* Get the file access property list */
fapl = H5Fget_access_plist(hdfid);
if(fapl < 0)
{stat = NC_EHDFERR; goto done;}
/* Now get the callbacks */
memset(&callbacks,0,sizeof(callbacks));
/* For some reason, this is not returning the callbacks */
hstat = H5Pget_file_image_callbacks(fapl, &callbacks);
if(hstat < 0)
{stat = NC_EHDFERR; goto done;}
/* Replace the realloc callback */
callbacks.image_realloc = local_image_realloc;
/* Now reset the callbacks */
hstat = H5Pset_file_image_callbacks(fapl, &callbacks);
if(hstat < 0)
{stat = NC_EHDFERR; goto done;}
done:
return stat;
}
#endif

712
libsrc4/nc4memcb.c Normal file
View File

@ -0,0 +1,712 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* ********************************************************************/
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* @internal Inmemory support
*
* This code is derived from H5Lt.c#H5LTopen_file_image.
* In order to make the netcdf inmemory code work, it is necessary
* to modify some of the callback functions; specifically
* image_malloc, image_realloc, and image_memcpy.
* <p>
* The changes are directed at allowing the caller to
* specify two things.
* <ol>
* <li> To specify (indirectly) the H5LT_FILE_IMAGE_DONT_COPY flag.
* This means that no attempt to realloc the caller provided memory
* should be made. This also means that the memory block pointer
* provided by the caller will be thesame returned by <em>nc_close_memio()</em>.
* <li> The caller overallocates the memory so that there is space
* to allow the file to be modified in place.
* </ol>
* <p>
* The existing implementation of H5LTopen_file_image has two flaws
* with respect to these properties.
* <ol>
* <li> The image_realloc callback fails if
* H5LT_FILE_IMAGE_DONT_COPY flag is set even if there is room
* to allow the memory block to pretend to expand (because
* of overallocation).
* <li> When the caller attempts to get the final memory block,
* the HDF5 library makes a copy, unless theH5LT_FILE_IMAGE_DONT_COPY
* flag is set. This is unnecessary. Note that in this situation,
* the HDF5 library will use
* <em>image_malloc()</em>
* followed by
* <em>image_memcpy()</em>
* </ol>
* <p>
* So, the callback changes to support this properly are as follows.
* <dl>
* <dt>image_realloc</dt>
* <dd>If there is sufficient space (because of overallocation),
* the pretend to realloc and return the incoming memory block
* instead of taking the chance of doing a real realloc.</dd>
* <dt>image_malloc</dt>
* <dd>If the operation being performed is to obtain
* the space to copy the final memory, then just return
* the original memory block. Note that this case is detectable
* because the callback is given the value
* H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET.</dd>
* <dt>image_memcpy</dt>
* <dd>Similar to the image_malloc change.
* Namely, if the operation being performed is to copy
* out the final memory contents, and the final memory block
* is the same as that provided by the caller originally, then
* just do nothing. Again, this case can be detected
* by the occurrence of H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET.</dd>
* </dl>
* @author Dennis Heimbigner
*/
#include <string.h>
#include <stdlib.h>
#include <assert.h>
#include <stdio.h>
#include <hdf5.h>
#include <hdf5_hl.h>
#include "nc4internal.h"
#ifndef HDrealloc
#define HDrealloc(x,y) realloc(x,y)
#endif
#ifndef SUCCEED
#define SUCCEED 0
#define FAIL -1
#endif
#ifndef FALSE
#define FALSE 0
#endif
/* For Lex and Yacc */
#define COL 3
#define LIMIT 512
#define INCREMENT 1024
#define TMP_LEN 256
#define MAX(a,b) (((a)>(b)) ? (a) : (b))
size_t input_len;
char *myinput;
size_t indent = 0;
/* File Image operations
A file image is a representation of an HDF5 file in a memory
buffer. In order to perform operations on an image in a similar way
to a file, the application buffer is copied to a FAPL buffer, which
in turn is copied to a VFD buffer. Buffer copying can decrease
performance, especially when using large file images. A solution to
this issue is to simulate the copying of the application buffer,
when actually the same buffer is used for the FAPL and the VFD.
This is implemented by using callbacks that simulate the standard
functions for memory management (additional callbacks are used for
the management of associated data structures). From the application
standpoint, a file handle can be obtained from a file image by using
the API routine H5LTopen_file_image(). This function takes a flag
argument that indicates the HDF5 library how to handle the given image;
several flag values can be combined by using the bitwise OR operator.
Valid flag values include:
H5LT_FILE_IMAGE_OPEN_RW indicates the HDF5 library to open the file
image in read/write mode. Default is read-only mode.
H5LT_FILE_IMAGE_DONT_COPY indicates the HDF5 library to not copy the
supplied user buffer; the same buffer will be handled by the FAPL and
the VFD driver. Default operation copies the user buffer to the FAPL and
VFD driver.
H5LT_FILE_IMAGE_DONT_RELEASE indicates the HDF5 library to not release
the buffer handled by the FAPL and the VFD upon closing. This flag value
is only applicable when the flag value H5LT_FILE_IMAGE_DONT_COPY is set as
well. The application is responsible to release the image buffer.
*/
/* Data structure to pass application data to callbacks. */
typedef struct {
void *app_image_ptr; /* Pointer to application buffer */
size_t app_image_size; /* Size of application buffer */
void *fapl_image_ptr; /* Pointer to FAPL buffer */
size_t fapl_image_size; /* Size of FAPL buffer */
int fapl_ref_count; /* Reference counter for FAPL buffer */
void *vfd_image_ptr; /* Pointer to VFD buffer */
size_t vfd_image_size; /* Size of VFD buffer */
int vfd_ref_count; /* Reference counter for VFD buffer */
unsigned flags; /* Flags indicate how the file image will */
/* be open */
int ref_count; /* Reference counter on udata struct */
} H5LT_file_image_ud_t;
/* callbacks prototypes for file image ops */
static void *local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *udata);
static void *local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op, void *udata);
static herr_t local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *udata);
static void *local_udata_copy(void *udata);
static herr_t local_udata_free(void *udata);
/* Definition of callbacks for file image operations. */
/*-------------------------------------------------------------------------
* Function: image_malloc
*
* Purpose: Simulates malloc() function to avoid copying file images.
* The application buffer is set to the buffer on only one FAPL.
* Then the FAPL buffer can be copied to other FAPL buffers or
* to only one VFD buffer.
*
* Return: Address of "allocated" buffer, if successful. Otherwise, it returns
* NULL.
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static void *
local_image_malloc(size_t size, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
void * return_value = NULL;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
switch ( file_image_op ) {
/* the app buffer is "copied" to only one FAPL. Afterwards, FAPLs can be "copied" */
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
if (udata->app_image_ptr == NULL)
goto out;
if (udata->app_image_size != size)
goto out;
if (udata->fapl_image_ptr != NULL)
goto out;
if (udata->fapl_image_size != 0)
goto out;
if (udata->fapl_ref_count != 0)
goto out;
udata->fapl_image_ptr = udata->app_image_ptr;
udata->fapl_image_size = udata->app_image_size;
return_value = udata->fapl_image_ptr;
udata->fapl_ref_count++;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
if (udata->fapl_image_ptr == NULL)
goto out;
if (udata->fapl_image_size != size)
goto out;
if (udata->fapl_ref_count == 0)
goto out;
return_value = udata->fapl_image_ptr;
udata->fapl_ref_count++;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
if(!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
/* fake the malloc by returning the original memory */
return_value = udata->fapl_image_ptr;
break;
case H5FD_FILE_IMAGE_OP_FILE_OPEN:
/* FAPL buffer is "copied" to only one VFD buffer */
if (udata->vfd_image_ptr != NULL)
goto out;
if (udata->vfd_image_size != 0)
goto out;
if (udata->vfd_ref_count != 0)
goto out;
if (udata->fapl_image_ptr == NULL)
goto out;
if (udata->fapl_image_size != size)
goto out;
if (udata->fapl_ref_count == 0)
goto out;
udata->vfd_image_ptr = udata->fapl_image_ptr;
udata->vfd_image_size = size;
udata->vfd_ref_count++;
return_value = udata->vfd_image_ptr;
break;
/* added unused labels to shut the compiler up */
case H5FD_FILE_IMAGE_OP_NO_OP:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE:
case H5FD_FILE_IMAGE_OP_FILE_RESIZE:
case H5FD_FILE_IMAGE_OP_FILE_CLOSE:
default:
goto out;
} /* end switch */
return(return_value);
out:
return NULL;
} /* end image_malloc() */
/*-------------------------------------------------------------------------
* Function: image_memcpy
*
* Purpose: Simulates memcpy() function to avoid copying file images.
* The image buffer can be set to only one FAPL buffer, and
* "copied" to only one VFD buffer. The FAPL buffer can be
* "copied" to other FAPLs buffers.
*
* Return: The address of the destination buffer, if successful. Otherwise, it
* returns NULL.
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static void *
local_image_memcpy(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op,
void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
switch(file_image_op) {
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
if (dest != udata->fapl_image_ptr)
goto out;
if (src != udata->app_image_ptr)
goto out;
if (size != udata->fapl_image_size)
goto out;
if (size != udata->app_image_size)
goto out;
if (udata->fapl_ref_count == 0)
goto out;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
if (dest != udata->fapl_image_ptr)
goto out;
if (src != udata->fapl_image_ptr)
goto out;
if (size != udata->fapl_image_size)
goto out;
if (udata->fapl_ref_count < 2)
goto out;
break;
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
/* test: src == dest == original */
if(src != dest || src != udata->fapl_image_ptr)
goto out;
break;
case H5FD_FILE_IMAGE_OP_FILE_OPEN:
if (dest != udata->vfd_image_ptr)
goto out;
if (src != udata->fapl_image_ptr)
goto out;
if (size != udata->vfd_image_size)
goto out;
if (size != udata->fapl_image_size)
goto out;
if (udata->fapl_ref_count == 0)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
break;
/* added unused labels to shut the compiler up */
case H5FD_FILE_IMAGE_OP_NO_OP:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE:
case H5FD_FILE_IMAGE_OP_FILE_RESIZE:
case H5FD_FILE_IMAGE_OP_FILE_CLOSE:
default:
goto out;
} /* end switch */
return(dest);
out:
return NULL;
} /* end image_memcpy() */
/*-------------------------------------------------------------------------
* Function: image_realloc
*
* Purpose: Reallocates the shared application image buffer and updates data
* structures that manage buffer "copying".
*
* Return: Address of reallocated buffer, if successful. Otherwise, it returns
* NULL.
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static void *
local_image_realloc(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
void * return_value = NULL;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
/* realloc() is not allowed if the image is open in read-only mode */
if (!(udata->flags & H5LT_FILE_IMAGE_OPEN_RW))
goto out;
if (file_image_op == H5FD_FILE_IMAGE_OP_FILE_RESIZE) {
if (udata->vfd_image_ptr != ptr)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
/* Modified:
1. If the realloc new size is <= existing size,
then pretend we did a realloc and return success.
This avoids unneccessary heap operations.
2. If the H5LT_FILE_IMAGE_DONT_COPY or
H5LT_FILE_IMAGE_DONT_RELEASE flag is set and the
realloc new size is > existing size, then fail
because the realloc() call may change the address
of the buffer. The new address cannot be
communicated to the application to release it.
3. Otherwise, use realloc(). Note that this may have the
side effect of freeing the previous memory chunk.
*/
if(size <= udata->vfd_image_size) {
/* Ok, pretend we did a realloc */
} else if((udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)
|| (udata->flags & H5LT_FILE_IMAGE_DONT_COPY)) {
goto out; /* realloc MAY violate these flags */
} else {
if (NULL == (udata->vfd_image_ptr = HDrealloc(ptr, size)))
goto out;
udata->vfd_image_size = size;
}
return_value = udata->vfd_image_ptr;
} /* end if */
else
goto out;
return(return_value);
out:
return NULL;
} /* end local_image_realloc() */
/*-------------------------------------------------------------------------
* Function: image_free
*
* Purpose: Simulates deallocation of FAPL and VFD buffers by decreasing
* reference counters. Shared application buffer is actually
* deallocated if there are no outstanding references.
*
* Return: SUCCEED or FAIL
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static herr_t
local_image_free(void *ptr, H5FD_file_image_op_t file_image_op, void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
switch(file_image_op) {
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE:
if (udata->fapl_image_ptr != ptr)
goto out;
if (udata->fapl_ref_count == 0)
goto out;
udata->fapl_ref_count--;
/* release the shared buffer only if indicated by the respective flag and there are no outstanding references */
if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 &&
!(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) {
free(udata->fapl_image_ptr);
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
udata->vfd_image_ptr = NULL;
} /* end if */
break;
case H5FD_FILE_IMAGE_OP_FILE_CLOSE:
if (udata->vfd_image_ptr != ptr)
goto out;
if (udata->vfd_ref_count != 1)
goto out;
udata->vfd_ref_count--;
/* release the shared buffer only if indicated by the respective flag and there are no outstanding references */
if (udata->fapl_ref_count == 0 && udata->vfd_ref_count == 0 &&
!(udata->flags & H5LT_FILE_IMAGE_DONT_RELEASE)) {
free(udata->vfd_image_ptr);
udata->app_image_ptr = NULL;
udata->fapl_image_ptr = NULL;
udata->vfd_image_ptr = NULL;
} /* end if */
break;
/* added unused labels to keep the compiler quite */
case H5FD_FILE_IMAGE_OP_NO_OP:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY:
case H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET:
case H5FD_FILE_IMAGE_OP_FILE_OPEN:
case H5FD_FILE_IMAGE_OP_FILE_RESIZE:
default:
goto out;
} /* end switch */
return(SUCCEED);
out:
return(FAIL);
} /* end image_free() */
/*-------------------------------------------------------------------------
* Function: udata_copy
*
* Purpose: Simulates the copying of the user data structure utilized in the
* management of the "copying" of file images.
*
* Return: Address of "newly allocated" structure, if successful. Otherwise, it
* returns NULL.
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static void *
local_udata_copy(void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
if (udata->ref_count == 0)
goto out;
udata->ref_count++;
return(udata);
out:
return NULL;
} /* end udata_copy */
/*-------------------------------------------------------------------------
* Function: udata_free
*
* Purpose: Simulates deallocation of the user data structure utilized in the
* management of the "copying" of file images. The data structure is
* actually deallocated when there are no outstanding references.
*
* Return: SUCCEED or FAIL
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
static herr_t
local_udata_free(void *_udata)
{
H5LT_file_image_ud_t *udata = (H5LT_file_image_ud_t *)_udata;
/* callback is only used if the application buffer is not actually copied */
if (!(udata->flags & H5LT_FILE_IMAGE_DONT_COPY))
goto out;
if (udata->ref_count == 0)
goto out;
udata->ref_count--;
/* checks that there are no references outstanding before deallocating udata */
if (udata->ref_count == 0 && udata->fapl_ref_count == 0 &&
udata->vfd_ref_count == 0)
free(udata);
return(SUCCEED);
out:
return(FAIL);
} /* end udata_free */
/* End of callbacks definitions for file image operations */
/*-------------------------------------------------------------------------
*
* Public functions
*
*-------------------------------------------------------------------------
*/
/*-------------------------------------------------------------------------
* Function: H5LTopen_file_image
*
* Purpose: Open a user supplied file image using the core file driver.
*
* Return: File identifier, Failure: -1
*
* Programmer: Christian Chilan
*
* Date: October 3, 2011
*
*-------------------------------------------------------------------------
*/
int
NC4_open_image(NC_HDF5_FILE_INFO_T* h5)
{
hid_t fapl, file_id; /* HDF5 identifiers */
unsigned file_open_flags;/* Flags for image open */
char file_name[64]; /* Filename buffer */
size_t alloc_incr; /* Buffer allocation increment */
size_t min_incr = 65536; /* Minimum buffer increment */
double buf_prcnt = 0.1f; /* Percentage of buffer size to set
as increment */
size_t buf_size = h5->mem.memio.size;
void* buf_ptr = h5->mem.memio.memory;
unsigned flags = h5->mem.flags;
static long file_name_counter;
/* Note we use our modified image_realloc -- local_image_realloc */
H5FD_file_image_callbacks_t callbacks = {&local_image_malloc, &local_image_memcpy,
&local_image_realloc, &local_image_free,
&local_udata_copy, &local_udata_free,
(void *)NULL};
/* check arguments */
if (buf_ptr == NULL)
goto out;
if (buf_size == 0)
goto out;
if (flags & (unsigned)~(H5LT_FILE_IMAGE_ALL))
goto out;
/* Create FAPL to transmit file image */
if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
goto out;
/* set allocation increment to a percentage of the supplied buffer size, or
* a pre-defined minimum increment value, whichever is larger
*/
if ((buf_prcnt * buf_size) > min_incr)
alloc_incr = (size_t)(buf_prcnt * buf_size);
else
alloc_incr = min_incr;
/* Configure FAPL to use the core file driver */
if (H5Pset_fapl_core(fapl, alloc_incr, FALSE) < 0)
goto out;
/* Set callbacks for file image ops ONLY if the file image is NOT copied */
if (flags & H5LT_FILE_IMAGE_DONT_COPY) {
H5LT_file_image_ud_t *udata; /* Pointer to udata structure */
/* Allocate buffer to communicate user data to callbacks */
if (NULL == (udata = (H5LT_file_image_ud_t *)malloc(sizeof(H5LT_file_image_ud_t))))
goto out;
/* Initialize udata with info about app buffer containing file image and flags */
udata->app_image_ptr = buf_ptr;
udata->app_image_size = buf_size;
udata->fapl_image_ptr = NULL;
udata->fapl_image_size = 0;
udata->fapl_ref_count = 0;
udata->vfd_image_ptr = NULL;
udata->vfd_image_size = 0;
udata->vfd_ref_count = 0;
udata->flags = flags;
udata->ref_count = 1; /* corresponding to the first FAPL */
/* copy address of udata into callbacks */
callbacks.udata = (void *)udata;
/* Set file image callbacks */
if (H5Pset_file_image_callbacks(fapl, &callbacks) < 0) {
free(udata);
goto out;
} /* end if */
} /* end if */
/* Assign file image in user buffer to FAPL */
if (H5Pset_file_image(fapl, buf_ptr, buf_size) < 0)
goto out;
/* set file open flags */
if (flags & H5LT_FILE_IMAGE_OPEN_RW)
file_open_flags = H5F_ACC_RDWR;
else
file_open_flags = H5F_ACC_RDONLY;
/* define a unique file name */
snprintf(file_name, (sizeof(file_name) - 1), "file_image_%ld", file_name_counter++);
/* Assign file image in FAPL to the core file driver */
if ((file_id = H5Fopen(file_name, file_open_flags, fapl)) < 0)
goto out;
h5->mem.fapl = fapl;
#if 0
/* Close FAPL */
if (H5Pclose(fapl) < 0)
goto out;
#endif
/* Return file identifier */
return file_id;
out:
H5E_BEGIN_TRY {
H5Pclose(fapl);
} H5E_END_TRY;
return -1;
} /* end H5LTopen_file_image() */

2
libsrcp/ncpdispatch.c
View File

@ -236,7 +236,7 @@ done:
static int
NCP_close(int ncid)
NCP_close(int ncid, void* ignored)
{
NC* nc;
NCP_INFO* nc5;

3
nc_test/CMakeLists.txt
View File

@ -59,7 +59,7 @@ IF(LARGE_FILE_TESTS)
ENDIF()
IF(BUILD_DISKLESS)
SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5)
SET(TESTFILES ${TESTFILES} tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory)
IF(USE_NETCDF4)
SET(TESTFILES ${TESTFILES} tst_diskless2)
ENDIF()
@ -89,6 +89,7 @@ IF(BUILD_UTILITIES)
add_sh_test(nc_test run_diskless2)
ENDIF(LARGE_FILE_TESTS)
add_sh_test(nc_test run_diskless5)
add_sh_test(nc_test run_inmemory)
ENDIF(BUILD_DISKLESS)
ENDIF(BUILD_UTILITIES)

8
nc_test/Make0
View File

@ -1,14 +1,14 @@
# Test c output
T=tst_addvar
T=tst_inmemory
ARGS=test_pnetcdf.nc
#ARGS=t
#CMD=valgrind --leak-check=full
CMD=gdb --args
PAR=1
#PAR=1
CFLAGS=-Wall -Wno-unused-variable -Wno-unused-function -g -O -I.. -I../include
CFLAGS=-Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
ifdef PAR
CC=mpicc

6
nc_test/Makefile.am
View File

@ -58,7 +58,7 @@ check_PROGRAMS = $(TESTPROGRAMS)
# Build Diskless test helpers
if BUILD_DISKLESS
check_PROGRAMS += tst_diskless tst_diskless3 tst_diskless4 tst_diskless5
check_PROGRAMS += tst_diskless tst_diskless3 tst_diskless4 tst_diskless5 tst_inmemory
if USE_NETCDF4
check_PROGRAMS += tst_diskless2
endif
@ -68,7 +68,7 @@ TESTS = $(TESTPROGRAMS)
if BUILD_UTILITIES
if BUILD_DISKLESS
TESTS += run_diskless.sh run_diskless5.sh
TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh
if BUILD_MMAP
TESTS += run_mmap.sh
run_mmap.log: run_diskless.log
@ -99,7 +99,7 @@ endif # USE_VALGRIND_TESTS
EXTRA_DIST = test_get.m4 test_put.m4 run_valgrind_tests.sh \
run_diskless.sh run_diskless2.sh run_diskless5.sh run_mmap.sh \
run_pnetcdf_test.sh test_read.m4 test_write.m4 ref_tst_diskless2.cdl \
tst_diskless5.cdl CMakeLists.txt
tst_diskless5.cdl run_inmemory.sh CMakeLists.txt
# These files are created by the tests.
CLEANFILES = nc_test_classic.nc nc_test_64bit.nc nc_test_netcdf4.nc \

49
nc_test/run_inmemory.sh Executable file
View File

@ -0,0 +1,49 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`
#Constants
FILE3=tst_inmemory3
CREATE3=tst_inmemory3_create
FILE4=tst_inmemory4
CREATE4=tst_inmemory4_create
echo ""
echo "*** Testing in-memory operations"
HASNC4=`${top_builddir}/nc-config --has-nc4`
# Execute the core of the inmemory tests
if ! ${execdir}/tst_inmemory; then
echo "FAIL: tst.inmemory"
fi
echo "**** Test ncdump of the resulting inmemory data"
${NCDUMP} -n "${FILE3}" ${FILE3}.nc > ${FILE3}.cdl
${NCDUMP} -n "${FILE3}" ${CREATE3}.nc > ${CREATE3}.cdl
if ! diff -wb ${FILE3}.cdl ${CREATE3}.cdl ; then
echo "FAIL: ${FILE3}.cdl vs ${CREATE3}.cdl
fi
if test "x$HASNC4" = "xyes" ; then
${NCDUMP} ${FILE4}.nc > ${FILE4}.cdl
${NCDUMP} ${CREATE4}.nc > ${CREATE4}.cdl
if diff -wb ${FILE4}.cdl ${CREATE4}.cdl ; then
echo "FAIL: ${FILE4}.cdl vs ${CREATE4}.cdl
fi
# cleanup
rm -f ${FILE3}.nc ${FILE4}.nc ${CREATE3}.nc ${CREATE4}.nc
rm -f ${FILE3}.cdl ${FILE4}.cdl ${CREATE3}.cdl ${CREATE4}.cdl
echo "PASS: all inmemory tests"
exit 0

7
nc_test/tst_def_var_fill.c
View File

@ -10,11 +10,6 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_LIBGEN_H
#include <libgen.h> /* basename() */
#else
#define basename(X) X
#endif
#include <netcdf.h>
@ -52,7 +47,7 @@ int main(int argc, char** argv) {
else strcpy(filename, "tst_def_var_fill.nc");
char *cmd_str = (char*)malloc(strlen(argv[0]) + 256);
sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", basename(argv[0]));
sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", argv[0]);
printf("%-66s ------ ", cmd_str); fflush(stdout);
free(cmd_str);

2
nc_test/tst_diskless5.c
View File

@ -17,7 +17,7 @@
#include "netcdf_mem.h"
#undef MEM
#define DISKLESS
#undef DISKLESS
#define USEINT
#undef DUMP

645
nc_test/tst_inmemory.c Normal file
View File

@ -0,0 +1,645 @@
/** \file \internal
Basic NC_INMEMORY API tests both for netcdf-3 and netcdf-4
Copyright 2011, UCAR/Unidata. See COPYRIGHT file for copying and
redistribution conditions.
*/
#undef DDBG
#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "netcdf.h"
#include "netcdf_mem.h"
#include "ncbytes.h"
#include "nc_tests.h"
#include "err_macros.h"
#include <hdf5.h>
#ifdef USE_NETCDF4
extern int H5Eprint1(FILE * stream);
#endif
#define FLAGS4 (NC_INMEMORY|NC_NETCDF4|NC_CLOBBER)
#define FLAGS3 (NC_INMEMORY|NCCLOBBER)
#define NC_NETCDF3 0
#define MODIFIED 1
#define LARGE_SPACE (1<<18)
#define FILE3 "tst_inmemory3.nc"
#define FILE4 "tst_inmemory4.nc"
#define CREATE3 "tst_inmemory3_create.nc"
#define CREATE4 "tst_inmemory4_create.nc"
#define XFAIL "tst_xfail.nc"
#define MISC "tst_misc.nc"
#define CREATEFILE3 "tst_memcreate3.nc"
#define CREATEFILE4 "tst_memcreate4.nc"
/* Make no dimension larger than this */
#define MAXDIMLEN 100
#define NDIMS 2
#define UNLIM_LEN 2
#define DIM0_NAME "fun"
#define DIM1_NAME "money"
#define DIM1_LEN 8
#define ATT0_NAME "home"
#define ATT0_TEXT "earthship"
#define NVARS 3
#define VAR0_NAME "nightlife"
#define VAR1_NAME "time"
#define VAR2_NAME "taxi_distance"
#define VAR3_NAME "miles"
#define FLOATVAL ((float)42.22)
/*
CDL for created file:
netcdf tst_inmemory3 {
dimensions:
fun = UNLIMITED ; // (2 currently)
money = 8 ;
variables:
int nightlife(fun, money) ;
float time ;
short taxi_distance(money) ;
// global attributes:
:home = "earthship" ;
data:
nightlife =
0, 100, 200, 300, 400, 500, 600, 700,
800, 900, 1000, 1100, 1200, 1300, 1400, 1500 ;
time = 42.22 ;
taxi_distance = 0, 1, 2, 3, 4, 5, 6, 7 ;
}
*/
#ifdef DDBG
#undef ERR
static void
fail(int line) {
fflush(stdout);
fprintf(stderr,"\nline=%d\n",line);
fflush(stderr);
exit(1);
}
#define ERR fail(__LINE__)
#endif
/* Store any error message */
static char errmsg[4096];
static int
check(int stat, const char* file, int line, int xfail)
{
int pass = ((!xfail && stat == NC_NOERR) || (xfail && stat != NC_NOERR));
fflush(stdout);
if(!xfail) {
if(stat != NC_NOERR) {
fprintf(stderr,"***Fail: line: %d; status=%d %s\n",
line,stat,nc_strerror(stat));
err++;
}
} else { /*xfail*/
if(stat == NC_NOERR) {
fprintf(stderr,"***XFail Fail: line: %d; passed instead of failed\n",
line);
err++;
} else {
fprintf(stderr,"\t***XFail: status=%d %s\n",
stat,nc_strerror(stat));
}
stat = NC_NOERR; /* because xfail */
}
fflush(stderr);
return stat;
}
#define CHECK(expr) {stat = check((expr),__FILE__,__LINE__,0); if(stat) return stat;}
#define XCHECK(expr) {stat = check((expr),__FILE__,__LINE__,1); if(stat) return stat;}
#define REPORT(xfail,expr) {if((xfail)) {XCHECK((expr));} else {CHECK((expr));}}
/**************************************************/
static void
removefile(const char* path)
{
unlink(path);
}
static int
readfile(const char* path, NC_memio* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t filesize = 0;
size_t count = 0;
char* memory = NULL;
char* p = NULL;
/* Open the file for reading */
#ifdef _MSC_VER
f = fopen(path,"rb");
#else
f = fopen(path,"r");
#endif
if(f == NULL)
{status = errno; goto done;}
/* get current filesize */
if(fseek(f,0,SEEK_END) < 0)
{status = errno; goto done;}
filesize = (size_t)ftell(f);
/* allocate memory */
memory = malloc((size_t)filesize);
if(memory == NULL)
{status = NC_ENOMEM; goto done;}
/* move pointer back to beginning of file */
rewind(f);
count = filesize;
p = memory;
while(count > 0) {
size_t actual;
actual = fread(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
}
if(memio) {
memio->size = (size_t)filesize;
memio->memory = memory;
}
done:
if(status != NC_NOERR && memory != NULL)
free(memory);
if(f != NULL) fclose(f);
return status;
}
static int
writefile(const char* path, NC_memio* memio)
{
int status = NC_NOERR;
FILE* f = NULL;
size_t count = 0;
char* memory = NULL;
char* p = NULL;
/* Open the file for writing */
#ifdef _MSC_VER
f = fopen(path,"wb");
#else
f = fopen(path,"w");
#endif
if(f == NULL)
{status = errno; goto done;}
count = memio->size;
p = memio->memory;
while(count > 0) {
size_t actual;
actual = fwrite(p,1,count,f);
if(actual == 0 || ferror(f))
{status = NC_EIO; goto done;}
count -= actual;
p += actual;
}
done:
if(f != NULL) fclose(f);
return status;
}
/* Duplicate an NC_memio instance; needed to avoid
attempting to use memory that might have been realloc'd
Allow the new memory to be larger than the src memory
*/
int
duplicatememory(NC_memio* src, NC_memio* target, size_t alloc)
{
if(src == NULL || target == NULL || src->size == 0 || src->memory == NULL)
return NC_EINVAL;
*target = *src;
if(alloc == 0) alloc = src->size;
target->memory = malloc(alloc);
if(target->memory == NULL)
return NC_ENOMEM;
memcpy(target->memory,src->memory,src->size);
target->size = alloc;
return NC_NOERR;
}
/*
Given an ncid of a created file, fill in the meta-data
and data as described by the above CDL. Do not close
the file.
*/
static int
define_metadata(int ncid)
{
int stat = NC_NOERR;
int dimid[NDIMS], varid0, varid1, varid2;
short short_data[DIM1_LEN];
size_t start[1] = {0};
size_t count[1] = {DIM1_LEN};
int dimprod = (UNLIM_LEN*DIM1_LEN);
int i;
float float_data;
int nightdata[UNLIM_LEN*DIM1_LEN] ;
/* Create data to write */
float_data = FLOATVAL;
for (i = 0; i < DIM1_LEN; i++)
short_data[i] = i;
for (i = 0; i < dimprod; i++)
nightdata[i] = (100*i);
CHECK(nc_put_att_text(ncid, NC_GLOBAL, ATT0_NAME,
sizeof(ATT0_TEXT), ATT0_TEXT));
CHECK(nc_def_dim(ncid, DIM0_NAME, NC_UNLIMITED, &dimid[0]));
CHECK(nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimid[1]));
CHECK(nc_def_var(ncid, VAR0_NAME, NC_INT, NDIMS, dimid, &varid0));
CHECK(nc_def_var(ncid, VAR1_NAME, NC_FLOAT, 0, NULL, &varid1));
CHECK(nc_def_var(ncid, VAR2_NAME, NC_SHORT, 1, &dimid[1], &varid2));
CHECK(nc_enddef(ncid));
CHECK(nc_put_vara_float(ncid, varid1, NULL, NULL, &float_data));
CHECK(nc_put_vara_short(ncid, varid2, start, count, short_data));
{
size_t start[2] = {0,0};
size_t count[2] = {2,DIM1_LEN};
CHECK(nc_put_vara_int(ncid, varid0, start, count, nightdata));
}
return stat;
}
/*
Create our reference file as a real on-disk file
and read it back into memory
*/
static int
create_reference_file(const char* filename, int mode, NC_memio* filedata)
{
int stat = NC_NOERR;
int ncid;
CHECK(nc_create(filename, mode|NC_CLOBBER, &ncid)); /* overwrite */
CHECK(define_metadata(ncid));
CHECK(nc_close(ncid));
/* Read back the contents of the file into memory */
if(filedata != NULL) {
memset(filedata,0,sizeof(NC_memio));
CHECK(readfile(filename,filedata));
}
return stat;
}
static int
modify_file(int ncid)
{
int stat = NC_NOERR;
size_t i;
int varid3;
int dimid[1];
size_t len;
int data[MAXDIMLEN];
/* Get id of the unlimited dimension */
if((stat=nc_inq_dimid(ncid, DIM0_NAME, dimid))) goto done;
/* get current dim length */
if((stat=nc_inq_dimlen(ncid, dimid[0], &len))) goto done;
/* open file for new meta-data */
if((stat=nc_redef(ncid))) goto done;
/* Define a new variable */
if((stat=nc_def_var(ncid, VAR3_NAME, NC_INT, 1, dimid, &varid3))) goto done;
/* close metadata */
if((stat=nc_enddef(ncid))) goto done;
/* Write data to new variable */
for(i=0;i<len;i++)
data[i] = i;
if((stat=nc_put_var_int(ncid,varid3,data))) goto done;
done:
return stat;
}
/* Verify the content of a file */
static int
verify_file(int ncid, int modified)
{
int stat = NC_NOERR;
int i,tmp;
int dimid_in[NDIMS];
int dimid[NDIMS];
int ndims_in, nvars_in, natts_in, unlimdimid_in;
char name_in[NC_MAX_NAME + 1], att0_in[NC_MAX_NAME + 1];
nc_type type_in;
size_t len_in;
int varid[4];
int nightdata_in[UNLIM_LEN*DIM1_LEN] ;
float float_data_in;
int milesdata_in[MAXDIMLEN];
int dimprod = UNLIM_LEN * DIM1_LEN;
int modcount = (modified?1:0);
CHECK(nc_inq(ncid, &ndims_in, &nvars_in, &natts_in, &unlimdimid_in));
if (ndims_in != 2 || nvars_in != NVARS+modified || natts_in != 1 || unlimdimid_in != 0)
CHECK(NC_EINVAL);
/* Get all the dimids */
tmp = 0;
#ifdef USE_NETCDF4
CHECK((nc_inq_dimids(ncid,&tmp,dimid,1)));
if(tmp != NDIMS) CHECK(NC_EINVAL);
/* Get all the varids */
tmp = 0;
CHECK((nc_inq_varids(ncid,&tmp,varid)));
if(tmp != (NVARS+modified)) CHECK(NC_EINVAL);
#else
{ /* Simulate nc_inq_varids and nc_inq_dimids */
int j;
int dimcnt = 0;
int varcnt = 0;
CHECK(nc_inq(ncid, &dimcnt, &varcnt, NULL, NULL));
for(j=0;j<dimcnt;j++) dimid[j] = j;
for(j=0;j<varcnt;j++) varid[j] = j;
}
#endif
CHECK(nc_get_att_text(ncid, NC_GLOBAL, ATT0_NAME, att0_in));
att0_in[sizeof(ATT0_TEXT)] = '\0';
if (strcmp(att0_in, ATT0_TEXT)) CHECK(NC_EINVAL);
/* CHECK dimensions. */
CHECK(nc_inq_dim(ncid, dimid[0], name_in, &len_in));
if (strcmp(name_in, DIM0_NAME)) CHECK(NC_EINVAL);
CHECK(nc_inq_dim(ncid, dimid[1], name_in, &len_in));
if (strcmp(name_in, DIM1_NAME) || len_in != DIM1_LEN) CHECK(NC_EINVAL);
/* CHECK variables. */
CHECK(nc_inq_var(ncid, varid[0], name_in, &type_in, &ndims_in, dimid_in, &natts_in));
if (strcmp(name_in, VAR0_NAME) || type_in != NC_INT || ndims_in != NDIMS ||
dimid_in[0] != 0 || dimid_in[1] != 1 || natts_in != 0) CHECK(NC_EINVAL);
CHECK(nc_inq_var(ncid, varid[1], name_in, &type_in, &ndims_in, dimid_in, &natts_in));
if (strcmp(name_in, VAR1_NAME) || type_in != NC_FLOAT || ndims_in != 0 ||
natts_in != 0) CHECK(NC_EINVAL);
CHECK(nc_inq_var(ncid, varid[2], name_in, &type_in, &ndims_in, dimid_in, &natts_in));
if (strcmp(name_in, VAR2_NAME) || type_in != NC_SHORT || ndims_in != 1 ||
dimid_in[0] != 1 || natts_in != 0) CHECK(NC_EINVAL);
CHECK(nc_get_var_int(ncid, varid[0], nightdata_in));
for(i=0;i<dimprod;i++) {
if(nightdata_in[i] != (100*i)) CHECK(NC_EINVAL);
}
CHECK(nc_get_vara_float(ncid, varid[1], NULL, NULL, &float_data_in));
if (float_data_in != FLOATVAL) CHECK(NC_EINVAL);
if(modified) {
size_t unlimlen;
CHECK(nc_inq_var(ncid, varid[3], name_in, &type_in, &ndims_in, dimid_in, &natts_in));
if (strcmp(name_in, VAR3_NAME) || type_in != NC_INT || ndims_in != 1 ||
dimid_in[0] != 0 || natts_in != 0) CHECK(NC_EINVAL);
CHECK(nc_inq_dimlen(ncid, dimid_in[0], &unlimlen));
CHECK(nc_get_var_int(ncid, varid[3], milesdata_in));
for(i=0;i<unlimlen;i++) {
if(milesdata_in[i] != i) CHECK(NC_EINVAL);
}
}
return stat;
}
void
memiofree(NC_memio* memio)
{
if(memio != NULL) {
if(memio->memory != NULL)
free(memio->memory);
}
}
static int
test_open(const char* path, NC_memio* filedata, int mode)
{
int stat = NC_NOERR;
NC_memio duplicate;
NC_memio* finaldata = NULL;
int ncid;
int xmode = mode; /* modified mode */
finaldata = calloc(1,sizeof(NC_memio));
fprintf(stderr,"\n\t***Test open 1: nc_open_mem(): read-only\n");
CHECK(duplicatememory(filedata,&duplicate,0));
CHECK(nc_open_mem(path, xmode, duplicate.size, duplicate.memory, &ncid));
CHECK(verify_file(ncid,!MODIFIED));
CHECK(nc_close(ncid));
free(duplicate.memory);
fprintf(stderr,"\n\t***Test open 2: nc_open_memio(): read-only\n");
CHECK(duplicatememory(filedata,&duplicate,0));
duplicate.flags = NC_MEMIO_LOCKED;
CHECK(nc_open_memio(path, xmode, &duplicate, &ncid))
CHECK(verify_file(ncid,!MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
/* Verify that finaldata is same */
if(finaldata->size != duplicate.size) CHECK(NC_EINVAL);
if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata->memory);
fprintf(stderr,"\n\t***Test open 3: nc_open_memio(): read-write, copy\n");
xmode |= NC_WRITE; /* allow file to be modified */
CHECK(duplicatememory(filedata,&duplicate,0));
CHECK(nc_open_memio(path, xmode, &duplicate, &ncid))
/* modify file */
CHECK(modify_file(ncid));
CHECK(verify_file(ncid,MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
/* Verify that finaldata is same */
if(finaldata->size < filedata->size) CHECK(NC_EINVAL);
/* As a safeguard, the memory in duplicate should have been set to NULL*/
free(finaldata->memory);
fprintf(stderr,"\n\t***Test open 4: nc_open_memio(): read-write, locked, extra space\n");
/* Store the filedata in a memory chunk that leaves room for modification */
CHECK(duplicatememory(filedata,&duplicate,LARGE_SPACE));
/* Lock the duplicate memory */
duplicate.flags |= NC_MEMIO_LOCKED;
xmode |= NC_WRITE; /* allow file to be modified */
CHECK(nc_open_memio(path, xmode, &duplicate, &ncid))
/* modify file */
CHECK(modify_file(ncid));
CHECK(verify_file(ncid,MODIFIED));
CHECK(nc_close_memio(ncid,finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
/* Check returned finaldata */
if(finaldata->size != duplicate.size) CHECK(NC_EINVAL);
if(finaldata->memory != duplicate.memory) CHECK(NC_EINVAL);
free(finaldata->memory);
return stat;
}
static int
test_create(const char* path, int mode)
{
int stat = NC_NOERR;
NC_memio* finaldata = NULL;
int ncid;
int xmode = mode;
fprintf(stderr,"\n\t***Test create 1: nc_create_memio(): no initialsize\n");
CHECK(nc_create_mem(path, xmode, 0, &ncid))
/* create file metadata */
CHECK(define_metadata(ncid));
CHECK(verify_file(ncid,!MODIFIED));
finaldata = calloc(1,sizeof(NC_memio));
CHECK(nc_close_memio(ncid,finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
free(finaldata->memory);
fprintf(stderr,"\n\t***Test create 2: nc_create_memio(): initialsize; save file\n");
CHECK(nc_create_mem(path, xmode, LARGE_SPACE, &ncid))
/* create file metadata */
CHECK(define_metadata(ncid));
CHECK(verify_file(ncid,!MODIFIED));
finaldata = calloc(1,sizeof(NC_memio));
CHECK(nc_close_memio(ncid,finaldata));
/* Published returned finaldata */
fprintf(stderr,"\tfinaldata: size=%lld memory=%p\n",(unsigned long long)finaldata->size,finaldata->memory);
/* Write out the final data as a .nc file */
CHECK(writefile(path,finaldata));
free(finaldata->memory);
return stat;
}
static int
test_misc(const char* path, int mode, NC_memio* filedata)
{
int stat = NC_NOERR;
int ncid;
int xmode = mode;
NC_memio duplicate;
fprintf(stderr,"\n\t***Test misc 1: use nc_close on created inmemory file\n");
CHECK(nc_create_mem(MISC, xmode, 0, &ncid))
CHECK(nc_close(ncid));
removefile(MISC);
fprintf(stderr,"\n\t***Test misc 2: use nc_close on opened inmemory file\n");
CHECK(duplicatememory(filedata,&duplicate,0));
CHECK(nc_open_memio(path, xmode, &duplicate, &ncid))
CHECK(verify_file(ncid,!MODIFIED));
CHECK(nc_close(ncid));
removefile(MISC);
return stat;
}
/* Test various edge conditions to ensure they fail correctly */
static int
test_xfail(const char* path, int mode, NC_memio* filedata)
{
int stat = NC_NOERR;
NC_memio duplicate;
int ncid;
int xmode = mode; /* modified mode */
fprintf(stderr,"\n\t***Test xfail 1: nc_open_mem(): write to read-only\n");
CHECK(duplicatememory(filedata,&duplicate,0));
CHECK(nc_open_mem(XFAIL, xmode, duplicate.size, duplicate.memory, &ncid));
XCHECK(nc_redef(ncid));
CHECK(nc_abort(ncid));
free(duplicate.memory);
fprintf(stderr,"\n\t***Test xfail 2: nc_open_memio(): modify without overallocating\n");
if((mode & NC_NETCDF4)) {
fprintf(stderr,"\tSuppressed because of HDF5 library bug\n");
} else {
/* With HDF5 1.8.20, and possibly other versions,
this tests causes a seg fault in the HDF5 Library.
So until it is fixed, just leave well enough alone */
NC_memio* finaldata = NULL;
finaldata = calloc(1,sizeof(NC_memio));
CHECK(duplicatememory(filedata,&duplicate,0));
duplicate.flags = NC_MEMIO_LOCKED;
xmode |= NC_WRITE;
CHECK(nc_open_memio(XFAIL, xmode, &duplicate, &ncid))
XCHECK(modify_file(ncid));
CHECK(nc_abort(ncid));
free(finaldata->memory);
}
return stat;
}
int
main(int argc, char **argv)
{
int stat = NC_NOERR;
int i;
void* filedata = NULL;
NC_memio filedata3;
NC_memio filedata4;
nc_set_log_level(0);
#ifdef USE_NETCDF4
H5Eprint1(stderr);
#endif
fprintf(stderr,"\n*** Testing the inmemory API: netcdf-3.\n");
CHECK(create_reference_file(FILE3,NC_NETCDF3,&filedata3)); /* netcdf-3 */
CHECK(test_open(FILE3,&filedata3,NC_NETCDF3));
CHECK(test_create(CREATE3,NC_NETCDF3));
CHECK(test_misc(FILE3, NC_NETCDF3, &filedata3));
CHECK(test_xfail(FILE3, NC_NETCDF3, &filedata3));
memiofree(&filedata3);
#ifdef USE_NETCDF4
fprintf(stderr,"\n*** Testing the inmemory API: netcdf-4.\n");
CHECK(create_reference_file(FILE4,NC_NETCDF4,&filedata4));
CHECK(test_open(FILE4,&filedata4,NC_NETCDF4));
CHECK(test_create(CREATE4,NC_NETCDF4));
CHECK(test_misc(FILE4,NC_NETCDF4, &filedata4));
CHECK(test_xfail(FILE4, NC_NETCDF4, &filedata4));
memiofree(&filedata4);
#endif
SUMMARIZE_ERR;
FINAL_RESULTS;
return 0;
}