mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

add NC_MMAP flag and tests

Browse Source
This commit is contained in:
Dennis Heimbigner 2012-06-23 19:25:49 +00:00
parent 1041a4c25b
commit a6b932826b
11 changed files with 180 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
RELEASE_NOTES
View File

@ -6,6 +6,11 @@ https://www.unidata.ucar.edu/jira/browse/NCF-XXX .
VERSION COMMENTS
------- --------
4.2.1 Released 2012-??-??
Added a Nspecific NC_MMAP mode flag to modify
behavior of NC_DISKLESS.
4.2.1-rc1 Released 2012-06-18
Ported static and shared libraries (DLL's) for both

2
cf
View File

@ -129,11 +129,11 @@ FLAGS="$FLAGS --enable-extra-tests"
FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --enable-large-file-tests"
#FLAGS="$FLAGS --disable-testsets"
#FLAGS="$FLAGS --enable-mmap"
#FLAGS="$FLAGS --disable-dap-remote-tests"
#FLAGS="$FLAGS --enable-doxygen"
FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --enable-mmap"
FLAGS="$FLAGS --disable-shared"
#FLAGS="$FLAGS --enable-shared"

6
configure.ac
View File

@ -594,11 +594,11 @@ AC_MSG_RESULT($enable_diskless)
# check for useful, but not essential, memio support
AC_CHECK_FUNCS([memmove getpagesize sysconf])
# Does the user want to use the mmap for NC_DISKLESS?
AC_MSG_CHECKING([whether mmap will be used for in-memory files])
# Does the user want to allow use of mmap for NC_DISKLESS?
AC_MSG_CHECKING([whether mmap is enabled for in-memory files])
AC_ARG_ENABLE([mmap],
[AS_HELP_STRING([--enable-mmap],
[use mmap for in-memory (NC_DISKLESS) files])])
[allow mmap for in-memory files])])
test "x$enable_mmap" = xyes || enable_mmap=no
AC_MSG_RESULT($enable_mmap)

3
include/netcdf.h
View File

@ -121,6 +121,7 @@ extern "C" {
#define NC_NOCLOBBER 0x0004 /**< Don't destroy existing file. Mode flag for nc_create(). */
#define NC_DISKLESS 0x0008 /**< Use diskless file. Mode flag for nc_open() or nc_create(). */
#define NC_MMAP 0x0010 /**< Use diskless file with mmap. Mode flag for nc_open() or nc_create(). */
#define NC_CLASSIC_MODEL 0x0100 /**< Enforce classic model. Mode flag for nc_create(). */
#define NC_64BIT_OFFSET 0x0200 /**< Use large (64-bit) file offsets. Mode flag for nc_create(). */
@ -368,7 +369,7 @@ by the desired type. */
#define NC_ESTORAGE (-126) /**< Can't specify both contiguous and chunking. */
#define NC_EBADCHUNK (-127) /**< Bad chunksize. */
#define NC_ENOTBUILT (-128) /**< Attempt to use feature that was not turned on when netCDF was built. */
#define NC_EDISKLESS (-129) /**< Error in using diskless access. */
#define NC_EDISKLESS (-129) /**< Error in using diskless access. */
#define NC4_LAST_ERROR (-129)

13
libdispatch/dfile.c
View File

@ -219,7 +219,9 @@ to many extensions to the in-memory space for the file.
Normally, NC_DISKLESS allocates space in the heap for storing
the in-memory file. If, however, the ./configure flags --enable-mmap
is used, then mmap will be used.
is used, and the additional mode flag NC_MMAP
is specified, then the file will be created using the operating system
MMAP facility.
Note that nc_create(path,cmode,ncidp) is equivalent to the invocation of
nc__create(path,cmode,NC_SIZEHINT_DEFAULT,NULL,ncidp).
@ -307,7 +309,7 @@ diskless.nc whose content will be lost when nc_close() is called.
int status = NC_NOERR;
int ncid;
...
status = nc_create("foo_HDF5_classic.nc", NC_DISKLESS, &ncid);
status = nc_create("diskless.nc", NC_DISKLESS, &ncid);
if (status != NC_NOERR) handle_error(status);
@endcode
@ -321,7 +323,7 @@ in a file named diskless.nc when nc_close() is called.
int status = NC_NOERR;
int ncid;
...
status = nc_create("foo_HDF5_classic.nc", NC_DISKLESS|NC_WRITE, &ncid);
status = nc_create("diskless.nc", NC_DISKLESS|NC_WRITE, &ncid);
if (status != NC_NOERR) handle_error(status);
@endcode
@ -457,8 +459,9 @@ access, programs that do not access data sequentially may see some
performance improvement by setting the NC_SHARE flag.
This procedure may also be invoked with the NC_DISKLESS flag
set in the mode argument, but ONLY if the file type is NOT NC_NETCDF4,
which means it must be a classic format file.
set in the mode argument if the file to be opened
is a classic format file. If the file is of type NC_NETCDF4,
then the NC_DISKLESS flag will be ignored.
If NC_DISKLESS is specified, then the whole file is read completely into
memory. In effect this creates an in-memory cache of the file.
If the mode flag also specifies NC_WRITE, then the in-memory cache

5
libsrc/Makefile.am
View File

@ -19,11 +19,10 @@ nclistmgr.c putget.c attr.c nc3dispatch.c nc.c var.c dim.c ncx.c \
ncx.h lookup3.c pstdint.h ncio.c ncio.h
if BUILD_DISKLESS
libnetcdf3_la_SOURCES += memio.c
if BUILD_MMAP
libnetcdf3_la_SOURCES += mmapio.c
else !BUILD_MMAP
libnetcdf3_la_SOURCES += memio.c
endif !BUILD_MMAP
endif BUILD_MMAP
endif BUILD_DISKLESS
# Does the user want to use ffio, a replacement for posixio for Cray

20
libsrc/ncio.c
View File

@ -28,10 +28,9 @@ extern int ffio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
# ifdef USE_MMAP
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
# else
# endif
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int memio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
# endif
#endif
int
@ -40,12 +39,14 @@ ncio_create(const char *path, int ioflags, size_t initialsz,
ncio** iopp, void** const mempp)
{
#ifdef USE_DISKLESS
if(fIsSet(ioflags,NC_DISKLESS))
if(fIsSet(ioflags,NC_DISKLESS)) {
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP))
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
# else
else
# endif /*USE_MMAP*/
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
# endif
}
#endif
#ifdef USE_FFIO
@ -66,10 +67,11 @@ ncio_open(const char *path, int ioflags,
#ifdef USE_DISKLESS
if(fIsSet(ioflags,NC_DISKLESS)) {
# ifdef USE_MMAP
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
# else
return memio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
# endif
if(fIsSet(ioflags,NC_MMAP))
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
else
# endif /*USE_MMAP*/
return memio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
}
#endif
#ifdef USE_FFIO

5
nc_test/Makefile.am
View File

@ -62,6 +62,9 @@ TESTS = $(TESTPROGRAMS)
if BUILD_DISKLESS
TESTS += run_diskless.sh
if BUILD_MMAP
TESTS += run_mmap.sh
endif
if LARGE_FILE_TESTS
TESTS += run_diskless2.sh
endif
@ -81,7 +84,7 @@ endif # USE_VALGRIND_TESTS
# Distribute the .c files so that m4 isn't required on the users
# machine.
EXTRA_DIST = test_get.m4 test_put.m4 run_valgrind_tests.sh \
run_diskless.sh run_diskless2.sh
run_diskless.sh run_diskless2.sh run_mmap.sh
# ref_tst_diskless2.cdl is for diff comparison and to produce tst_diskless2.c
EXTRA_DIST += ref_tst_diskless2.cdl

63
nc_test/run_mmap.sh Executable file
View File

@ -0,0 +1,63 @@
#!/bin/sh
set -e
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`
#Constants
FILE1=tst_diskless.nc
FILE2=tst_diskless2.nc
FILE3=tst_diskless3.nc
echo ""
echo "*** Testing in-memory (diskless) files with mmap"
HASNC4=`../nc-config --has-nc4`
echo "**** Test diskless+mmap netCDF classic file without persistence"
./tst_diskless mmap
echo "PASS: diskless+mmap netCDF classic file without persistence"
echo ""
echo "**** Test diskless+mmap netCDF classic file with persistence"
rm -f $FILE1
./tst_diskless mmap persist
if test -f $FILE1 ; then
echo "**** $FILE1 created"
# ../ncdump/ncdump $FILE1
echo "PASS: diskless+mmap netCDF classic file with persistence"
else
echo "#### $FILE1 not created"
echo "FAIL: diskless+mmap netCDF classic file with persistence"
fi
rm -f tmp1.cdl tmp2.cdl tmp1.nc tmp2.nc
echo ""
echo "**** Testing nc_open in-memory (diskless+mmap) files"
# clear old files
rm -f tst_diskless3_file.cdl tst_diskless3_memory.cdl
echo ""
echo "**** Create and modify file without using diskless+mmap"
rm -f $FILE3
./tst_diskless3
../ncdump/ncdump $FILE3 >tst_diskless3_file.cdl
echo ""
echo "**** Create and modify file using diskless+mmap"
rm -f $FILE3
./tst_diskless3 diskless mmap
../ncdump/ncdump $FILE3 >tst_diskless3_memory.cdl
# compare
diff tst_diskless3_file.cdl tst_diskless3_memory.cdl
# cleanup
rm -f $FILE3 tst_diskless3_file.cdl tst_diskless3_memory.cdl
exit

53
nc_test/tst_diskless.c
View File

@ -5,6 +5,8 @@ Copyright 2011, UCAR/Unidata. See COPYRIGHT file for copying and
redistribution conditions.
*/
#undef DDBG
#include <config.h>
#include <nc_tests.h>
#include <stdio.h>
@ -20,6 +22,20 @@ redistribution conditions.
#define CAPACITOR "capacitor_value"
#define NUM555 "number_of_555_timer_chips"
#ifdef DDBG
#undef ERR
void fail(int line) {
fflush(stdout);
fprintf(stderr,"\nline=%d\n",line);
fflush(stderr);
exit(1);
}
#define ERR fail(__LINE__)
#endif
/* Control flags */
static int flags, persist, usenetcdf4, mmap;
static void
removefile(int persist, char* filename)
{
@ -37,17 +53,18 @@ removefile(int persist, char* filename)
int
main(int argc, char **argv)
{
int i, flags, persist, usenetcdf4;
int i;
char* filename = "tst_diskless.nc";
/* Set defaults */
persist = 0;
usenetcdf4 = 0;
mmap = 0;
for(i=1;i<argc;i++) {
if(strcmp(argv[i],"netcdf4")==0) usenetcdf4=1;
else if(strcmp(argv[i],"persist")==0) persist=1;
else if(strcmp(argv[i],"mmap")==0) mmap=1;
/* ignore anything not recognized */
}
@ -55,9 +72,12 @@ main(int argc, char **argv)
usenetcdf4 = 0;
#endif
if(mmap)
usenetcdf4 = 0;
flags = usenetcdf4?FLAGS4:FLAGS3;
if(persist) flags |= PERSIST;
if(mmap) flags |= NC_MMAP;
printf("\n*** Testing the diskless API.\n");
printf("*** testing diskless file with scalar vars...");
@ -113,6 +133,33 @@ printf("*** testing diskless file with scalar vars...");
abort(); //ERR;
}
SUMMARIZE_ERR;
if(!usenetcdf4 && persist) {
int ncid, varid0, varid1, varid2;
float float_data = 3.14, float_data_in;
int int_data = 42, int_data_in;
short short_data = 2, short_data_in;
printf("*** testing diskless open of previously created file...");
if (nc_open(filename, flags, &ncid)) ERR;
/* Read and compare */
if (nc_inq_varid(ncid, RESISTOR, &varid0)) ERR;
if (nc_inq_varid(ncid, CAPACITOR, &varid1)) ERR;
if (nc_inq_varid(ncid, NUM555, &varid2)) ERR;
if (nc_get_vara_int(ncid, varid0, NULL, NULL, &int_data_in)) ERR;
if (int_data_in != int_data) ERR;
if (nc_get_vara_float(ncid, varid1, NULL, NULL, &float_data_in)) ERR;
if (float_data_in != float_data) ERR;
if (nc_get_vara_short(ncid, varid2, NULL, NULL, &short_data_in)) ERR;
if (short_data_in != short_data) ERR;
nc_close(ncid);
}
SUMMARIZE_ERR;
printf("*** testing creation of simple diskless file...");
{
#define NDIMS 2

39
nc_test/tst_diskless3.c
View File

@ -34,14 +34,16 @@
#define TITLE " OUTPUT FROM WRF V2.0.3.1 MODEL"
#define ATT_NAME2 "TITLE"
/* Global */
int diskless = 0;
#undef ERR
#define ERR report(status,__LINE__)
static int status = NC_NOERR;
/* Control flags */
static int persist, usenetcdf4, mmap, diskless;
static int diskmode;
void report(int stat, int lno)
{
fprintf(stderr,"line: %d ; %s\n",stat,nc_strerror(stat));
@ -91,7 +93,7 @@ test_two_growing_with_att(const char *testfile)
if((status=nc_close(ncid))) ERR;
/* Reopen the file and check it. */
if((status=nc_open(testfile, NC_DISKLESS|NC_WRITE, &ncid))) ERR;
if((status=nc_open(testfile, diskmode|NC_WRITE, &ncid))) ERR;
if((status=nc_inq_dimlen(ncid, 0, &len_in))) ERR;
if (len_in != r + 1) ERR;
index[0] = r;
@ -131,7 +133,7 @@ test_one_with_att(const char *testfile)
if((status=nc_close(ncid))) ERR;
/* Reopen the file and check it. */
if((status=nc_open(testfile, NC_DISKLESS|NC_WRITE, &ncid))) ERR;
if((status=nc_open(testfile, diskmode|NC_WRITE, &ncid))) ERR;
if((status=nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid))) ERR;
if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) ERR;
if((status=nc_get_var_text(ncid, varid, &data_in))) ERR;
@ -146,10 +148,31 @@ test_one_with_att(const char *testfile)
int
main(int argc, char **argv)
{
diskless = (argc > 1);
int i;
printf("\n*** Testing diskless file: create/modify %s: %s\n",
diskless?"in-memory":"in-file",NCFILENAME);
/* Set defaults */
persist = 0;
usenetcdf4 = 0;
mmap = 0;
diskless = 0;
diskmode = 0;
for(i=1;i<argc;i++) {
if(strcmp(argv[i],"diskless")==0) diskless=1;
else if(strcmp(argv[i],"mmap")==0) mmap=1;
/* ignore anything not recognized */
}
if(diskless)
diskmode |= NC_DISKLESS;
if(diskless && mmap)
diskmode |= NC_MMAP;
printf("\n*** Testing diskless file: create/modify %s",
diskless?"in-memory":"in-file");
if(diskless && mmap)
printf("+mmap");
printf(" %s\n",NCFILENAME);
/* case NC_FORMAT_CLASSIC: only test this format */
nc_set_default_format(NC_FORMAT_CLASSIC, NULL);