mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-17 16:50:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

allow NC_DISKLESS with nc_open() to operate as a in-memory cache

Browse Source
This commit is contained in:
Dennis Heimbigner 2012-04-08 22:47:38 +00:00
parent 60e48cf51d
commit 5536bccee1
26 changed files with 841 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
RELEASE_NOTES
View File

@ -11,7 +11,8 @@ VERSION COMMENTS
[NCF-164]
Implemented diskless files for both netcdf classic format
and extended (netcdf-4) format.
and extended (netcdf-4) format. For netcdf classic files,
diskless files also support in-memory caching of disk files.
[NCF-110][NCF-109][NCF-5]
Modified ncio mechanism to support multiple ncio packages;

4
cf
View File

@ -125,8 +125,8 @@ FLAGS="$FLAGS --disable-pnetcdf"
#FLAGS="$FLAGS --enable-dap-long-tests"
#FLAGS="$FLAGS --enable-ffio"
#FLAGS="$FLAGS --enable-benchmarks"
FLAGS="$FLAGS --enable-extra-tests"
FLAGS="$FLAGS --enable-large-file-tests"
#FLAGS="$FLAGS --enable-extra-tests"
#FLAGS="$FLAGS --enable-large-file-tests"
FLAGS="$FLAGS --enable-logging"
FLAGS="$FLAGS --disable-shared"

7
configure.ac
View File

@ -560,8 +560,11 @@ AC_CHECK_FUNCS([strlcat strerror snprintf strchr strrchr strcat strcpy \
strdup strcasecmp strtod strtoll strtoull strstr \
mkstemp getpid \
getrlimit gettimeofday fsync MPI_Comm_f2c])
# memio would like to have this
AC_CHECK_FUNCS([memmove],[],[echo WARNING: memmove not available])
# check for memio support
AC_FUNC_MMAP
AC_CHECK_FUNCS([memmove getpagesize sysconf mremap])
AC_FUNC_ALLOCA
AC_CHECK_DECLS([isnan, isinf, isfinite, signbit],,,[#include <math.h>])
AC_STRUCT_ST_BLKSIZE

8
libdap2/cdf3.c
View File

@ -63,8 +63,12 @@ computevarnodes3(NCDAPCOMMON* nccomm, NClist* allnodes, NClist* varnodes)
NClist* allvarnodes = nclistnew();
for(i=0;i<nclistlength(allnodes);i++) {
CDFnode* node = (CDFnode*)nclistget(allnodes,i);
/* If this node has a bad name, make it invisible */
if(dap_badname(node->ocname)) node->visible=0;
/* If this node has a bad name, repair it */
if(dap_badname(node->ocname)) {
char* newname = dap_repairname(node->ocname);
nullfree(node->ocname);
node->ocname = newname;
}
if(!node->visible) continue;
if(node->nctype == NC_Primitive)
nclistpush(allvarnodes,(ncelem)node);

6
libdap2/cdf4.c
View File

@ -70,7 +70,11 @@ computevarnodes4(NCDAPCOMMON* nccomm, NClist* varnodes)
for(i=0;i<nclistlength(toplevel);i++) {
CDFnode* var = (CDFnode*)nclistget(toplevel,i);
/* If this node has a bad name, make it invisible */
if(dap_badname(var->ncbasename)) var->visible = 0;
if(dap_badname(var->ncbasename)) {
char* newname = dap_repairname(var->ncbasename);
nullfree(var->ncbasename);
var->ncbasename = newname;
}
if(!var->visible) continue;
if(var->nctype == NC_Sequence && singletonsequence(var)) {
var->singleton = 1;

31
libdap2/daputil.c
View File

@ -740,10 +740,12 @@ oc_dumpnode(conn,*rootp);
return ocstat;
}
/* Check a name to see if it contains illegal dap characters */
/* Check a name to see if it contains illegal dap characters
*/
static char* badchars = "./";
int
dap_badname(char* name)
{
@ -754,3 +756,30 @@ dap_badname(char* name)
}
return 0;
}
/* Check a name to see if it contains illegal dap characters
and repair them
*/
char*
dap_repairname(char* name)
{
char* newname;
char *p, *q; int c;
if(name == NULL) return NULL;
/* assume that dap_badname was called on this name and returned 1 */
newname = (char*)malloc(1+(3*strlen(name))); /* max needed */
newname[0] = '\0'; /* so we can use strcat */
for(p=name,q=newname;(c=*p);p++) {
if(strchr(badchars,c) != NULL) {
char newchar[4];
snprintf(newchar,sizeof(newchar),"%%%hhx",c);
strcat(newname,newchar);
q += 3; /*strlen(newchar)*/
} else
*q++ = c;
}
*q = '\0'; /* ensure trailing null */
return newname;
}

1
libdap2/daputil.h
View File

@ -71,5 +71,6 @@ extern int nc__testurl(const char* path, char** basename);
extern OCerror dap_fetch(struct NCDAPCOMMON*,OCconnection,const char*,OCdxd,OCobject*);
extern int dap_badname(char* name);
extern char* dap_repairname(char* name);
#endif /*DAPUTIL_H*/

9
libdap2/env
View File

@ -1,16 +1,17 @@
TOPDIR="/home/dmh/mach/major"
PARMS=""; ARGS=""; CON="" ; CE=""; OCON="" ; VAR=""
PARMS="[log]"
#PARMS="[log]"
#PARMS="${PARMS}[netcdf3]"
#PARMS="${PARMS}[fetch=memory]"
PARMS="${PARMS}[cache]"
#PARMS="${PARMS}[cache]"
#PARMS="${PARMS}[prefetch]"
#PARMS="${PARMS}[nocache]"
#PARMS="${PARMS}[noprefetch]"
#PARMS="${PARMS}[wholevar]"
PARMS="${PARMS}[show=fetch]"
#PARMS="${PARMS}[show=fetch]"
F="file:///home/dmh/mach/minor/ncdap_test/testdata3/data.nc"
F="http://motherlode.ucar.edu:8080/thredds/dodsC/testdods/in.nc"
VAR="var_nm%2edot"
PROG="./ncd"
#PROG="../ncdump/ncdump"

19
libdispatch/dfile.c
View File

@ -134,9 +134,9 @@ NC_check_file_type(const char *path, int use_parallel, void *mpi_info,
else if(magic[0] == 'C' && magic[1] == 'D' && magic[2] == 'F')
{
if(magic[3] == '\001')
*cdf = 1;
*cdf = 1; /* netcdf classic version 1 */
else if(magic[3] == '\002')
*cdf = 2;
*cdf = 2; /* netcdf classic version 2 */
}
return NC_NOERR;
@ -425,7 +425,7 @@ support is enabled, then the path may be an OPeNDAP URL rather than a
file path.
\param mode The mode flag may include NC_WRITE (for read/write
access) and NC_SHARE (see below).
access) and NC_SHARE (see below) and NC_DISKLESS (see below).
\param ncidp Pointer to location where returned netCDF ID is to be
stored.
@ -450,6 +450,19 @@ limited. Since the buffering scheme is optimized for sequential
access, programs that do not access data sequentially may see some
performance improvement by setting the NC_SHARE flag.
This procedure may also be invoked with the NC_DISKLESS flag
set in the mode argument, but ONLY if the file type is NOT NC_NETCDF4,
which means it must be a classic format file.
If NC_DISKLESS is specified, then the whole file is read completely into
memory. In effect this creates an in-memory cache of the file.
If the mode flag also specifies NC_WRITE, then the in-memory cache
will be re-written to the disk file when nc_close() is called.
For some kinds of manipulations, having the in-memory cache can
speed up file processing. But in simple cases, non-cached
processing may actually be faster than using cached processing.
You will need to experiment to determine if the in-memory caching
is worthwhile for your application.
It is not necessary to pass any information about the format of the
file being opened. The file type will be detected automatically by the
netCDF library.

328
libsrc/memio.c
View File

@ -18,6 +18,37 @@
#endif
#include "nc.h"
/* Turn off MMAP until we can figure out why it doesn't work */
#undef HAVE_MMAP
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
/* !HAVE_MREMAP => !HAVE_MMAP */
#ifndef HAVE_MREMAP
# undef HAVE_MMAP
#endif
#ifndef MAP_ANONYMOUS
# ifdef MAP_ANON
# define MAP_ANONYMOUS MAP_ANON
# endif
#endif
/* !MAP_ANONYMOUS => !HAVE_MMAP */
#ifndef MAP_ANONYMOUS
# undef HAVE_MMAP
#endif
#ifdef HAVE_MMAP
/* This is conditionalized by __USE_GNU ; why? */
extern void *mremap(void*,size_t,size_t,int);
# ifndef MREMAP_MAYMOVE
# define MREMAP_MAYMOVE 1
# endif
#endif /*HAVE_MMAP*/
#ifndef HAVE_SSIZE_T
#define ssize_t int
#endif
@ -28,6 +59,10 @@
#define SEEK_END 2
#endif
/* Define the mode flags for create: rw by owner, no access by anyone else */
#define OPENMODE 0600
#define OPENANYMODE 0666
#include "ncio.h"
#include "fbits.h"
#include "rnd.h"
@ -39,6 +74,10 @@
#include "instr.h"
#endif
#ifndef MEMIO_MAXBLOCKSIZE
#define MEMIO_MAXBLOCKSIZE 268435456 /* sanity check, about X_SIZE_T_MAX/8 */
#endif
#undef MIN /* system may define MIN somewhere and complain */
#define MIN(mm,nn) (((mm) < (nn)) ? (mm) : (nn))
@ -53,7 +92,7 @@
#endif
/* Define the amount by which memory is incremented on realloc */
#define DEFAULT_BLOCKSIZE 4096
#define DEFAULT_BLOCKSIZE (0x2000)
#define TACTIC_INCR 1
#define TACTIC_DOUBLE 2
@ -68,6 +107,10 @@ typedef struct NCMEMIO {
off_t alloc;
off_t size;
off_t pos;
int mmap; /* 1=>ok to use mmap if available */
#ifdef HAVE_MMAP
int mapfd;
#endif
} NCMEMIO;
/* Forward */
@ -78,24 +121,35 @@ static int memio_sync(ncio *const nciop);
static int memio_filesize(ncio* nciop, off_t* filesizep);
static int memio_pad_length(ncio* nciop, off_t length);
static int memio_close(ncio* nciop, int);
static void memio_free(void *const pvt);
/* Mnemonic */
#define DOOPEN 1
/* Create a new ncio struct to hold info about the file. */
static ncio*
memio_new(const char* filepath, int ioflags, size_t initialsize, int persist)
static int
memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEMIO** memiop)
{
int status = NC_NOERR;
ncio* nciop = NULL;
NCMEMIO* memio = NULL;
int openfd = -1;
#if defined HAVE_SYSCONF
long pagesize = sysconf(_SC_PAGE_SIZE);
#elif defined HAVE_GETPAGESIZE
long pagesize = getpagesize();
#else
long pagesize = 4096; /* good guess */
#endif
errno = 0;
nciop = (ncio* )calloc(1,sizeof(ncio));
if(nciop == NULL)
goto fail;
if(nciop == NULL) {status = NC_ENOMEM; goto fail;}
nciop->ioflags = ioflags;
*((int*)&nciop->fd) = -1;
*((int*)&nciop->fd) = -1; /* caller will fix */
*((char**)&nciop->path) = strdup(filepath);
if(nciop->path == NULL) goto fail;
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
*((ncio_relfunc**)&nciop->rel) = memio_rel;
*((ncio_getfunc**)&nciop->get) = memio_get;
@ -106,26 +160,43 @@ memio_new(const char* filepath, int ioflags, size_t initialsize, int persist)
*((ncio_closefunc**)&nciop->close) = memio_close;
memio = (NCMEMIO*)calloc(1,sizeof(NCMEMIO));
if(memio == NULL) goto fail;
if(memio == NULL) {status = NC_ENOMEM; goto fail;}
*((void* *)&nciop->pvt) = memio;
/* See if ok to use mmap */
#ifdef HAVE_MMAP
memio->mmap = (sizeof(void*) < 8 && fIsSet(ioflags,NC_64BIT_OFFSET)?0:1);
#else
memio->mmap = 0;
#endif
memio->memory = NULL;
memio->alloc = initialsize;
if(memio->alloc < DEFAULT_BLOCKSIZE)
memio->alloc = DEFAULT_BLOCKSIZE;
if(memio->alloc < pagesize)
memio->alloc = pagesize;
#ifdef HAVE_MMAP
if((memio->alloc % pagesize) != 0)
memio->alloc += (pagesize - (memio->alloc % pagesize));
#endif
memio->size = 0;
memio->pos = 0;
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) goto fail;
memio->persist = persist;
memio->persist = fIsSet(ioflags,NC_WRITE);
#ifdef HAVE_MMAP
memio->mapfd = -1;
#endif
if(nciopp) *nciopp = nciop;
if(memiop) *memiop = memio;
done:
if(openfd >= 0) close(openfd);
return status;
return nciop;
fail:
if(nciop != NULL) {
if(nciop->path != NULL) free((char*)nciop->path);
if(memio != NULL) {
}
}
return NULL;
goto done;
}
/* Create a file, and the ncio struct to go with it. This function is
@ -151,20 +222,78 @@ memio_create(const char* path, int ioflags,
ncio* nciop;
int fd;
int status;
NCMEMIO* memio = NULL;
int persist = (ioflags & NC_WRITE?1:0);
int oflags;
if(path == NULL ||* path == 0)
return NC_EINVAL;
fSet(ioflags, NC_WRITE);
/* For diskless open has, the file must be classic version 1 or 2.*/
if(fIsSet(ioflags,NC_NETCDF4))
return NC_EDISKLESS; /* violates constraints */
nciop = memio_new(path, ioflags, initialsz, persist);
if(nciop == NULL)
return NC_ENOMEM;
status = memio_new(path, ioflags, initialsz, &nciop, &memio);
if(status != NC_NOERR)
return status;
if(!persist) { /* mmap => use anonymous mmap */
if(memio->mmap) { /* use mmap if available */
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = -1;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_POPULATE,
memio->mapfd,0);
{/* test writing of the mmap'd memory */
memio->memory[0] = 0;}
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap; just malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
} else { /*persist */
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= (O_CREAT|O_TRUNC);
if(fIsSet(ioflags,NC_NOCLOBBER))
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
if(fd < 0) {status = errno; goto unwind_open;}
if(memio->mmap) {
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = fd;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_SHARED|MAP_FIXED,
memio->mapfd,0);
{/* test reading of the mmap'd memory */
int tst = memio->memory[0];}
#endif /*HAVE_MMAP*/
} else {/*memio->mmap*/
(void)close(fd); /* will reopen at nc_close */
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
} /*!persist*/
fd = nc__pseudofd();
*((int* )&nciop->fd) = fd;
fSet(nciop->ioflags, NC_WRITE);
if(igetsz != 0)
{
status = nciop->get(nciop,
@ -207,22 +336,71 @@ memio_open(const char* path,
off_t igeto, size_t igetsz, size_t* sizehintp,
ncio* *nciopp, void** const mempp)
{
/* Currrently, diskless open not supported */
#if 1
return NC_EDISKLESS;
#else
ncio* nciop;
int fd;
int status;
int persist = (fIsSet(ioflags,NC_WRITE)?1:0);
int oflags;
NCMEMIO* memio = NULL;
size_t sizehint;
off_t filesize;
ssize_t count;
if(path == NULL ||* path == 0)
return EINVAL;
nciop = memio_new(path, ioflags, 0);
if(nciop == NULL)
return ENOMEM;
assert(sizehintp != NULL);
sizehint = *sizehintp;
fd = --pseudofd;
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
if(fd < 0) {status = errno; goto unwind_open;}
/* get current filesize = max(|file|,initialize)*/
filesize = lseek(fd,0,SEEK_END);
if(filesize < 0) {status = errno; goto unwind_open;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
if(filesize < (off_t)sizehint)
filesize = (off_t)sizehint;
status = memio_new(path, ioflags, filesize, &nciop, &memio);
if(status != NC_NOERR)
return status;
if(memio->mmap) {
#ifdef HAVE_MMAP
memio->mapfd = fd;
memio->memory = (char*)mmap(NULL,memio->alloc,
persist?(PROT_READ|PROT_WRITE):(PROT_READ),
MAP_SHARED,
memio->mapfd,0);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
/* Read the file into the memio memory */
count = read(fd, memio->memory, (size_t)filesize);
if(count < filesize) {status = NC_ENOTNC; goto unwind_open;}
(void)close(fd); /* until memio_close() */
}
memio->size = memio->alloc;
/* Use half the filesize as the blocksize */
sizehint = filesize/2;
fd = nc__pseudofd();
*((int* )&nciop->fd) = fd;
if(igetsz != 0)
@ -235,16 +413,13 @@ memio_open(const char* path,
goto unwind_open;
}
/* Pick a default sizehint */
if(sizehintp) *sizehintp = DEFAULT_BLOCKSIZE;
*sizehintp = sizehint;
*nciopp = nciop;
return NC_NOERR;
unwind_open:
memio_close(nciop,1);
memio_close(nciop,0);
return status;
#endif
}
@ -281,24 +456,33 @@ memio_pad_length(ncio* nciop, off_t length)
/* Realloc the allocated memory */
if(memio->locked > 0)
return NC_EDISKLESS;
if(length > memio->alloc) {
off_t actual;
off_t newsize;
char* newmem;
switch(TACTIC) {
case TACTIC_DOUBLE:
actual = (memio->alloc * 2);
newsize = (memio->alloc * 2);
break;
case TACTIC_INCR:
default:
actual = length + (length % DEFAULT_BLOCKSIZE);
newsize = length + (length % DEFAULT_BLOCKSIZE);
break;
}
newmem = (char*)realloc(memio->memory,actual);
if(newmem == NULL) return NC_ENOMEM;
if(memio->mmap) {
#ifdef HAVE_MMAP
newmem = (char*)mremap(memio->memory,memio->alloc,newsize,MREMAP_MAYMOVE);
if(newmem == NULL) return NC_ENOMEM;
#endif /*HAVE_MMAP*/
} else {
newmem = (char*)realloc(memio->memory,newsize);
if(newmem == NULL) return NC_ENOMEM;
/* zero out the extra memory */
memset((void*)(newmem+memio->alloc),0,(newsize - memio->alloc));
}
memio->memory = newmem;
/* zero out the extra memory */
memset((void*)(newmem+memio->alloc),0,(actual - memio->alloc));
memio->alloc = actual;
memio->alloc = newsize;
}
memio->size = length;
return NC_NOERR;
@ -318,27 +502,40 @@ memio_close(ncio* nciop, int doUnlink)
int status = NC_NOERR;
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
/* See if the user wants the contents persisted to a file */
memio = (NCMEMIO*)nciop->pvt;
if(memio != NULL) {
assert(memio != NULL);
if(memio->mmap) {
#ifdef HAVE_MMAP
status = munmap(memio->memory,memio->alloc);
if(status) goto done;
/* Do we need to close the file? */
if(memio->mapfd >= 0)
(void)close(memio->mapfd);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
/* See if the user wants the contents persisted to a file */
if(memio->persist) {
/* Try to open the file for writing */
int fd = open(nciop->path, O_WRONLY|O_CREAT|O_TRUNC, 0666);
int fd = open(nciop->path, O_WRONLY|O_CREAT|O_TRUNC, OPENMODE);
if(fd >= 0) {
long count = write(fd, memio->memory, memio->size);
if(count < 0)
long count = write(fd, memio->memory, memio->size);
if(count < 0)
status = errno;
else if(count < memio->size)
status = NC_EDISKLESS;
(void)close(fd);
else if(count < memio->size)
{status = NC_EDISKLESS; goto done;}
(void)close(fd);
} else
status = errno;
status = errno;
}
/* Free up things */
memio_free(memio);
free(memio);
}
/* do cleanup of ncio structure */
/* Free up things */
if(memio->memory != NULL) free(memio->memory);
}
done:
/* do cleanup */
if(memio != NULL) free(memio);
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
return status;
@ -348,7 +545,7 @@ static int
guarantee(ncio* nciop, off_t endpoint)
{
NCMEMIO* memio = (NCMEMIO*)nciop->pvt;
if(endpoint >= memio->alloc) {
if(endpoint > memio->alloc) {
/* extend the allocated memory and size */
int status = memio_pad_length(nciop,endpoint);
if(status != NC_NOERR) return status;
@ -449,16 +646,3 @@ memio_sync(ncio* const nciop)
{
return NC_NOERR; /* do nothing */
}
/*
* free up anything hanging off pvt;
*/
static void
memio_free(void *const pvt)
{
if(pvt != NULL) {
NCMEMIO* memio;
memio = (NCMEMIO*)pvt;
if(memio->memory != NULL) free(memio->memory);
}
}

8
libsrc/ncio.c
View File

@ -45,8 +45,12 @@ ncio_open(const char *path, int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
ncio** iopp, void** const mempp)
{
if(fIsSet(ioflags,NC_DISKLESS))
return NC_EDISKLESS; /* create only */
/* Diskless open has the following constraints:
1. file must be classic version 1 or 2
*/
if(fIsSet(ioflags,NC_DISKLESS)) {
return memio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
}
#ifdef USE_FFIO
return ffio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
#else

1
libsrc4/nc4file.c
View File

@ -277,7 +277,6 @@ nc4_create_file(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
}
#else /* only set cache for non-parallel... */
if(cmode & NC_DISKLESS) {
int persist = (cmode & NC_WRITE)?1:0;
if (H5Pset_fapl_core(fapl_id, 4096, persist))
BAIL(NC_EDISKLESS);
}

7
nc_test/Makefile.am
View File

@ -9,7 +9,10 @@ include $(top_srcdir)/lib_flags.am
# These files are created by the tests.
CLEANFILES = nc_test_classic.nc nc_test_64bit.nc nc_test_netcdf4.nc \
tst_*.nc t_nc.nc large_files.nc quick_large_files.nc tst_diskless.nc tst_diskless2.nc
tst_*.nc t_nc.nc large_files.nc quick_large_files.nc \
tst_diskless.nc tst_diskless2.nc \
tst_diskless3.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \
tst_diskless4.cdl tst_diskless4.nc
# These are the tests which are always run.
TESTPROGRAMS = t_nc tst_small nc_test tst_misc tst_norm tst_names \
@ -41,7 +44,7 @@ endif
# Set up the tests.
check_PROGRAMS = $(TESTPROGRAMS) tst_diskless
check_PROGRAMS = $(TESTPROGRAMS) tst_diskless tst_diskless3 tst_diskless4
if USE_NETCDF4
check_PROGRAMS += tst_diskless2
endif

81
nc_test/run_diskless.sh
View File

@ -5,6 +5,11 @@ set -e
#Constants
FILE1=tst_diskless.nc
FILE2=tst_diskless2.nc
FILE3=tst_diskless3.nc
FILE4=tst_diskless4.nc
#SIZE=1073741824
SIZE=536870912
echo ""
echo "Testing in-memory (diskless) files with and without persistence"
@ -13,26 +18,20 @@ HASNC4=`../nc-config --has-nc4`
echo ""
echo "Test diskless netCDF classic file without persistence"
cmd="./tst_diskless";
echo "cmd=$cmd"
$cmd
./tst_diskless
echo "*** PASS: diskless netCDF classic file without persistence"
if test "x$HASNC4" = "xyes" ; then
echo ""
echo "Test diskless netCDF enhanced file without persistence"
cmd="./tst_diskless netcdf4";
echo "cmd=$cmd"
$cmd
./tst_diskless netcdf4
echo "*** PASS: diskless netCDF enhanced file without persistence"
fi #HASNC4
echo ""
echo "Test diskless netCDF classic file with persistence"
cmd="./tst_diskless persist";
echo "cmd=$cmd"
rm -f $FILE1
$cmd
./tst_diskless persist
if test -f $FILE1 ; then
echo "$FILE1 created"
../ncdump/ncdump $FILE1
@ -45,10 +44,8 @@ fi
if test "x$HASNC4" = "xyes" ; then
echo ""
echo "Test diskless netCDF enhanced file with persistence"
cmd="./tst_diskless netcdf4 persist";
echo "cmd=$cmd"
rm -f $FILE1
$cmd
./tst_diskless netcdf4 persist
if test -f $FILE1 ; then
echo "$FILE1 created"
../ncdump/ncdump $FILE1
@ -66,10 +63,8 @@ if test "x$HASNC4" = "xyes" ; then
ok=""
echo ""
echo "Test extended enhanced diskless netCDF with persistence"
cmd="./tst_diskless2"
echo "cmd=$cmd"
rm -f $FILE2 tst_diskless2.cdl
$cmd
./tst_diskless2
if test -f $FILE2 ; then
echo "$FILE2 created"
# Do a cyle test
@ -94,3 +89,59 @@ else
fi
fi #HASNC4
echo ""
echo "Testing nc_open in-memory (diskless) files"
# clear old files
rm -f tst_diskless3_file.cdl tst_diskless3_memory.cdl
echo ""
echo "Create and modify file without using diskless"
rm -f $FILE3
./tst_diskless3
../ncdump/ncdump $FILE3 >tst_diskless3_file.cdl
echo ""
echo "Create and modify file using diskless"
rm -f $FILE3
./tst_diskless3 diskless
../ncdump/ncdump $FILE3 >tst_diskless3_memory.cdl
# compare
diff tst_diskless3_file.cdl tst_diskless3_memory.cdl
# cleanup
rm -f $FILE3 tst_diskless3_file.cdl tst_diskless3_memory.cdl
# Create the reference ncdump output for tst_diskless4
rm -fr tst_diskless4.cdl
echo "netcdf tst_diskless4 {" >>tst_diskless4.cdl
echo "dimensions:" >>tst_diskless4.cdl
echo " dim = $SIZE ;" >>tst_diskless4.cdl
echo "variables:" >>tst_diskless4.cdl
echo " byte var(dim) ;" >>tst_diskless4.cdl
echo "}" >>tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 create
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 creatediskless
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
time ./tst_diskless4 open
echo ""
time ./tst_diskless4 opendiskless
# cleanup
rm -f $FILE4 tst_diskless4.cdl
exit

2
nc_test/tst_diskless.c
View File

@ -80,7 +80,6 @@ printf("*** testing diskless file with scalar vars...");
if (nc_def_var(ncid, CAPACITOR, NC_FLOAT, 0, NULL, &varid1)) ERR;
if (nc_def_var(ncid, NUM555, NC_SHORT, 0, NULL, &varid2)) ERR;
if (nc_enddef(ncid)) ERR;
/* Write some data to this file. */
if (nc_put_vara_int(ncid, varid0, NULL, NULL, &int_data)) ERR;
@ -244,6 +243,7 @@ printf("*** testing diskless file with scalar vars...");
natts_in != 0) ERR;
if (nc_inq_var(ncid, varid1, name_in, &type_in, &ndims_in, NULL, &natts_in)) ERR;
if (strcmp(name_in, STAR_TREK) || type_in != NC_FLOAT || ndims_in != 0 ||
natts_in != 0) ERR;
if (nc_inq_var(ncid, varid2, name_in, &type_in, &ndims_in, NULL, &natts_in)) ERR;
if (strcmp(name_in, STAR_WARS) || type_in != NC_SHORT || natts_in != 0) ERR;

153
nc_test/tst_diskless3.c Normal file
View File

@ -0,0 +1,153 @@
/* This is part of the netCDF package. Copyright 2005 University
Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
conditions of use. See www.unidata.ucar.edu for more info.
Test small files.
$Id: tst_small.c,v 1.15 2008/10/20 01:48:08 ed Exp $
*/
#include <nc_tests.h>
#include <netcdf.h>
/* Derived from tst_small.c */
/* Test everything for classic using in-memory files */
#define NCFILENAME "tst_diskless3.nc"
#define ATT_NAME "Atom"
#define MAX_LEN 7
#define VAR_NAME2 "var2"
#define NUM_VARS 2
#define ONE_DIM 1
#define MAX_RECS 10
#define DIM1_NAME "Time"
#define DIM2_NAME "DataStrLen"
#define VAR_NAME "Times"
#define STR_LEN 19
#define NUM_VALS 2
#define NDIMS 2
#define TITLE " OUTPUT FROM WRF V2.0.3.1 MODEL"
#define ATT_NAME2 "TITLE"
/* Global */
int diskless = 0;
/* Test a diskless file with two record vars, which grow, and has
* attributes added. */
static int
test_two_growing_with_att(const char *testfile)
{
int ncid, dimid, varid[NUM_VARS];
char data[MAX_RECS], data_in;
char att_name[NC_MAX_NAME + 1];
size_t start[ONE_DIM], count[ONE_DIM], index[ONE_DIM], len_in;
int v, r;
/* Create a file with one ulimited dimensions, and one var. */
if (nc_create(testfile, NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid[0])) ERR;
if (nc_def_var(ncid, VAR_NAME2, NC_CHAR, 1, &dimid, &varid[1])) ERR;
if (nc_close(ncid)) ERR;
/* Create some phoney data. */
for (data[0] = 'a', r = 1; r < MAX_RECS; r++)
data[r] = data[r - 1] + 1;
/* Normally one would not close and reopen the file for each
* record, nor add an attribute each time I add a record, but I am
* giving the library a little work-out here... */
for (r = 0; r < MAX_RECS; r++)
{
/* Write one record of var data, a single character. */
if (nc_open(testfile, NC_WRITE, &ncid)) ERR;
count[0] = 1;
start[0] = r;
sprintf(att_name, "a_%d", data[r]);
for (v = 0; v < NUM_VARS; v++)
{
if (nc_put_vara_text(ncid, varid[v], start, count, &data[r])) ERR;
if (nc_redef(ncid)) ERR;
if (nc_put_att_text(ncid, varid[v], att_name, 1, &data[r])) ERR;
if (nc_enddef(ncid)) ERR;
}
if (nc_close(ncid)) ERR;
/* Reopen the file and check it. */
if (nc_open(testfile, NC_DISKLESS|NC_WRITE, &ncid)) ERR;
if (nc_inq_dimlen(ncid, 0, &len_in)) ERR;
if (len_in != r + 1) ERR;
index[0] = r;
for (v = 0; v < NUM_VARS; v++)
{
if (nc_get_var1_text(ncid, varid[v], index, &data_in)) ERR;
if (data_in != data[r]) ERR;
}
if (nc_close(ncid)) ERR;
} /* Next record. */
return 0;
}
/* Test a diskless file with one var and one att. */
static int
test_one_with_att(const char *testfile)
{
int ncid, dimid, varid;
char data = 'h', data_in;
int ndims, nvars, natts, unlimdimid;
size_t start[NDIMS], count[NDIMS];
/* Create a file with one ulimited dimensions, and one var. */
if (nc_create(testfile, NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM1_NAME, NC_UNLIMITED, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME, NC_CHAR, 1, &dimid, &varid)) ERR;
if (nc_put_att_text(ncid, NC_GLOBAL, ATT_NAME, 1, &data)) ERR;
if (nc_enddef(ncid)) ERR;
/* Write one record of var data, a single character. */
count[0] = 1;
start[0] = 0;
if (nc_put_vara_text(ncid, varid, start, count, &data)) ERR;
/* We're done! */
if (nc_close(ncid)) ERR;
/* Reopen the file and check it. */
if (nc_open(testfile, NC_DISKLESS|NC_WRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
if (ndims != 1 && nvars != 1 && natts != 0 && unlimdimid != 0) ERR;
if (nc_get_var_text(ncid, varid, &data_in)) ERR;
if (data_in != data) ERR;
if (nc_get_att_text(ncid, NC_GLOBAL, ATT_NAME, &data_in));
if (data_in != data) ERR;
if (nc_close(ncid)) ERR;
return 0;
}
int
main(int argc, char **argv)
{
int i;
diskless = (argc > 1);
printf("\n*** Testing diskless file: create/modify %s: %s\n",
diskless?"in-memory":"in-file",NCFILENAME);
/* case NC_FORMAT_CLASSIC: only test this format */
nc_set_default_format(NC_FORMAT_CLASSIC, NULL);
printf("*** testing diskless file with two growing record "
"variables, with attributes added...");
test_two_growing_with_att(NCFILENAME);
SUMMARIZE_ERR;
FINAL_RESULTS;
}

177
nc_test/tst_diskless4.c Normal file
View File

@ -0,0 +1,177 @@
/*
Copyright 2008, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
This program tests the large file bug in netCDF 3.6.2,
creating byte and short variables larger than 4 GiB.
$Id: tst_big_var.c,v 1.9 2010/05/15 00:50:10 russ Exp $
*/
#include <nc_tests.h>
#include <netcdf.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#undef XFAIL
/*
* Depending on argv, do one of the following:
1. create a file with var of size 0x40000000 (~ 1 gig)
2. create a file with var of size 0x40000000 (~ 1 gig) but using NC_DISKLESS
3. open a file with var of size 0x40000000 (~ 1 gig) and read it 4096 bytes
at a time
4. open a file with var of size 0x40000000 (~ 1 gig) and read it 4096 bytes
at a time using NC_DISKLESS
*/
#define FILE_NAME "tst_diskless4.nc"
#define CHUNKSIZE 4096
#define GIG (1024*1024*1024)
#define DATASIZE (CHUNKSIZE/sizeof(int))
typedef enum Tag {Create,CreateDiskless,Open,OpenDiskless} Tag;
static size_t varsize;
static size_t pieces;
static float gigsize;
#define REPORT err_report(status,__FILE__,__LINE__)
static void
err_report(int status, char* file, int line)
{
printf("***FAIL: %s: line=%d status=%d %s\n",file,line,status,nc_strerror(status));
#ifdef XFAIL
exit(0);
#else
exit(1);
#endif
}
int
main(int argc, char **argv)
{
int status = NC_NOERR;
int i,j;
unsigned int data[DATASIZE];
size_t start[1];
size_t count[1];
Tag tag = Create;
int cmode = 0;
int ncid,varid;
int dimids[1];
/* First, determine a reasonable file size that malloc will actually allocate
on this machine
*/
for(varsize=GIG/2;;varsize/=2) {
void* memory = malloc(varsize);
if(memory != NULL) {free(memory); break;}
free(memory);
}
/* Compute other values relative to varsize */
pieces = varsize/CHUNKSIZE;
gigsize = (varsize/GIG);
if(argc > 1) {
if(strcmp(argv[1],"create")==0) tag = Create;
else if(strcmp(argv[1],"creatediskless")==0) tag = CreateDiskless;
else if(strcmp(argv[1],"open")==0) tag = Open;
else if(strcmp(argv[1],"opendiskless")==0) tag = OpenDiskless;
else {
fprintf(stderr,"illegal tag: %s",argv[1]);
exit(1);
}
}
switch (tag) {
case Create: printf("\n*** Create file\n"); break;
case CreateDiskless: printf("\n*** Create file diskless\n"); break;
case Open: printf("\n*** Open file\n"); break;
case OpenDiskless: printf("\n*** Open file diskless\n"); break;
}
switch (tag) {
case Create: cmode = NC_CLOBBER; break;
case CreateDiskless: cmode = NC_CLOBBER|NC_DISKLESS|NC_WRITE; break;
case Open: cmode = 0; break;
case OpenDiskless: cmode = NC_DISKLESS; break;
}
switch (tag) {
case Create:
case CreateDiskless:
if((status=nc_create(FILE_NAME, cmode, &ncid)))
REPORT;
if((status=nc_set_fill(ncid, NC_NOFILL, NULL)))
REPORT;
break;
case Open:
case OpenDiskless:
if((status=nc_open(FILE_NAME, cmode, &ncid)))
REPORT;
break;
}
switch (tag) {
case Create:
case CreateDiskless:
if((status=nc_def_dim(ncid, "dim", varsize, &dimids[0])))
REPORT;
if((status=nc_def_var(ncid, "var", NC_BYTE, 1, dimids, &varid)))
REPORT;
if((status=nc_enddef(ncid)))
REPORT;
break;
case Open:
case OpenDiskless:
if((status=nc_inq_dimid(ncid, "dim", &dimids[0])))
REPORT;
if((status=nc_inq_varid(ncid, "var", &varid)))
REPORT;
break;
}
switch (tag) {
case Create:
case CreateDiskless:
/* Fill and put as integers */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
for(j=0;j<DATASIZE;j++) data[j] = (i*CHUNKSIZE)+j;
if((status=nc_put_vara(ncid,varid,start,count,(void*)data)))
REPORT;
}
break;
case Open:
case OpenDiskless:
/* Read the var contents and validate */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
if((status=nc_get_vara(ncid,varid,start,count,(void*)data)))
REPORT;
for(j=0;j<DATASIZE;j++) {
unsigned int expected = (i*CHUNKSIZE)+j;
if(data[j] != expected) {
printf("mismatch: i=%u j=%u data=%u; should be %u\n",
i,j,data[j],expected);
err++;
}
}
}
break;
}
if((status=nc_close(ncid)))
REPORT;
SUMMARIZE_ERR;
exit(0);
return 0;
}

20
ncdap_test/expected3/1990-S1700101.HDF.WVC_Lat.dmp
View File

@ -3,15 +3,15 @@ dimensions:
WVC = 24 ;
row = 558 ;
variables:
short NSCAT\ Rev\ 17.WVC_Lat(row, WVC) ;
NSCAT\ Rev\ 17.WVC_Lat:long_name = "latitude" ;
NSCAT\ Rev\ 17.WVC_Lat:units = "deg" ;
NSCAT\ Rev\ 17.WVC_Lat:scale_factor = 0.01 ;
NSCAT\ Rev\ 17.WVC_Lat:scale_factor_err = 0. ;
NSCAT\ Rev\ 17.WVC_Lat:add_offset = 0. ;
NSCAT\ Rev\ 17.WVC_Lat:add_offset_err = 0. ;
NSCAT\ Rev\ 17.WVC_Lat:calibrated_nt = 22 ;
NSCAT\ Rev\ 17.WVC_Lat:valid_range = -6281s, 8051s ;
short NSCAT\%20Rev\%2017.WVC_Lat(row, WVC) ;
NSCAT\%20Rev\%2017.WVC_Lat:long_name = "latitude" ;
NSCAT\%20Rev\%2017.WVC_Lat:units = "deg" ;
NSCAT\%20Rev\%2017.WVC_Lat:scale_factor = 0.01 ;
NSCAT\%20Rev\%2017.WVC_Lat:scale_factor_err = 0. ;
NSCAT\%20Rev\%2017.WVC_Lat:add_offset = 0. ;
NSCAT\%20Rev\%2017.WVC_Lat:add_offset_err = 0. ;
NSCAT\%20Rev\%2017.WVC_Lat:calibrated_nt = 22 ;
NSCAT\%20Rev\%2017.WVC_Lat:valid_range = -6281s, 8051s ;
// global attributes:
:Producer_Agency = "NASA" ;
@ -39,7 +39,7 @@ variables:
:Data_Format_Type = "HDF" ;
data:
NSCAT\ Rev\ 17.WVC_Lat =
NSCAT\%20Rev\%2017.WVC_Lat =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -5928,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -5889,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -5851,

2
ncdap_test/expected3/synth1.dmp
View File

@ -3,7 +3,7 @@ dimensions:
S1.v1_0 = 3 ;
variables:
int S1.v1(S1.v1_0) ;
S1.v1:a\ 1 = 32 ;
S1.v1:a\%201 = 32 ;
data:
S1.v1 = 132, 232, 332 ;

37
ncdap_test/expected3/test.PointFile.dmp
View File

@ -4,46 +4,47 @@ dimensions:
Sensor = 5 ;
maxStrlen64 = 64 ;
variables:
byte Simple\ Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
double Simple\ Point.Data\ Vgroup.Sensor.Time.Time__0(Sensor) ;
double Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__0(Sensor) ;
double Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__1(Sensor) ;
double Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__2(Sensor) ;
double Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__3(Sensor) ;
char Simple\ Point.Data\ Vgroup.Sensor.Species.Species__0(Sensor, maxStrlen64) ;
byte FixedBuoy\ Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
byte FloatBuoy\ Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
byte Simple\%20Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
double Simple\%20Point.Data\%20Vgroup.Sensor.Time.Time__0(Sensor) ;
double Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__0(Sensor) ;
double Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__1(Sensor) ;
double Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__2(Sensor) ;
double Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__3(Sensor) ;
char Simple\%20Point.Data\%20Vgroup.Sensor.Species.Species__0(Sensor, maxStrlen64) ;
byte FixedBuoy\%20Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
byte FloatBuoy\%20Point.LevelWritten.LevelWritten.LevelWritten__0(LevelWritten) ;
data:
Simple\ Point.LevelWritten.LevelWritten.LevelWritten__0 = 0, 1, 2, 3, 4 ;
Simple\%20Point.LevelWritten.LevelWritten.LevelWritten__0 = 0, 1, 2, 3, 4 ;
Simple\ Point.Data\ Vgroup.Sensor.Time.Time__0 = 1000, 998.750260394966,
Simple\%20Point.Data\%20Vgroup.Sensor.Time.Time__0 = 1000, 998.750260394966,
995.004165278026, 988.771077936042, 980.066577841242 ;
Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__0 =
Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__0 =
999.950000416665, 998.200539935204, 993.956097956697, 987.227283375627,
978.030914724148 ;
Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__1 =
Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__1 =
999.800006666578, 997.55100025328, 992.808635853866, 985.584766909561,
975.897449330605 ;
Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__2 =
Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__2 =
999.550033748988, 996.801706302619, 991.561893714788, 983.843692788121,
973.666395005375 ;
Simple\ Point.Data\ Vgroup.Sensor.Concentration.Concentration__3 =
Simple\%20Point.Data\%20Vgroup.Sensor.Concentration.Concentration__3 =
999.200106660978, 995.952733011994, 990.215996212637, 982.00423511727,
971.33797485203 ;
Simple\ Point.Data\ Vgroup.Sensor.Species.Species__0 =
Simple\%20Point.Data\%20Vgroup.Sensor.Species.Species__0 =
"This is a data test string (pass 0).",
"This is a data test string (pass 1).",
"This is a data test string (pass 2).",
"This is a data test string (pass 3).",
"This is a data test string (pass 4)." ;
FixedBuoy\ Point.LevelWritten.LevelWritten.LevelWritten__0 = 5, 6, 7, 8, 9 ;
FixedBuoy\%20Point.LevelWritten.LevelWritten.LevelWritten__0 = 5, 6, 7, 8, 9 ;
FloatBuoy\ Point.LevelWritten.LevelWritten.LevelWritten__0 = 10, 11, 12, 13, 14 ;
FloatBuoy\%20Point.LevelWritten.LevelWritten.LevelWritten__0 = 10, 11, 12,
13, 14 ;
}

18
ncdap_test/expected3/test.dfr1.dmp
View File

@ -1,16 +1,16 @@
netcdf test {
dimensions:
Raster\ Image\ \#0__X = 5 ;
Raster\ Image\ \#0__Y = 6 ;
Raster\ Image\ \#1__X = 5 ;
Raster\ Image\ \#1__Y = 6 ;
Raster\ Image\ \#1__comps = 3 ;
Raster\%20Image\%20\%230__X = 5 ;
Raster\%20Image\%20\%230__Y = 6 ;
Raster\%20Image\%20\%231__X = 5 ;
Raster\%20Image\%20\%231__Y = 6 ;
Raster\%20Image\%20\%231__comps = 3 ;
variables:
byte Raster\ Image\ \#0(Raster\ Image\ \#0__Y, Raster\ Image\ \#0__X) ;
byte Raster\ Image\ \#1(Raster\ Image\ \#1__comps, Raster\ Image\ \#1__Y, Raster\ Image\ \#1__X) ;
byte Raster\%20Image\%20\%230(Raster\%20Image\%20\%230__Y, Raster\%20Image\%20\%230__X) ;
byte Raster\%20Image\%20\%231(Raster\%20Image\%20\%231__comps, Raster\%20Image\%20\%231__Y, Raster\%20Image\%20\%231__X) ;
data:
Raster\ Image\ \#0 =
Raster\%20Image\%20\%230 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,
@ -18,7 +18,7 @@ data:
20, 21, 22, 23, 24,
25, 26, 27, 28, 29 ;
Raster\ Image\ \#1 =
Raster\%20Image\%20\%231 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,

24
ncdap_test/expected3/test.dfr2.dmp
View File

@ -1,18 +1,18 @@
netcdf test {
dimensions:
Raster\ Image\ \#0__X = 5 ;
Raster\ Image\ \#0__Y = 6 ;
Raster\ Image\ \#1__X = 5 ;
Raster\ Image\ \#1__Y = 6 ;
Raster\ Image\ \#2__X = 5 ;
Raster\ Image\ \#2__Y = 6 ;
Raster\%20Image\%20\%230__X = 5 ;
Raster\%20Image\%20\%230__Y = 6 ;
Raster\%20Image\%20\%231__X = 5 ;
Raster\%20Image\%20\%231__Y = 6 ;
Raster\%20Image\%20\%232__X = 5 ;
Raster\%20Image\%20\%232__Y = 6 ;
variables:
byte Raster\ Image\ \#0(Raster\ Image\ \#0__Y, Raster\ Image\ \#0__X) ;
byte Raster\ Image\ \#1(Raster\ Image\ \#1__Y, Raster\ Image\ \#1__X) ;
byte Raster\ Image\ \#2(Raster\ Image\ \#2__Y, Raster\ Image\ \#2__X) ;
byte Raster\%20Image\%20\%230(Raster\%20Image\%20\%230__Y, Raster\%20Image\%20\%230__X) ;
byte Raster\%20Image\%20\%231(Raster\%20Image\%20\%231__Y, Raster\%20Image\%20\%231__X) ;
byte Raster\%20Image\%20\%232(Raster\%20Image\%20\%232__Y, Raster\%20Image\%20\%232__X) ;
data:
Raster\ Image\ \#0 =
Raster\%20Image\%20\%230 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,
@ -20,7 +20,7 @@ data:
20, 21, 22, 23, 24,
25, 26, 27, 28, 29 ;
Raster\ Image\ \#1 =
Raster\%20Image\%20\%231 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,
@ -28,7 +28,7 @@ data:
20, 21, 22, 23, 24,
25, 26, 27, 28, 29 ;
Raster\ Image\ \#2 =
Raster\%20Image\%20\%232 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,

10
ncdap_test/expected3/test.dfr3.dmp
View File

@ -1,13 +1,13 @@
netcdf test {
dimensions:
Raster\ Image\ \#0__X = 5 ;
Raster\ Image\ \#0__Y = 6 ;
Raster\ Image\ \#0__comps = 3 ;
Raster\%20Image\%20\%230__X = 5 ;
Raster\%20Image\%20\%230__Y = 6 ;
Raster\%20Image\%20\%230__comps = 3 ;
variables:
byte Raster\ Image\ \#0(Raster\ Image\ \#0__comps, Raster\ Image\ \#0__Y, Raster\ Image\ \#0__X) ;
byte Raster\%20Image\%20\%230(Raster\%20Image\%20\%230__comps, Raster\%20Image\%20\%230__Y, Raster\%20Image\%20\%230__X) ;
data:
Raster\ Image\ \#0 =
Raster\%20Image\%20\%230 =
0, 1, 2, 3, 4,
5, 6, 7, 8, 9,
10, 11, 12, 13, 14,

34
oc/daplex.c
View File

@ -16,16 +16,25 @@ static int tohex(int c);
#endif
/****************************************************/
/* Set of all ascii printable characters */
#ifdef NOTUSED
static char ascii[] = " !\"#$%&'()*+,-./:;<=>?@[]\\^_`|{}~";
/* Define the set of legal nonalphanum characters as specified in the DAP2 spec. */
static char* daplegal ="_!~*'-\"";
#endif
static char* ddsworddelims =
"{}[]:;=,";
/* Define 1 and > 1st legal characters */
static char* ddswordchars1 =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%.\\*";
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%\\.*";
static char* ddswordcharsn =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%.\\*#";
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%\\.*#";
static char* daswordcharsn =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%.\\*:#";
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%\\.*#:";
static char* cewordchars1 =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-+_/%\\";
static char* cewordcharsn =
@ -315,16 +324,27 @@ daplexcleanup(DAPlexstate** lexstatep)
*lexstatep = NULL;
}
/* Dap identifiers will come to use encoded,
so we must decode them; It turns out that we
can use ocuridecode because dap specifies
%xx encoding.
/* Dap identifiers will come to us with some
characters escaped using the URL notation of
%HH. The assumption here is that any character
that is encoded is left encoded, except as follows:
1. if the encoded character is in fact a legal DAP2 character
(alphanum+"_!~*'-\"") then it is decoded, otherwise not.
*/
#ifndef DECODE_IDENTIFIERS
static char* decodelist =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_!~*'-\"";
#endif
char*
dapdecode(DAPlexstate* lexstate, char* name)
{
char* decoded;
#ifdef DECODE_IDENTIFIERS
decoded = ocuridecode(name);
#else
decoded = ocuridecodeonly(name,decodelist);
#endif
oclistpush(lexstate->reclaim,(ocelem)decoded);
return decoded;
}

31
oc/ocuri.c
View File

@ -622,7 +622,7 @@ ocparamreplace(char** params, const char* key, const char* value)
/* Provide % encoders and decoders */
static char* hexchars = "0123456789abcdef";
static char* hexchars = "0123456789abcdefABCDEF";
static void
toHex(unsigned int b, char hex[2])
@ -688,6 +688,15 @@ ocuriencode(char* s, char* allowable)
/* Return a string representing decoding of input; caller must free;*/
char*
ocuridecode(char* s)
{
return ocuridecodeonly(s,NULL);
}
/* Return a string representing decoding of input only for specified
characters; caller must free
*/
char*
ocuridecodeonly(char* s, char* only)
{
size_t slen;
char* decoded;
@ -696,6 +705,7 @@ ocuridecode(char* s)
unsigned int c;
if (s == NULL) return NULL;
if(only == NULL) only = "";
slen = strlen(s);
decoded = (char*)malloc(slen+1); /* Should be max we need */
@ -703,17 +713,18 @@ ocuridecode(char* s)
outptr = decoded;
inptr = s;
while((c = *inptr++)) {
if(c == '+')
if(c == '+' && strchr(only,'+') != NULL)
*outptr++ = ' ';
else if(c == '%') {
/* try to pull two more characters */
int x0, x1;
x0 = inptr[0];
if(x0 != '\0') {
x1 = inptr[1];
if(x0 != '\0') {
c = (fromHex(x0) << 4) | (fromHex(x1));
inptr += 2;
/* try to pull two hex more characters */
if(inptr[0] != '\0' && inptr[1] != '\0'
&& strchr(hexchars,inptr[0]) != NULL
&& strchr(hexchars,inptr[1]) != NULL) {
/* test conversion */
int xc = (fromHex(inptr[0]) << 4) | (fromHex(inptr[1]));
if(strchr(only,xc) != NULL) {
inptr += 2; /* decode it */
c = xc;
}
}
}

1
oc/ocuri.h
View File

@ -47,5 +47,6 @@ extern const char* ocurilookup(OCURI*, const char* param);
extern char* ocuriencode(char* s, char* allowable);
extern char* ocuridecode(char* s);
extern char* ocuridecodeonly(char* s, char*);
#endif /*OCURI_H*/