mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-25 17:40:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

diskless: make read and write loop to read/write whole file when persisting

Browse Source
This commit is contained in:
Dennis Heimbigner 2012-04-09 22:03:02 +00:00
parent 817a2cc978
commit 48ca394d2e
8 changed files with 887 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cf
View File

@ -5,8 +5,8 @@ if test $# != 0 ; then
cmds=$@
fi
HDF5=1
DAP=1
#HDF5=1
#DAP=1
#CDMR=1
#RPC=1
#PGI=1

199
libsrc/memio.c
View File

@ -18,37 +18,6 @@
#endif
#include "nc.h"
/* Turn off MMAP until we can figure out why it doesn't work */
#undef HAVE_MMAP
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
/* !HAVE_MREMAP => !HAVE_MMAP */
#ifndef HAVE_MREMAP
# undef HAVE_MMAP
#endif
#ifndef MAP_ANONYMOUS
# ifdef MAP_ANON
# define MAP_ANONYMOUS MAP_ANON
# endif
#endif
/* !MAP_ANONYMOUS => !HAVE_MMAP */
#ifndef MAP_ANONYMOUS
# undef HAVE_MMAP
#endif
#ifdef HAVE_MMAP
/* This is conditionalized by __USE_GNU ; why? */
extern void *mremap(void*,size_t,size_t,int);
# ifndef MREMAP_MAYMOVE
# define MREMAP_MAYMOVE 1
# endif
#endif /*HAVE_MMAP*/
#ifndef HAVE_SSIZE_T
#define ssize_t int
#endif
@ -107,10 +76,6 @@ typedef struct NCMEMIO {
off_t alloc;
off_t size;
off_t pos;
int mmap; /* 1=>ok to use mmap if available */
#ifdef HAVE_MMAP
int mapfd;
#endif
} NCMEMIO;
/* Forward */
@ -142,6 +107,11 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM
#endif
errno = 0;
/* Always force the allocated size to be a multiple of pagesize */
if(initialsize == 0) initialsize = pagesize;
if((initialsize % pagesize) != 0)
initialsize += (pagesize - (initialsize % pagesize));
nciop = (ncio* )calloc(1,sizeof(ncio));
if(nciop == NULL) {status = NC_ENOMEM; goto fail;}
@ -163,27 +133,12 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM
if(memio == NULL) {status = NC_ENOMEM; goto fail;}
*((void* *)&nciop->pvt) = memio;
/* See if ok to use mmap */
#ifdef HAVE_MMAP
memio->mmap = (sizeof(void*) < 8 && fIsSet(ioflags,NC_64BIT_OFFSET)?0:1);
#else
memio->mmap = 0;
#endif
memio->alloc = initialsize;
memio->memory = NULL;
memio->alloc = initialsize;
if(memio->alloc < pagesize)
memio->alloc = pagesize;
#ifdef HAVE_MMAP
if((memio->alloc % pagesize) != 0)
memio->alloc += (pagesize - (memio->alloc % pagesize));
#endif
memio->size = 0;
memio->pos = 0;
memio->persist = fIsSet(ioflags,NC_WRITE);
#ifdef HAVE_MMAP
memio->mapfd = -1;
#endif
if(nciopp) *nciopp = nciop;
if(memiop) *memiop = memio;
@ -236,24 +191,11 @@ memio_create(const char* path, int ioflags,
status = memio_new(path, ioflags, initialsz, &nciop, &memio);
if(status != NC_NOERR)
return status;
memio->size = 0;
if(!persist) { /* mmap => use anonymous mmap */
if(memio->mmap) { /* use mmap if available */
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = -1;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_POPULATE,
memio->mapfd,0);
{/* test writing of the mmap'd memory */
memio->memory[0] = 0;}
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap; just malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
if(!persist) {
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
} else { /*persist */
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
@ -270,23 +212,10 @@ memio_create(const char* path, int ioflags,
#endif
if(fd < 0) {status = errno; goto unwind_open;}
if(memio->mmap) {
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = fd;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_SHARED|MAP_FIXED,
memio->mapfd,0);
{/* test reading of the mmap'd memory */
int tst = memio->memory[0];}
#endif /*HAVE_MMAP*/
} else {/*memio->mmap*/
(void)close(fd); /* will reopen at nc_close */
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
(void)close(fd); /* will reopen at nc_close */
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
} /*!persist*/
fd = nc__pseudofd();
@ -344,7 +273,6 @@ memio_open(const char* path,
NCMEMIO* memio = NULL;
size_t sizehint;
off_t filesize;
ssize_t count;
if(path == NULL ||* path == 0)
return EINVAL;
@ -376,26 +304,28 @@ memio_open(const char* path,
status = memio_new(path, ioflags, filesize, &nciop, &memio);
if(status != NC_NOERR)
return status;
memio->size = filesize;
if(memio->mmap) {
#ifdef HAVE_MMAP
memio->mapfd = fd;
memio->memory = (char*)mmap(NULL,memio->alloc,
persist?(PROT_READ|PROT_WRITE):(PROT_READ),
MAP_SHARED,
memio->mapfd,0);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
/* Read the file into the memio memory */
count = read(fd, memio->memory, (size_t)filesize);
if(count < filesize) {status = NC_ENOTNC; goto unwind_open;}
(void)close(fd); /* until memio_close() */
/* Read the file into the memio memory */
/* We need to do multiple reads because there is no
guarantee that the amount read will be the full amount */
{
off_t red = memio->size;
char* pos = memio->memory;
while(red > 0) {
ssize_t count = read(fd, pos, red);
if(count < 0)
{close(fd); status = errno; goto unwind_open;}
if(count == 0)
{close(fd); status = NC_ENOTNC; goto unwind_open;}
red -= count;
pos += count;
}
}
memio->size = memio->alloc;
(void)close(fd); /* until memio_close() */
/* Use half the filesize as the blocksize */
sizehint = filesize/2;
@ -470,17 +400,11 @@ memio_pad_length(ncio* nciop, off_t length)
break;
}
if(memio->mmap) {
#ifdef HAVE_MMAP
newmem = (char*)mremap(memio->memory,memio->alloc,newsize,MREMAP_MAYMOVE);
if(newmem == NULL) return NC_ENOMEM;
#endif /*HAVE_MMAP*/
} else {
newmem = (char*)realloc(memio->memory,newsize);
if(newmem == NULL) return NC_ENOMEM;
/* zero out the extra memory */
memset((void*)(newmem+memio->alloc),0,(newsize - memio->alloc));
}
newmem = (char*)realloc(memio->memory,newsize);
if(newmem == NULL) return NC_ENOMEM;
/* zero out the extra memory */
memset((void*)(newmem+memio->alloc),0,(newsize - memio->alloc));
memio->memory = newmem;
memio->alloc = newsize;
}
@ -501,40 +425,39 @@ memio_close(ncio* nciop, int doUnlink)
{
int status = NC_NOERR;
NCMEMIO* memio;
int fd = -1;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
memio = (NCMEMIO*)nciop->pvt;
assert(memio != NULL);
if(memio->mmap) {
#ifdef HAVE_MMAP
status = munmap(memio->memory,memio->alloc);
if(status) goto done;
/* Do we need to close the file? */
if(memio->mapfd >= 0)
(void)close(memio->mapfd);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
/* See if the user wants the contents persisted to a file */
if(memio->persist) {
/* Try to open the file for writing */
int fd = open(nciop->path, O_WRONLY|O_CREAT|O_TRUNC, OPENMODE);
if(fd >= 0) {
long count = write(fd, memio->memory, memio->size);
/* See if the user wants the contents persisted to a file */
if(memio->persist) {
/* Try to open the file for writing */
fd = open(nciop->path, O_WRONLY|O_CREAT|O_TRUNC, OPENMODE);
if(fd >= 0) {
/* We need to do multiple writes because there is no
guarantee that the amount written will be the full amount */
off_t written = memio->size;
char* pos = memio->memory;
while(written > 0) {
ssize_t count = write(fd, pos, written);
if(count < 0)
status = errno;
else if(count < memio->size)
{status = NC_EDISKLESS; goto done;}
(void)close(fd);
} else
status = errno;
}
/* Free up things */
if(memio->memory != NULL) free(memio->memory);
{status = errno; goto done;}
if(count == 0)
{status = NC_ENOTNC; goto done;}
written -= count;
pos += count;
}
} else
status = errno;
/* Free up things */
if(memio->memory != NULL) free(memio->memory);
}
done:
/* do cleanup */
if(fd >= 0) (void)close(fd);
if(memio != NULL) free(memio);
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);

648
libsrc/mmapio.c Normal file
View File

@ -0,0 +1,648 @@
/*
* Copyright 1996, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <assert.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#ifdef _MSC_VER /* Microsoft Compilers */
#include <io.h>
#else
#include <unistd.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#include "nc.h"
/* Turn off MMAP until we can figure out why it doesn't work */
#undef HAVE_MMAP
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
/* !HAVE_MREMAP => !HAVE_MMAP */
#ifndef HAVE_MREMAP
# undef HAVE_MMAP
#endif
#ifndef MAP_ANONYMOUS
# ifdef MAP_ANON
# define MAP_ANONYMOUS MAP_ANON
# endif
#endif
/* !MAP_ANONYMOUS => !HAVE_MMAP */
#ifndef MAP_ANONYMOUS
# undef HAVE_MMAP
#endif
#ifdef HAVE_MMAP
/* This is conditionalized by __USE_GNU ; why? */
extern void *mremap(void*,size_t,size_t,int);
# ifndef MREMAP_MAYMOVE
# define MREMAP_MAYMOVE 1
# endif
#endif /*HAVE_MMAP*/
#ifndef HAVE_SSIZE_T
#define ssize_t int
#endif
#ifndef SEEK_SET
#define SEEK_SET 0
#define SEEK_CUR 1
#define SEEK_END 2
#endif
/* Define the mode flags for create: rw by owner, no access by anyone else */
#define OPENMODE 0600
#define OPENANYMODE 0666
#include "ncio.h"
#include "fbits.h"
#include "rnd.h"
/* #define INSTRUMENT 1 */
#if INSTRUMENT /* debugging */
#undef NDEBUG
#include <stdio.h>
#include "instr.h"
#endif
#ifndef MEMIO_MAXBLOCKSIZE
#define MEMIO_MAXBLOCKSIZE 268435456 /* sanity check, about X_SIZE_T_MAX/8 */
#endif
#undef MIN /* system may define MIN somewhere and complain */
#define MIN(mm,nn) (((mm) < (nn)) ? (mm) : (nn))
#if !defined(NDEBUG) && !defined(X_INT_MAX)
#define X_INT_MAX 2147483647
#endif
#if 0 /* !defined(NDEBUG) && !defined(X_ALIGN) */
#define X_ALIGN 4
#else
#undef X_ALIGN
#endif
/* Define the amount by which memory is incremented on realloc */
#define DEFAULT_BLOCKSIZE (0x2000)
#define TACTIC_INCR 1
#define TACTIC_DOUBLE 2
#define TACTIC TACTIC_DOUBLE
/* Private data for memio */
typedef struct NCMEMIO {
int locked; /* => we cannot realloc */
int persist; /* => save to a file; triggered by NC_WRITE */
char* memory;
off_t alloc;
off_t size;
off_t pos;
int mmap; /* 1=>ok to use mmap if available */
#ifdef HAVE_MMAP
int mapfd;
#endif
} NCMEMIO;
/* Forward */
static int memio_rel(ncio *const nciop, off_t offset, int rflags);
static int memio_get(ncio *const nciop, off_t offset, size_t extent, int rflags, void **const vpp);
static int memio_move(ncio *const nciop, off_t to, off_t from, size_t nbytes, int rflags);
static int memio_sync(ncio *const nciop);
static int memio_filesize(ncio* nciop, off_t* filesizep);
static int memio_pad_length(ncio* nciop, off_t length);
static int memio_close(ncio* nciop, int);
/* Mnemonic */
#define DOOPEN 1
/* Create a new ncio struct to hold info about the file. */
static int
memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEMIO** memiop)
{
int status = NC_NOERR;
ncio* nciop = NULL;
NCMEMIO* memio = NULL;
int openfd = -1;
#if defined HAVE_SYSCONF
long pagesize = sysconf(_SC_PAGE_SIZE);
#elif defined HAVE_GETPAGESIZE
long pagesize = getpagesize();
#else
long pagesize = 4096; /* good guess */
#endif
errno = 0;
nciop = (ncio* )calloc(1,sizeof(ncio));
if(nciop == NULL) {status = NC_ENOMEM; goto fail;}
nciop->ioflags = ioflags;
*((int*)&nciop->fd) = -1; /* caller will fix */
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
*((ncio_relfunc**)&nciop->rel) = memio_rel;
*((ncio_getfunc**)&nciop->get) = memio_get;
*((ncio_movefunc**)&nciop->move) = memio_move;
*((ncio_syncfunc**)&nciop->sync) = memio_sync;
*((ncio_filesizefunc**)&nciop->filesize) = memio_filesize;
*((ncio_pad_lengthfunc**)&nciop->pad_length) = memio_pad_length;
*((ncio_closefunc**)&nciop->close) = memio_close;
memio = (NCMEMIO*)calloc(1,sizeof(NCMEMIO));
if(memio == NULL) {status = NC_ENOMEM; goto fail;}
*((void* *)&nciop->pvt) = memio;
/* See if ok to use mmap */
#ifdef HAVE_MMAP
memio->mmap = (sizeof(void*) < 8 && fIsSet(ioflags,NC_64BIT_OFFSET)?0:1);
#else
memio->mmap = 0;
#endif
memio->memory = NULL;
memio->alloc = initialsize;
if(memio->alloc < pagesize)
memio->alloc = pagesize;
#ifdef HAVE_MMAP
if((memio->alloc % pagesize) != 0)
memio->alloc += (pagesize - (memio->alloc % pagesize));
#endif
memio->size = 0;
memio->pos = 0;
memio->persist = fIsSet(ioflags,NC_WRITE);
#ifdef HAVE_MMAP
memio->mapfd = -1;
#endif
if(nciopp) *nciopp = nciop;
if(memiop) *memiop = memio;
done:
if(openfd >= 0) close(openfd);
return status;
fail:
if(nciop != NULL) {
if(nciop->path != NULL) free((char*)nciop->path);
}
goto done;
}
/* Create a file, and the ncio struct to go with it. This function is
only called from nc__create_mp.
path - path of file to create.
ioflags - flags from nc_create
initialsz - From the netcdf man page: "The argument
Iinitialsize sets the initial size of the file at creation time."
igeto -
igetsz -
sizehintp - the size of a page of data for buffered reads and writes.
nciopp - pointer to a pointer that will get location of newly
created and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
*/
int
memio_create(const char* path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t* sizehintp,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;
int fd;
int status;
NCMEMIO* memio = NULL;
int persist = (ioflags & NC_WRITE?1:0);
int oflags;
if(path == NULL ||* path == 0)
return NC_EINVAL;
/* For diskless open has, the file must be classic version 1 or 2.*/
if(fIsSet(ioflags,NC_NETCDF4))
return NC_EDISKLESS; /* violates constraints */
status = memio_new(path, ioflags, initialsz, &nciop, &memio);
if(status != NC_NOERR)
return status;
if(!persist) { /* mmap => use anonymous mmap */
if(memio->mmap) { /* use mmap if available */
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = -1;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED|MAP_POPULATE,
memio->mapfd,0);
{/* test writing of the mmap'd memory */
memio->memory[0] = 0;}
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap; just malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
} else { /*persist */
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= (O_CREAT|O_TRUNC);
if(fIsSet(ioflags,NC_NOCLOBBER))
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
if(fd < 0) {status = errno; goto unwind_open;}
if(memio->mmap) {
#ifdef HAVE_MMAP
/* force map at sbrk */
void* top = sbrk(0);
memio->mapfd = fd;
memio->memory = (char*)mmap(top,memio->alloc,
PROT_READ|PROT_WRITE,MAP_SHARED|MAP_FIXED,
memio->mapfd,0);
{/* test reading of the mmap'd memory */
int tst = memio->memory[0];}
#endif /*HAVE_MMAP*/
} else {/*memio->mmap*/
(void)close(fd); /* will reopen at nc_close */
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
}
} /*!persist*/
fd = nc__pseudofd();
*((int* )&nciop->fd) = fd;
fSet(nciop->ioflags, NC_WRITE);
if(igetsz != 0)
{
status = nciop->get(nciop,
igeto, igetsz,
RGN_WRITE,
mempp);
if(status != NC_NOERR)
goto unwind_open;
}
/* Pick a default sizehint */
if(sizehintp) *sizehintp = DEFAULT_BLOCKSIZE;
*nciopp = nciop;
return NC_NOERR;
unwind_open:
memio_close(nciop,1);
return status;
}
/* This function opens the data file. It is only called from nc.c,
from nc__open_mp and nc_delete_mp.
path - path of data file.
ioflags - flags passed into nc_open.
igeto - looks like this function can do an initial page get, and
igeto is going to be the offset for that. But it appears to be
unused
igetsz - the size in bytes of initial page get (a.k.a. extent). Not
ever used in the library.
sizehintp - the size of a page of data for buffered reads and writes.
nciopp - pointer to pointer that will get address of newly created
and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
*/
int
memio_open(const char* path,
int ioflags,
off_t igeto, size_t igetsz, size_t* sizehintp,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;
int fd;
int status;
int persist = (fIsSet(ioflags,NC_WRITE)?1:0);
int oflags;
NCMEMIO* memio = NULL;
size_t sizehint;
off_t filesize;
ssize_t count;
if(path == NULL ||* path == 0)
return EINVAL;
assert(sizehintp != NULL);
sizehint = *sizehintp;
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
#endif
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
#endif
if(fd < 0) {status = errno; goto unwind_open;}
/* get current filesize = max(|file|,initialize)*/
filesize = lseek(fd,0,SEEK_END);
if(filesize < 0) {status = errno; goto unwind_open;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
if(filesize < (off_t)sizehint)
filesize = (off_t)sizehint;
status = memio_new(path, ioflags, filesize, &nciop, &memio);
if(status != NC_NOERR)
return status;
if(memio->mmap) {
#ifdef HAVE_MMAP
memio->mapfd = fd;
memio->memory = (char*)mmap(NULL,memio->alloc,
persist?(PROT_READ|PROT_WRITE):(PROT_READ),
MAP_SHARED,
memio->mapfd,0);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
/* Read the file into the memio memory */
count = read(fd, memio->memory, (size_t)filesize);
if(count < filesize) {status = NC_ENOTNC; goto unwind_open;}
(void)close(fd); /* until memio_close() */
}
memio->size = memio->alloc;
/* Use half the filesize as the blocksize */
sizehint = filesize/2;
fd = nc__pseudofd();
*((int* )&nciop->fd) = fd;
if(igetsz != 0)
{
status = nciop->get(nciop,
igeto, igetsz,
0,
mempp);
if(status != NC_NOERR)
goto unwind_open;
}
*sizehintp = sizehint;
*nciopp = nciop;
return NC_NOERR;
unwind_open:
memio_close(nciop,0);
return status;
}
/*
* Get file size in bytes.
*/
static int
memio_filesize(ncio* nciop, off_t* filesizep)
{
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
if(filesizep != NULL) *filesizep = memio->size;
return NC_NOERR;
}
/*
* Sync any changes to disk, then truncate or extend file so its size
* is length. This is only intended to be called before close, if the
* file is open for writing and the actual size does not match the
* calculated size, perhaps as the result of having been previously
* written in NOFILL mode.
*/
static int
memio_pad_length(ncio* nciop, off_t length)
{
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
if(!fIsSet(nciop->ioflags, NC_WRITE))
return EPERM; /* attempt to write readonly file*/
/* Realloc the allocated memory */
if(memio->locked > 0)
return NC_EDISKLESS;
if(length > memio->alloc) {
off_t newsize;
char* newmem;
switch(TACTIC) {
case TACTIC_DOUBLE:
newsize = (memio->alloc * 2);
break;
case TACTIC_INCR:
default:
newsize = length + (length % DEFAULT_BLOCKSIZE);
break;
}
if(memio->mmap) {
#ifdef HAVE_MMAP
newmem = (char*)mremap(memio->memory,memio->alloc,newsize,MREMAP_MAYMOVE);
if(newmem == NULL) return NC_ENOMEM;
#endif /*HAVE_MMAP*/
} else {
newmem = (char*)realloc(memio->memory,newsize);
if(newmem == NULL) return NC_ENOMEM;
/* zero out the extra memory */
memset((void*)(newmem+memio->alloc),0,(newsize - memio->alloc));
}
memio->memory = newmem;
memio->alloc = newsize;
}
memio->size = length;
return NC_NOERR;
}
/* Write out any dirty buffers to disk and
ensure that next read will get data from disk.
Sync any changes, then close the open file associated with the ncio
struct, and free its memory.
nciop - pointer to ncio to close.
doUnlink - if true, unlink file
*/
static int
memio_close(ncio* nciop, int doUnlink)
{
int status = NC_NOERR;
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
memio = (NCMEMIO*)nciop->pvt;
assert(memio != NULL);
if(memio->mmap) {
#ifdef HAVE_MMAP
status = munmap(memio->memory,memio->alloc);
if(status) goto done;
/* Do we need to close the file? */
if(memio->mapfd >= 0)
(void)close(memio->mapfd);
#endif /*HAVE_MMAP*/
} else {/*!memio->mmap*/
/* See if the user wants the contents persisted to a file */
if(memio->persist) {
/* Try to open the file for writing */
int fd = open(nciop->path, O_WRONLY|O_CREAT|O_TRUNC, OPENMODE);
if(fd >= 0) {
long count = write(fd, memio->memory, memio->size);
if(count < 0)
status = errno;
else if(count < memio->size)
{status = NC_EDISKLESS; goto done;}
(void)close(fd);
} else
status = errno;
}
/* Free up things */
if(memio->memory != NULL) free(memio->memory);
}
done:
/* do cleanup */
if(memio != NULL) free(memio);
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
return status;
}
static int
guarantee(ncio* nciop, off_t endpoint)
{
NCMEMIO* memio = (NCMEMIO*)nciop->pvt;
if(endpoint > memio->alloc) {
/* extend the allocated memory and size */
int status = memio_pad_length(nciop,endpoint);
if(status != NC_NOERR) return status;
}
if(memio->size < endpoint)
memio->size = endpoint;
return NC_NOERR;
}
/*
* Request that the region (offset, extent)
* be made available through *vpp.
*/
static int
memio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** const vpp)
{
int status = NC_NOERR;
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
status = guarantee(nciop, offset+extent);
memio->locked++;
if(status != NC_NOERR) return status;
if(vpp) *vpp = memio->memory+offset;
return NC_NOERR;
}
/*
* Like memmove(), safely move possibly overlapping data.
*/
static int
memio_move(ncio* const nciop, off_t to, off_t from, size_t nbytes, int ignored)
{
int status = NC_NOERR;
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
if(from < to) {
/* extend if "to" is not currently allocated */
status = guarantee(nciop,to+nbytes);
if(status != NC_NOERR) return status;
}
/* check for overlap */
if((to + nbytes) > from || (from + nbytes) > to) {
/* Ranges overlap */
#ifdef HAVE_MEMMOVE
memmove((void*)(memio->memory+to),(void*)(memio->memory+from),nbytes);
#else
off_t overlap;
off_t nbytes1;
if((from + nbytes) > to) {
overlap = ((from + nbytes) - to); /* # bytes of overlap */
nbytes1 = (nbytes - overlap); /* # bytes of non-overlap */
/* move the non-overlapping part */
memcpy((void*)(memio->memory+(to+overlap)),
(void*)(memio->memory+(from+overlap)),
nbytes1);
/* move the overlapping part */
memcpy((void*)(memio->memory+to),
(void*)(memio->memory+from),
overlap);
} else { /*((to + nbytes) > from) */
overlap = ((to + nbytes) - from); /* # bytes of overlap */
nbytes1 = (nbytes - overlap); /* # bytes of non-overlap */
/* move the non-overlapping part */
memcpy((void*)(memio->memory+to),
(void*)(memio->memory+from),
nbytes1);
/* move the overlapping part */
memcpy((void*)(memio->memory+(to+nbytes1)),
(void*)(memio->memory+(from+nbytes1)),
overlap);
}
#endif
} else {/* no overlap */
memcpy((void*)(memio->memory+to),(void*)(memio->memory+from),nbytes);
}
return status;
}
static int
memio_rel(ncio* const nciop, off_t offset, int rflags)
{
NCMEMIO* memio;
if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL;
memio = (NCMEMIO*)nciop->pvt;
memio->locked--;
return NC_NOERR; /* do nothing */
}
/*
* Write out any dirty buffers to disk and
* ensure that next read will get data from disk.
*/
static int
memio_sync(ncio* const nciop)
{
return NC_NOERR; /* do nothing */
}

15
nc_test/Make0 Normal file
View File

@ -0,0 +1,15 @@
# Test c output
T=tst_diskless3
#VG=valgrind --leak-check=full
CFLAGS=-g -O0 -I.. -I../include
LDFLAGS=../liblib/.libs/libnetcdf.a -L/share/ed/local/spock/lib -lhdf5_hl -lhdf5 -lz -lm -lcurl
LD_RUN_PATH=/share/ed/local/spock/lib
# cd .. ; ${MAKE} all
all::
export LD_RUN_PATH; export CFLAGS; export LDFLAGS; \
gcc -o t ${CFLAGS} ${T}.c ${LDFLAGS}
${VG} ./t

10
nc_test/Makefile.am
View File

@ -49,7 +49,12 @@ if USE_NETCDF4
check_PROGRAMS += tst_diskless2
endif
TESTS = $(TESTPROGRAMS) run_diskless.sh
TESTS = $(TESTPROGRAMS)
TESTS += run_diskless.sh
#if LARGE_FILE_TESTS
TESTS += run_diskless2.sh
#endif
# This will run a bunch of the test programs with valgrind, the memory
# checking tool. (Valgrind must be present for this to work.)
@ -64,7 +69,8 @@ endif # USE_VALGRIND_TESTS
# Distribute the .c files so that m4 isn't required on the users
# machine.
EXTRA_DIST = test_get.m4 test_put.m4 run_valgrind_tests.sh run_diskless.sh
EXTRA_DIST = test_get.m4 test_put.m4 run_valgrind_tests.sh \
run_diskless.sh run_diskless2.sh
# ref_tst_diskless2.cdl is for diff comparison and to produce tst_diskless2.c
EXTRA_DIST += ref_tst_diskless2.cdl

39
nc_test/run_diskless.sh
View File

@ -2,14 +2,15 @@
set -e
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`
#Constants
FILE1=tst_diskless.nc
FILE2=tst_diskless2.nc
FILE3=tst_diskless3.nc
FILE4=tst_diskless4.nc
#SIZE=1073741824
SIZE=536870912
echo ""
echo "Testing in-memory (diskless) files with and without persistence"
@ -114,34 +115,4 @@ diff tst_diskless3_file.cdl tst_diskless3_memory.cdl
# cleanup
rm -f $FILE3 tst_diskless3_file.cdl tst_diskless3_memory.cdl
# Create the reference ncdump output for tst_diskless4
rm -fr tst_diskless4.cdl
echo "netcdf tst_diskless4 {" >>tst_diskless4.cdl
echo "dimensions:" >>tst_diskless4.cdl
echo " dim = $SIZE ;" >>tst_diskless4.cdl
echo "variables:" >>tst_diskless4.cdl
echo " byte var(dim) ;" >>tst_diskless4.cdl
echo "}" >>tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 create
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 creatediskless
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
time ./tst_diskless4 open
echo ""
time ./tst_diskless4 opendiskless
# cleanup
rm -f $FILE4 tst_diskless4.cdl
exit

53
nc_test/run_diskless2.sh Executable file
View File

@ -0,0 +1,53 @@
#!/bin/sh
set -e
# Get the target OS and CPU
CPU=`uname -p`
OS=`uname`
#Constants
FILE4=tst_diskless4.nc
# Compute the file size for tst_diskless4
SIZE=0
case $CPU in
*_64*) SIZE=3000000000;;
*) SIZE=1000000000;;
esac
# Create the reference ncdump output for tst_diskless4
rm -fr tst_diskless4.cdl
echo "netcdf tst_diskless4 {" >>tst_diskless4.cdl
echo "dimensions:" >>tst_diskless4.cdl
echo " dim = 1000000000 ;" >>tst_diskless4.cdl
echo "variables:" >>tst_diskless4.cdl
echo " byte var0(dim) ;" >>tst_diskless4.cdl
if test $SIZE = 3000000000 ; then
echo " byte var1(dim) ;" >>tst_diskless4.cdl
echo " byte var2(dim) ;" >>tst_diskless4.cdl
fi
echo "}" >>tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 $SIZE create
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
rm -f $FILE4
time ./tst_diskless4 $SIZE creatediskless
# Validate it
../ncdump/ncdump -h $FILE4 |diff - tst_diskless4.cdl
echo ""
time ./tst_diskless4 $SIZE open
echo ""
time ./tst_diskless4 $SIZE opendiskless
# cleanup
rm -f $FILE4 tst_diskless4.cdl
exit

174
nc_test/tst_diskless4.c
View File

@ -16,27 +16,13 @@
#undef XFAIL
/*
* Depending on argv, do one of the following:
1. create a file with var of size 0x40000000 (~ 1 gig)
2. create a file with var of size 0x40000000 (~ 1 gig) but using NC_DISKLESS
3. open a file with var of size 0x40000000 (~ 1 gig) and read it 4096 bytes
at a time
4. open a file with var of size 0x40000000 (~ 1 gig) and read it 4096 bytes
at a time using NC_DISKLESS
*/
#define FILE_NAME "tst_diskless4.nc"
#define CHUNKSIZE 4096
#define GIG (1024*1024*1024)
#define DATASIZE (CHUNKSIZE/sizeof(int))
#define DIMMAX 1000000000
typedef enum Tag {Create,CreateDiskless,Open,OpenDiskless} Tag;
static size_t varsize;
static size_t pieces;
static float gigsize;
#define REPORT err_report(status,__FILE__,__LINE__)
static void
@ -54,39 +40,53 @@ int
main(int argc, char **argv)
{
int status = NC_NOERR;
int i,j;
int i,j,iv;
unsigned int data[DATASIZE];
size_t start[1];
size_t count[1];
Tag tag = Create;
int cmode = 0;
int ncid,varid;
int ncid;
int dimids[1];
void* memory;
int nvars;
int varids[4096];
size_t varsize;
size_t filesize;
/* First, determine a reasonable file size that malloc will actually allocate
on this machine
*/
for(varsize=GIG/2;;varsize/=2) {
void* memory = malloc(varsize);
if(memory != NULL) {free(memory); break;}
free(memory);
}
/* Compute other values relative to varsize */
pieces = varsize/CHUNKSIZE;
gigsize = (varsize/GIG);
/* Get the specified var/file size */
if(argc > 1) {
if(strcmp(argv[1],"create")==0) tag = Create;
else if(strcmp(argv[1],"creatediskless")==0) tag = CreateDiskless;
else if(strcmp(argv[1],"open")==0) tag = Open;
else if(strcmp(argv[1],"opendiskless")==0) tag = OpenDiskless;
filesize = atol(argv[1]);
} else {
if(sizeof(size_t) == 4)
filesize = 1000000000;
else if(sizeof(size_t) == 8)
filesize = 3000000000;
else {
fprintf(stderr,"illegal tag: %s",argv[1]);
fprintf(stderr,"Cannot compute filesize\n");
exit(1);
}
}
/* Test that we can malloc that much space */
memory = malloc(filesize);
if(memory == NULL) {
fprintf(stderr,"Cannot malloc %lu bytes\n",(unsigned long)filesize);
exit(1);
}
free(memory);
if(argc > 2) {
if(strcmp(argv[2],"create")==0) tag = Create;
else if(strcmp(argv[2],"creatediskless")==0) tag = CreateDiskless;
else if(strcmp(argv[2],"open")==0) tag = Open;
else if(strcmp(argv[2],"opendiskless")==0) tag = OpenDiskless;
else {
fprintf(stderr,"illegal tag: %s",argv[2]);
exit(1);
}
} else
tag = Create; /* default */
switch (tag) {
case Create: printf("\n*** Create file\n"); break;
@ -109,63 +109,79 @@ main(int argc, char **argv)
REPORT;
if((status=nc_set_fill(ncid, NC_NOFILL, NULL)))
REPORT;
/* Only need 1 dimension */
if((status=nc_def_dim(ncid, "dim", DIMMAX, &dimids[0])))
REPORT;
break;
case Open:
case OpenDiskless:
if((status=nc_open(FILE_NAME, cmode, &ncid)))
REPORT;
break;
}
switch (tag) {
case Create:
case CreateDiskless:
if((status=nc_def_dim(ncid, "dim", varsize, &dimids[0])))
REPORT;
if((status=nc_def_var(ncid, "var", NC_BYTE, 1, dimids, &varid)))
REPORT;
if((status=nc_enddef(ncid)))
REPORT;
break;
case Open:
case OpenDiskless:
if((status=nc_inq_dimid(ncid, "dim", &dimids[0])))
REPORT;
if((status=nc_inq_varid(ncid, "var", &varid)))
REPORT;
break;
}
switch (tag) {
case Create:
case CreateDiskless:
/* Fill and put as integers */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
for(j=0;j<DATASIZE;j++) data[j] = (i*CHUNKSIZE)+j;
if((status=nc_put_vara(ncid,varid,start,count,(void*)data)))
REPORT;
}
varsize = DIMMAX;
nvars = filesize / varsize;
assert((filesize % DIMMAX) == 0);
assert(nvars < 4096);
for(iv=0;iv<nvars;iv++) {
char varname[32];
sprintf(varname,"var%d",iv);
switch (tag) {
case Create:
case CreateDiskless:
if((status=nc_def_var(ncid, varname, NC_BYTE, 1, &dimids[0], &varids[iv])))
REPORT;
break;
case Open:
case OpenDiskless:
/* Read the var contents and validate */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
if((status=nc_get_vara(ncid,varid,start,count,(void*)data)))
REPORT;
for(j=0;j<DATASIZE;j++) {
unsigned int expected = (i*CHUNKSIZE)+j;
if(data[j] != expected) {
printf("mismatch: i=%u j=%u data=%u; should be %u\n",
i,j,data[j],expected);
err++;
case Open:
case OpenDiskless:
if((status=nc_inq_varid(ncid, varname, &varids[iv])))
REPORT;
break;
}
}
if(tag == Create || tag == CreateDiskless) {
if((status=nc_enddef(ncid)))
REPORT;
}
for(iv=0;iv<nvars;iv++) {
size_t pieces = varsize/CHUNKSIZE;
switch (tag) {
case Create:
case CreateDiskless:
/* Fill and put as integers */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
for(j=0;j<DATASIZE;j++) data[j] = iv*((i*CHUNKSIZE)+j);
if((status=nc_put_vara(ncid,varids[iv],start,count,(void*)data)))
REPORT;
}
break;
case Open:
case OpenDiskless:
/* Read the var contents and validate */
for(i=0;i<pieces;i++) {
start[0] = i*CHUNKSIZE;
count[0] = CHUNKSIZE;
if((status=nc_get_vara(ncid,varids[iv],start,count,(void*)data)))
REPORT;
for(j=0;j<DATASIZE;j++) {
unsigned int expected = iv*((i*CHUNKSIZE)+j);
if(data[j] != expected) {
printf("mismatch: iv=%d i=%u j=%u data=%u; should be %u\n",
iv, i,j,data[j],expected);
err++;
}
}
}
break;
}
break;
}
if((status=nc_close(ncid)))