mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-06 18:00:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

The original mem branch somehow got

Browse Source 
hosed, so I rebuilt it as a new mem2 branch.
This commit is contained in:
dmh 2015-05-28 15:03:02 -06:00
parent 6ed4770e0a
commit 3dd807a155
18 changed files with 674 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/Makefile.am
View File

@ -10,6 +10,10 @@ if BUILD_PARALLEL
include_HEADERS += netcdf_par.h
endif
if BUILD_DISKLESS
include_HEADERS += netcdf_mem.h
endif
noinst_HEADERS = nc_logging.h nc_tests.h fbits.h nc.h \
nclist.h ncuri.h utf8proc.h ncdispatch.h ncdimscale.h \
netcdf_f.h err_macros.h ncbytes.h nchashmap.h ceconstraints.h rnd.h \

13
include/ncdispatch.h
View File

@ -101,6 +101,11 @@ typedef struct NC_MPI_INFO {
MPI_Info info;
} NC_MPI_INFO;
typedef struct NC_MEM_INFO {
size_t size;
void* memory;
} NC_MEM_INFO;
/* Define known dispatch tables and initializers */
/*Forward*/
@ -164,11 +169,11 @@ struct NC;
int NC_create(const char *path, int cmode,
size_t initialsz, int basepe, size_t *chunksizehintp,
int useparallel,void* mpi_info,
int useparallel, void* parameters,
int *ncidp);
int NC_open(const char *path, int cmode,
int basepe, size_t *chunksizehintp,
int useparallel, void* mpi_info,
int useparallel, void* parameters,
int *ncidp);
/* Expose the default vars and varm dispatch entries */
@ -253,7 +258,8 @@ int (*inq_var_all)(int ncid, int varid, char *name, nc_type *xtypep,
int (*var_par_access)(int, int, int);
/* Note the following may still be invoked by netcdf client code
even when the file is a classic file
even when the file is a classic file; they will just return an error or
be ignored.
*/
#ifdef USE_NETCDF4
int (*show_metadata)(int);
@ -294,6 +300,7 @@ int (*def_var_endian)(int, int, int);
int (*set_var_chunk_cache)(int, int, size_t, size_t, float);
int (*get_var_chunk_cache)(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp);
#endif /*USE_NETCDF4*/
};
/* Following functions must be handled as non-dispatch */

7
include/netcdf.h
View File

@ -120,18 +120,21 @@ extern "C" {
#define NC_NOFILL 0x100 /**< Argument to nc_set_fill() to turn off filling of data. */
/* Define the ioflags bits for nc_create and nc_open.
currently unused: 0x0010,0x0020,0x0040,0x0080
currently unused:
0x0002
0x0040
0x0080
and the whole upper 16 bits
*/
#define NC_NOWRITE 0x0000 /**< Set read-only access for nc_open(). */
#define NC_WRITE 0x0001 /**< Set read-write access for nc_open(). */
/* unused: 0x0002 */
#define NC_CLOBBER 0x0000 /**< Destroy existing file. Mode flag for nc_create(). */
#define NC_NOCLOBBER 0x0004 /**< Don't destroy existing file. Mode flag for nc_create(). */
#define NC_DISKLESS 0x0008 /**< Use diskless file. Mode flag for nc_open() or nc_create(). */
#define NC_MMAP 0x0010 /**< Use diskless file with mmap. Mode flag for nc_open() or nc_create(). */
#define NC_INMEMORY 0x0020 /**< Read from memory. Mode flag for nc_open() or nc_create(). */
#define NC_CLASSIC_MODEL 0x0100 /**< Enforce classic model. Mode flag for nc_create(). */
#define NC_64BIT_OFFSET 0x0200 /**< Use large (64-bit) file offsets. Mode flag for nc_create(). */

255
libdispatch/dfile.c
View File

@ -78,77 +78,9 @@ nc_local_initialize(void)
}
static int
NC_check_file_type(const char *path, int use_parallel, void *mpi_info,
int* model, int* version)
NC_interpret_magic_number(char* magic, int* model, int* version, int use_parallel)
{
char magic[MAGIC_NUMBER_LEN];
int status = NC_NOERR;
*model = 0;
/* Get the 4-byte magic from the beginning of the file. Don't use posix
* for parallel, use the MPI functions instead. */
#ifdef USE_PARALLEL
if (use_parallel)
{
MPI_File fh;
MPI_Status mstatus;
int retval;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
if(mpi_info != NULL) {
comm = ((NC_MPI_INFO*)mpi_info)->comm;
info = ((NC_MPI_INFO*)mpi_info)->info;
}
if((retval = MPI_File_open(comm, (char *)path, MPI_MODE_RDONLY,info,
&fh)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
if((retval = MPI_File_read(fh, magic, MAGIC_NUMBER_LEN, MPI_CHAR,
&mstatus)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
if((retval = MPI_File_close(&fh)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
} else
#endif /* USE_PARALLEL */
{
FILE *fp;
size_t i;
#ifdef HAVE_SYS_STAT_H
struct stat st;
#endif
if(path == NULL || strlen(path)==0)
{status = NC_EINVAL; goto done;}
if (!(fp = fopen(path, "r")))
{status = errno; goto done;}
#ifdef HAVE_SYS_STAT_H
/* The file must be at least MAGIC_NUMBER_LEN in size,
or otherwise the following fread will exhibit unexpected
behavior. */
if(!(fstat(fileno(fp),&st) == 0)) {
fclose(fp);
status = errno;
goto done;
}
if(st.st_size < MAGIC_NUMBER_LEN) {
fclose(fp);
status = NC_ENOTNC;
goto done;
}
#endif
i = fread(magic, MAGIC_NUMBER_LEN, 1, fp);
fclose(fp);
if(i == 0)
{status = NC_ENOTNC; goto done;}
if(i != 1)
{status = errno; goto done;}
}
int status = NC_NOERR;
/* Look at the magic number */
/* Ignore the first byte for HDF */
#ifdef USE_NETCDF4
@ -177,6 +109,93 @@ NC_check_file_type(const char *path, int use_parallel, void *mpi_info,
*model = (use_parallel || *version == 5)?NC_DISPATCH_NC5:NC_DISPATCH_NC3;
} else
{status = NC_ENOTNC; goto done;}
done:
return status;
}
static int
NC_check_file_type(const char *path, int flags, void *parameters,
int* model, int* version)
{
char magic[MAGIC_NUMBER_LEN];
int status = NC_NOERR;
int diskless = ((flags & NC_DISKLESS) == NC_DISKLESS);
int persist = ((flags & NC_WRITE) == NC_WRITE);
int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO);
int inmemory = (diskless && ((flags & NC_INMEMORY) == NC_INMEMORY));
*model = 0;
if(inmemory) {
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
if(meminfo == NULL || meminfo->size < MAGIC_NUMBER_LEN)
{status = NC_EDISKLESS; goto done;}
memcpy(magic,meminfo->memory,MAGIC_NUMBER_LEN);
} else {/* presumably a real file */
/* Get the 4-byte magic from the beginning of the file. Don't use posix
* for parallel, use the MPI functions instead. */
#ifdef USE_PARALLEL
if (use_parallel) {
MPI_File fh;
MPI_Status mstatus;
int retval;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
if(parameters != NULL) {
comm = ((NC_MPI_INFO*)parameters)->comm;
info = ((NC_MPI_INFO*)parameters)->info;
}
if((retval = MPI_File_open(comm,(char*)path,MPI_MODE_RDONLY,info,
&fh)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
if((retval = MPI_File_read(fh, magic, MAGIC_NUMBER_LEN, MPI_CHAR,
&mstatus)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
if((retval = MPI_File_close(&fh)) != MPI_SUCCESS)
{status = NC_EPARINIT; goto done;}
} else
#endif /* USE_PARALLEL */
{
FILE *fp;
size_t i;
#ifdef HAVE_SYS_STAT_H
struct stat st;
#endif
if(path == NULL || strlen(path)==0)
{status = NC_EINVAL; goto done;}
if (!(fp = fopen(path, "r")))
{status = errno; goto done;}
#ifdef HAVE_SYS_STAT_H
/* The file must be at least MAGIC_NUMBER_LEN in size,
or otherwise the following fread will exhibit unexpected
behavior. */
if(!(fstat(fileno(fp),&st) == 0)) {
fclose(fp);
status = errno;
goto done;
}
if(st.st_size < MAGIC_NUMBER_LEN) {
fclose(fp);
status = NC_ENOTNC;
goto done;
}
#endif
i = fread(magic, MAGIC_NUMBER_LEN, 1, fp);
fclose(fp);
if(i == 0)
{status = NC_ENOTNC; goto done;}
if(i != 1)
{status = errno; goto done;}
}
} /* !inmemory */
/* Look at the magic number */
status = NC_interpret_magic_number(magic,model,version,use_parallel);
done:
return status;
@ -648,6 +667,45 @@ nc__open(const char *path, int mode,
NULL, ncidp);
}
/**
Open a netCDF file with the contents taken from a block of memory.
\param path Must be non-null, but otherwise only used to set the dataset name.
\param size The length of the block of memory being passed.
\param memory Pointer to the block of memory containing the contents
of a netcdf file.
\param ncidp Pointer to location where returned netCDF ID is to be
stored.
\returns ::NC_NOERR No error.
\returns ::NC_ENOMEM Out of memory.
\returns Various other netcdf classic and netcdf 4 errors.
*/
int
nc_open_mem(const char* path, int mode, size_t size, void* memory, int* ncidp)
{
#ifdef USE_DISKLESS
NC_MEM_INFO meminfo;
/* Sanity checks */
if(memory == NULL || size < MAGIC_NUMBER_LEN || path == NULL)
return NC_EINVAL;
if(mode & (NC_WRITE|NC_MPIIO|NC_MPIPOSIX|NC_MMAP))
return NC_EINVAL;
mode |= (NC_INMEMORY|NC_DISKLESS);
meminfo.size = size;
meminfo.memory = memory;
return NC_open(path, mode, 0, NULL, 0, &meminfo, ncidp);
#else
return NC_EDISKLESS;
#endif
}
/**
\internal
@ -1489,7 +1547,7 @@ applies to classic and 64-bit offset files.
\param useparallel Non-zero if parallel I/O is to be used on this
file.
\param mpi_info Pointer to MPI comm and info.
\param parameters Pointer to MPI comm and info.
\param ncidp Pointer to location where returned netCDF ID is to be
stored.
@ -1499,7 +1557,7 @@ stored.
int
NC_create(const char *path, int cmode, size_t initialsz,
int basepe, size_t *chunksizehintp, int useparallel,
void* mpi_info, int *ncidp)
void* parameters, int *ncidp)
{
int stat = NC_NOERR;
NC* ncp = NULL;
@ -1623,7 +1681,7 @@ NC_create(const char *path, int cmode, size_t initialsz,
/* Assume create will fill in remaining ncp fields */
if ((stat = dispatcher->create(path, cmode, initialsz, basepe, chunksizehintp,
useparallel, mpi_info, dispatcher, ncp))) {
useparallel, parameters, dispatcher, ncp))) {
del_from_NCList(ncp); /* oh well */
free_NC(ncp);
} else {
@ -1650,16 +1708,18 @@ For open, we have the following pieces of information to use to determine the di
int
NC_open(const char *path, int cmode,
int basepe, size_t *chunksizehintp,
int useparallel, void* mpi_info,
int useparallel, void* parameters,
int *ncidp)
{
int stat = NC_NOERR;
NC* ncp = NULL;
NC_Dispatch* dispatcher = NULL;
/* Need two pieces of information for now */
int inmemory = ((cmode & NC_INMEMORY) == NC_INMEMORY);
/* Need pieces of information for now to decide model*/
int model = 0;
int isurl = 0;
int version = 0;
int flags = 0;
if(!nc_initialized) {
stat = NC_initialize();
@ -1679,36 +1739,28 @@ NC_open(const char *path, int cmode,
}
#endif
isurl = NC_testurl(path);
if(isurl)
model = NC_urlmodel(path);
else {
version = 0;
model = 0;
/* Look at the file if it exists */
stat = NC_check_file_type(path,useparallel,mpi_info,
&model,&version);
if(stat == NC_NOERR) {
if(model == 0)
if(!inmemory) {
isurl = NC_testurl(path);
if(isurl)
model = NC_urlmodel(path);
}
if(model == 0) {
version = 0;
/* Try to find dataset type */
if(useparallel) flags |= NC_MPIIO;
if(inmemory) flags |= NC_INMEMORY;
stat = NC_check_file_type(path,flags,parameters,&model,&version);
if(stat == NC_NOERR) {
if(model == 0)
return NC_ENOTNC;
} else /* presumably not a netcdf file */
return stat;
}
} else /* presumably not a netcdf file */
return stat;
}
#if 1
if(model == 0) {
fprintf(stderr,"Model != 0\n");
fprintf(stderr,"Model == 0\n");
return NC_ENOTNC;
}
#else
Not longer needed
/* Look to the incoming cmode for hints */
if(model == 0) {
if(cmode & NC_PNETCDF) model = NC_DISPATCH_NC5;
else if(cmode & NC_NETCDF4) model = NC_DISPATCH_NC4;
}
if(model == 0) model = NC_DISPATCH_NC3; /* final default */
#endif
/* Force flag consistentcy */
if(model & NC_DISPATCH_NC4)
@ -1775,7 +1827,7 @@ havetable:
/* Assume open will fill in remaining ncp fields */
stat = dispatcher->open(path, cmode, basepe, chunksizehintp,
useparallel, mpi_info, dispatcher, ncp);
useparallel, parameters, dispatcher, ncp);
if(stat == NC_NOERR) {
if(ncidp) *ncidp = ncp->ext_ncid;
} else {
@ -1811,7 +1863,6 @@ nc__pseudofd(void)
pseudofd = maxfd+1;
#endif
}
return pseudofd++;
}

2
libsrc/ffio.c
View File

@ -612,6 +612,7 @@ int
ffio_create(const char *path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;
@ -709,6 +710,7 @@ int
ffio_open(const char *path,
int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;

186
libsrc/memio.c
View File

@ -22,6 +22,7 @@
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#include "ncdispatch.h"
#include "nc3internal.h"
#undef DEBUG
@ -98,14 +99,20 @@ static long pagesize = 0;
/* Create a new ncio struct to hold info about the file. */
static int
memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEMIO** memiop)
memio_new(const char* path, int ioflags, off_t initialsize, void* memory, ncio** nciopp, NCMEMIO** memiop)
{
int status = NC_NOERR;
ncio* nciop = NULL;
NCMEMIO* memio = NULL;
off_t minsize = initialsize;
int inmemory = (fIsSet(ioflags,NC_INMEMORY));
/* use asserts because this is an internal function */
assert(memiop != NULL && nciopp != NULL);
assert(path != NULL || (memory != NULL && initialsize > 0));
assert(!inmemory || (memory != NULL && initialsize > 0));
if(pagesize == 0) {
#if defined (_WIN32) || defined(_WIN64)
SYSTEM_INFO info;
GetSystemInfo (&info);
@ -132,9 +139,6 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM
nciop->ioflags = ioflags;
*((int*)&nciop->fd) = -1; /* caller will fix */
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
*((ncio_relfunc**)&nciop->rel) = memio_rel;
*((ncio_getfunc**)&nciop->get) = memio_get;
*((ncio_movefunc**)&nciop->move) = memio_move;
@ -142,30 +146,37 @@ memio_new(const char* path, int ioflags, off_t initialsize, ncio** nciopp, NCMEM
*((ncio_filesizefunc**)&nciop->filesize) = memio_filesize;
*((ncio_pad_lengthfunc**)&nciop->pad_length) = memio_pad_length;
*((ncio_closefunc**)&nciop->close) = memio_close;
memio = (NCMEMIO*)calloc(1,sizeof(NCMEMIO));
if(memio == NULL) {status = NC_ENOMEM; goto fail;}
*((void* *)&nciop->pvt) = memio;
*((char**)&nciop->path) = strdup(path);
if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;}
memio->alloc = initialsize;
memio->memory = NULL;
memio->size = 0;
memio->pos = 0;
memio->size = minsize;
memio->memory = NULL;
memio->persist = fIsSet(ioflags,NC_WRITE);
if(nciopp) *nciopp = nciop;
if(memiop && memio) *memiop = memio; else free(memio);
if(nciopp && nciop) *nciopp = nciop;
else {
free((char*)nciop->path);
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
}
if(memiop) *memiop = memio;
else free(memio);
if(inmemory) {
memio->memory = memory;
} else {
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto fail;}
}
done:
return status;
fail:
if(memio != NULL) free(memio);
if(nciop != NULL) {
if(nciop->path != NULL) free((char*)nciop->path);
free(nciop);
@ -173,16 +184,16 @@ fail:
goto done;
}
/* Create a file, and the ncio struct to go with it. This function is
only called from nc__create_mp.
/* Create a file, and the ncio struct to go with it.
path - path of file to create.
ioflags - flags from nc_create
initialsz - From the netcdf man page: "The argument
Iinitialsize sets the initial size of the file at creation time."
initialsize sets the initial size of the file at creation time."
igeto -
igetsz -
sizehintp - the size of a page of data for buffered reads and writes.
parameters - arbitrary data
nciopp - pointer to a pointer that will get location of newly
created and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
@ -191,6 +202,7 @@ int
memio_create(const char* path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;
@ -203,21 +215,12 @@ memio_create(const char* path, int ioflags,
if(path == NULL ||* path == 0)
return NC_EINVAL;
/* For diskless open has, the file must be classic version 1 or 2.*/
if(fIsSet(ioflags,NC_NETCDF4))
return NC_EDISKLESS; /* violates constraints */
status = memio_new(path, ioflags, initialsz, &nciop, &memio);
status = memio_new(path, ioflags, initialsz, NULL, &nciop, &memio);
if(status != NC_NOERR)
return status;
memio->size = 0;
/* malloc memory */
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
if(persist) {
/* Open the file, but make sure we can write it if needed */
/* Open the file just tomake sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
@ -261,14 +264,11 @@ fprintf(stderr,"memio_create: initial memory: %lu/%lu\n",(unsigned long)memio->m
return NC_NOERR;
unwind_open:
if(memio->memory != NULL)
free(memio->memory);
memio_close(nciop,1);
return status;
}
/* This function opens the data file. It is only called from nc.c,
from nc__open_mp and nc_delete_mp.
/* This function opens the data file.
path - path of data file.
ioflags - flags passed into nc_open.
@ -278,6 +278,7 @@ unwind_open:
igetsz - the size in bytes of initial page get (a.k.a. extent). Not
ever used in the library.
sizehintp - the size of a page of data for buffered reads and writes.
parameters - arbitrary data
nciopp - pointer to pointer that will get address of newly created
and inited ncio struct.
mempp - pointer to pointer to the initial memory read.
@ -286,85 +287,91 @@ int
memio_open(const char* path,
int ioflags,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
ncio* *nciopp, void** const mempp)
{
ncio* nciop = NULL;
int fd;
int status;
int fd = -1;
int status = NC_NOERR;
int persist = (fIsSet(ioflags,NC_WRITE)?1:0);
int oflags;
int inmemory = (fIsSet(ioflags,NC_INMEMORY));
int oflags = 0;
NCMEMIO* memio = NULL;
size_t sizehint;
off_t filesize;
size_t sizehint = 0;
off_t filesize = 0;
off_t red = 0;
char* pos = NULL;
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
if(path == NULL ||* path == 0)
return EINVAL;
if(path == NULL || strlen(path) == 0)
return NC_EINVAL;
assert(sizehintp != NULL);
sizehint = *sizehintp;
sizehint = *sizehintp;
/* Open the file, but make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
if(inmemory) {
filesize = meminfo->size;
} else {
/* Open the file,and make sure we can write it if needed */
oflags = (persist ? O_RDWR : O_RDONLY);
#ifdef O_BINARY
fSet(oflags, O_BINARY);
fSet(oflags, O_BINARY);
#endif
oflags |= O_EXCL;
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, OPENMODE);
fd = open(path, oflags, OPENMODE);
#endif
#ifdef DEBUG
if(fd < 0) {
fprintf(stderr,"open failed: file=%s err=",path);
perror("");
}
if(fd < 0) {
fprintf(stderr,"open failed: file=%s err=",path);
perror("");
}
#endif
if(fd < 0) {status = errno; goto unwind_open;}
if(fd < 0) {status = errno; goto unwind_open;}
/* get current filesize = max(|file|,initialize)*/
filesize = lseek(fd,0,SEEK_END);
if(filesize < 0) {status = errno; goto unwind_open;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
if(filesize < (off_t)sizehint)
filesize = (off_t)sizehint;
/* get current filesize = max(|file|,initialize)*/
filesize = lseek(fd,0,SEEK_END);
if(filesize < 0) {status = errno; goto unwind_open;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
if(filesize < (off_t)sizehint)
filesize = (off_t)sizehint;
}
status = memio_new(path, ioflags, filesize, &nciop, &memio);
if(inmemory)
status = memio_new(path, ioflags, filesize, meminfo->memory, &nciop, &memio);
else
status = memio_new(path, ioflags, filesize, NULL, &nciop, &memio);
if(status != NC_NOERR) {
if(fd >= 0)
close(fd);
if(fd >= 0)
close(fd);
return status;
}
memio->size = filesize;
memio->memory = (char*)malloc(memio->alloc);
if(memio->memory == NULL) {status = NC_ENOMEM; goto unwind_open;}
#ifdef DEBUG
fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->memory,(unsigned long)memio->alloc);
#endif
/* Read the file into the memio memory */
/* We need to do multiple reads because there is no
guarantee that the amount read will be the full amount */
{
off_t red = memio->size;
char* pos = memio->memory;
while(red > 0) {
ssize_t count = read(fd, pos, red);
if(count < 0)
{status = errno; goto unwind_open;}
if(count == 0)
{status = NC_ENOTNC; goto unwind_open;}
red -= count;
pos += count;
}
if(!inmemory) {
/* Read the file into the memio memory */
/* We need to do multiple reads because there is no
guarantee that the amount read will be the full amount */
red = memio->size;
pos = memio->memory;
while(red > 0) {
ssize_t count = read(fd, pos, red);
if(count < 0) {status = errno; goto unwind_open;}
if(count == 0) {status = NC_ENOTNC; goto unwind_open;}
red -= count;
pos += count;
}
(void)close(fd);
}
(void)close(fd); /* until memio_close() */
/* Use half the filesize as the blocksize */
sizehint = filesize/2;
/* Use half the filesize as the blocksize ; why? */
sizehint = filesize/2;
fd = nc__pseudofd();
*((int* )&nciop->fd) = fd;
@ -379,8 +386,8 @@ fprintf(stderr,"memio_open: initial memory: %lu/%lu\n",(unsigned long)memio->mem
goto unwind_open;
}
*sizehintp = sizehint;
*nciopp = nciop;
if(sizehintp) *sizehintp = sizehint;
if(nciopp) *nciopp = nciop; else {ncio_close(nciop,0);}
return NC_NOERR;
unwind_open:
@ -390,7 +397,6 @@ unwind_open:
return status;
}
/*
* Get file size in bytes.
*/
@ -464,17 +470,19 @@ memio_close(ncio* nciop, int doUnlink)
int status = NC_NOERR;
NCMEMIO* memio;
int fd = -1;
int inmemory = (fIsSet(nciop->ioflags,NC_INMEMORY));
if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR;
memio = (NCMEMIO*)nciop->pvt;
assert(memio != NULL);
/* See if the user wants the contents persisted to a file */
if(memio->persist) {
if(!inmemory && memio->persist) {
/* Try to open the file for writing */
int oflags = O_WRONLY|O_CREAT|O_TRUNC;
#ifdef O_BINARY
fSet(oflags, O_BINARY);
fisSet(oflags, O_BINARY);
#endif
fd = open(nciop->path, oflags, OPENMODE);
if(fd >= 0) {
@ -496,7 +504,7 @@ memio_close(ncio* nciop, int doUnlink)
}
done:
if(memio->memory != NULL)
if(!inmemory && memio->memory != NULL)
free(memio->memory);
/* do cleanup */
if(fd >= 0) (void)close(fd);

2
libsrc/mmapio.c
View File

@ -204,6 +204,7 @@ int
mmapio_create(const char* path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;
@ -312,6 +313,7 @@ int
mmapio_open(const char* path,
int ioflags,
off_t igeto, size_t igetsz, size_t* sizehintp,
void* parameters,
ncio* *nciopp, void** const mempp)
{
ncio* nciop;

139
libsrc/nc3internal.c
View File

@ -16,8 +16,6 @@
#endif
#include "nc3internal.h"
#include "ncdispatch.h"
#include "nc3dispatch.h"
#include "rnd.h"
#include "ncx.h"
@ -969,7 +967,7 @@ NC3_create(const char *path, int ioflags,
assert(nc3->xsz == ncx_len_NC(nc3,sizeof_off_t));
status = ncio_create(path, ioflags, initialsz,
0, nc3->xsz, &nc3->chunk,
0, nc3->xsz, &nc3->chunk, NULL,
&nc3->nciop, &xp);
if(status != NC_NOERR)
{
@ -1066,17 +1064,17 @@ NC3_open(const char * path, int ioflags,
if (status = NC_init_pe(nc3, basepe)) {
return status;
}
#else
#else
/*
* !_CRAYMPP, only pe 0 is valid
*/
if(basepe != 0) {
if(nc3) free(nc3);
return NC_EINVAL;
if(nc3) free(nc3);
return NC_EINVAL;
}
#endif
status = ncio_open(path, ioflags, 0, 0, &nc3->chunk, &nc3->nciop, 0);
status = ncio_open(path, ioflags, 0, 0, &nc3->chunk, parameters,
&nc3->nciop, NULL);
if(status)
goto unwind_alloc;
@ -1140,6 +1138,52 @@ NC3__enddef(int ncid,
return (NC_endef(nc3, h_minfree, v_align, v_minfree, r_align));
}
/*
* In data mode, same as ncclose.
* In define mode, restore previous definition.
* In create, remove the file.
*/
int
NC3_abort(int ncid)
{
int status;
NC *nc;
NC3_INFO* nc3;
int doUnlink = 0;
status = NC_check_id(ncid, &nc);
if(status != NC_NOERR)
return status;
nc3 = NC3_DATA(nc);
doUnlink = NC_IsNew(nc3);
if(nc3->old != NULL)
{
/* a plain redef, not a create */
assert(!NC_IsNew(nc3));
assert(fIsSet(nc3->flags, NC_INDEF));
free_NC3INFO(nc3->old);
nc3->old = NULL;
fClr(nc3->flags, NC_INDEF);
}
else if(!NC_readonly(nc3))
{
status = NC_sync(nc3);
if(status != NC_NOERR)
return status;
}
(void) ncio_close(nc3->nciop, doUnlink);
nc3->nciop = NULL;
free_NC3INFO(nc3);
if(nc)
NC3_DATA_SET(nc,NULL);
return NC_NOERR;
}
int
NC3_close(int ncid)
@ -1199,54 +1243,6 @@ NC3_close(int ncid)
return status;
}
/*
* In data mode, same as ncclose.
* In define mode, restore previous definition.
* In create, remove the file.
*/
int
NC3_abort(int ncid)
{
int status;
NC *nc;
NC3_INFO* nc3;
int doUnlink = 0;
status = NC_check_id(ncid, &nc);
if(status != NC_NOERR)
return status;
nc3 = NC3_DATA(nc);
doUnlink = NC_IsNew(nc3);
if(nc3->old != NULL)
{
/* a plain redef, not a create */
assert(!NC_IsNew(nc3));
assert(fIsSet(nc3->flags, NC_INDEF));
free_NC3INFO(nc3->old);
nc3->old = NULL;
fClr(nc3->flags, NC_INDEF);
}
else if(!NC_readonly(nc3))
{
status = NC_sync(nc3);
if(status != NC_NOERR)
return status;
}
(void) ncio_close(nc3->nciop, doUnlink);
nc3->nciop = NULL;
free_NC3INFO(nc3);
if(nc)
NC3_DATA_SET(nc,NULL);
return NC_NOERR;
}
int
NC3_redef(int ncid)
{
@ -1587,6 +1583,37 @@ NC3_inq_type(int ncid, nc_type typeid, char *name, size_t *size)
return NC_NOERR;
}
/**************************************************/
#if 0
int
NC3_set_content(int ncid, size_t size, void* memory)
{
int status = NC_NOERR;
NC *nc;
NC3_INFO* nc3;
status = NC_check_id(ncid, &nc);
if(status != NC_NOERR)
return status;
nc3 = NC3_DATA(nc);
#ifdef USE_DISKLESS
fClr(nc3->flags, NC_CREAT);
status = memio_set_content(nc3->nciop, size, memory);
if(status != NC_NOERR) goto done;
status = nc_get_NC(nc3);
if(status != NC_NOERR) goto done;
#else
status = NC_EDISKLESS;
#endif
done:
return status;
}
#endif
/**************************************************/
int
nc_delete_mp(const char * path, int basepe)
{

34
libsrc/ncio.c
View File

@ -16,49 +16,51 @@
*/
/* Define known ncio packages */
extern int posixio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int posixio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
extern int posixio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int posixio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#ifdef USE_FFIO
extern int ffio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int ffio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
extern int ffio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int ffio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
#ifdef USE_DISKLESS
# ifdef USE_MMAP
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
extern int mmapio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int mmapio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
# endif
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,ncio**,void** const);
extern int memio_open(const char*,int,off_t,size_t,size_t*,ncio**,void** const);
extern int memio_create(const char*,int,size_t,off_t,size_t,size_t*,void*,ncio**,void** const);
extern int memio_open(const char*,int,off_t,size_t,size_t*,void*,ncio**,void** const);
#endif
int
ncio_create(const char *path, int ioflags, size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio** iopp, void** const mempp)
{
#ifdef USE_DISKLESS
if(fIsSet(ioflags,NC_DISKLESS)) {
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP))
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
return mmapio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
else
# endif /*USE_MMAP*/
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
return memio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
#endif
#ifdef USE_FFIO
return ffio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
return ffio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#else
return posixio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,iopp,mempp);
return posixio_create(path,ioflags,initialsz,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#endif
}
int
ncio_open(const char *path, int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio** iopp, void** const mempp)
{
/* Diskless open has the following constraints:
@ -68,16 +70,16 @@ ncio_open(const char *path, int ioflags,
if(fIsSet(ioflags,NC_DISKLESS)) {
# ifdef USE_MMAP
if(fIsSet(ioflags,NC_MMAP))
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
return mmapio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
else
# endif /*USE_MMAP*/
return memio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
return memio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
}
#endif
#ifdef USE_FFIO
return ffio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
return ffio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#else
return posixio_open(path,ioflags,igeto,igetsz,sizehintp,iopp,mempp);
return posixio_open(path,ioflags,igeto,igetsz,sizehintp,parameters,iopp,mempp);
#endif
}

4
libsrc/ncio.h
View File

@ -155,10 +155,12 @@ extern int ncio_close(ncio* const, int);
extern int ncio_create(const char *path, int ioflags, size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters, /* new */
ncio** nciopp, void** const mempp);
extern int ncio_open(const char *path, int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters, /* new */
ncio** nciopp, void** const mempp);
/* With the advent of diskless io, we need to provide
@ -170,6 +172,4 @@ extern int ncio_open(const char *path, int ioflags,
the new package when appropriate.
*/
#endif /* _NCIO_H_ */

2
libsrc/posixio.c
View File

@ -1516,6 +1516,7 @@ int
posixio_create(const char *path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;
@ -1659,6 +1660,7 @@ int
posixio_open(const char *path,
int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;

2
libsrc/winceio.c
View File

@ -475,6 +475,7 @@ int
ncio_create(const char *path, int ioflags,
size_t initialsz,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;
@ -576,6 +577,7 @@ int
ncio_open(const char *path,
int ioflags,
off_t igeto, size_t igetsz, size_t *sizehintp,
void* parameters,
ncio **nciopp, void **const igetvpp)
{
ncio *nciop;

238
libsrc4/nc4file.c
View File

@ -12,18 +12,20 @@ COPYRIGHT file for copying and redistribution conditions.
#include "config.h"
#include <errno.h> /* netcdf functions sometimes return system errors */
#include "nc.h"
#include "nc4internal.h"
#include "nc4dispatch.h"
#include "H5DSpublic.h"
/* must be after nc4internal.h */
#include <H5DSpublic.h>
#ifdef USE_HDF4
#include <mfhdf.h>
#endif
#if 0 /*def USE_PNETCDF*/
#include <pnetcdf.h>
#ifdef USE_DISKLESS
#include <hdf5_hl.h>
#endif
/* This is to track opened HDF5 objects to make sure they are
@ -43,6 +45,11 @@ extern int num_spaces;
#define DIMENSION_LIST "DIMENSION_LIST"
#define NAME "NAME"
/* Define the illegal mode flags */
#define ILLEGAL_OPEN_FLAGS (NC_MMAP|NC_64BIT_OFFSET)
#define ILLEGAL_CREATE_FLAGS (NC_NOWRITE|NC_MMAP|NC_INMEMORY|NC_64BIT_OFFSET|NC_PNETCDF)
/*! Struct to track information about objects in a group, for nc4_rec_read_metadata()
\internal
@ -183,16 +190,32 @@ nc4typelen(nc_type type)
#define NC_HDF5_FILE 1
#define NC_HDF4_FILE 2
static int
nc_check_for_hdf(const char *path, int use_parallel, MPI_Comm comm, MPI_Info info,
int *hdf_file)
nc_check_for_hdf(const char *path, int flags, void* parameters, int *hdf_file)
{
char blob[MAGIC_NUMBER_LEN];
#ifdef USE_PARALLEL
int use_parallel = ((flags & NC_MPIIO) == NC_MPIIO);
NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
#endif
int inmemory = ((flags & NC_INMEMORY) == NC_INMEMORY);
#ifdef USE_DISKLESS
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
#endif
assert(hdf_file && path);
#ifdef USE_PARALLEL
if(use_parallel) {
comm = mpiinfo->comm;
info = mpiinfo->info;
}
#endif
assert(hdf_file);
LOG((3, "%s: path %s", __func__, path));
/* HDF5 function handles possible user block at beginning of file */
if(H5Fis_hdf5(path))
if(!inmemory && H5Fis_hdf5(path))
{
*hdf_file = NC_HDF5_FILE;
} else {
@ -200,7 +223,7 @@ nc_check_for_hdf(const char *path, int use_parallel, MPI_Comm comm, MPI_Info inf
/* Get the 4-byte blob from the beginning of the file. Don't use posix
* for parallel, use the MPI functions instead. */
#ifdef USE_PARALLEL
if (use_parallel)
if (!inmemory && use_parallel)
{
MPI_File fh;
MPI_Status status;
@ -216,7 +239,7 @@ nc_check_for_hdf(const char *path, int use_parallel, MPI_Comm comm, MPI_Info inf
}
else
#endif /* USE_PARALLEL */
{
if(!inmemory) {
FILE *fp;
if (!(fp = fopen(path, "r")) ||
fread(blob, MAGIC_NUMBER_LEN, 1, fp) != 1) {
@ -225,11 +248,17 @@ nc_check_for_hdf(const char *path, int use_parallel, MPI_Comm comm, MPI_Info inf
return errno;
}
fclose(fp);
} else { /*inmemory*/
if(meminfo->size < MAGIC_NUMBER_LEN)
return NC_ENOTNC;
memcpy(blob,meminfo->memory,MAGIC_NUMBER_LEN);
}
/* Check for HDF4. */
if (!strncmp(blob, "\016\003\023\001", MAGIC_NUMBER_LEN))
if (memcmp(blob, "\016\003\023\001", 4)==0)
*hdf_file = NC_HDF4_FILE;
else if (memcmp(blob, "HDF", 3)==0)
*hdf_file = NC_HDF5_FILE;
else
*hdf_file = 0;
}
@ -433,7 +462,7 @@ Create a netCDF-4/HDF5 file.
\param basepe Ignored by this function.
\param chunksizehintp Ignored by this function.
\param use_parallel 0 for sequential, non-zero for parallel I/O.
\param mpidata pointer to struct holdind data for parallel I/O
\param parameters pointer to struct holding extra data (e.g. for parallel I/O)
layer. Ignored if NULL.
\param dispatch Pointer to the dispatch table for this file.
\param nc_file Pointer to an instance of NC.
@ -441,7 +470,7 @@ layer. Ignored if NULL.
*/
int
NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
size_t *chunksizehintp, int use_parallel, void *mpidata,
size_t *chunksizehintp, int use_parallel, void *parameters,
NC_Dispatch *dispatch, NC* nc_file)
{
MPI_Comm comm = MPI_COMM_WORLD;
@ -454,10 +483,10 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
__func__, path, cmode, comm, info));
#ifdef USE_PARALLEL
if (mpidata)
if (parameters)
{
comm = ((NC_MPI_INFO *)mpidata)->comm;
info = ((NC_MPI_INFO *)mpidata)->info;
comm = ((NC_MPI_INFO *)parameters)->comm;
info = ((NC_MPI_INFO *)parameters)->info;
}
#endif /* USE_PARALLEL */
@ -471,16 +500,15 @@ NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
}
/* Check the cmode for validity. */
if (cmode & ~(NC_NOCLOBBER | NC_64BIT_OFFSET
| NC_NETCDF4 | NC_CLASSIC_MODEL
| NC_SHARE | NC_MPIIO | NC_MPIPOSIX | NC_LOCK | NC_PNETCDF
| NC_DISKLESS
| NC_WRITE /* to support diskless persistence */
)
|| (cmode & NC_MPIIO && cmode & NC_MPIPOSIX)
|| (cmode & NC_64BIT_OFFSET && cmode & NC_NETCDF4)
|| (cmode & (NC_MPIIO | NC_MPIPOSIX) && cmode & NC_DISKLESS)
)
if((cmode & ILLEGAL_CREATE_FLAGS) != 0)
return NC_EINVAL;
/* Cannot have both */
if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
return NC_EINVAL;
/* Currently no parallel diskless io */
if((cmode & (NC_MPIIO | NC_MPIPOSIX)) && (cmode & NC_DISKLESS))
return NC_EINVAL;
#ifndef USE_PARALLEL_POSIX
@ -2154,24 +2182,25 @@ exit:
* ncfunc.c in nc_open, but here the netCDF-4 part of opening a file
* is handled. */
static int
nc4_open_file(const char *path, int mode, MPI_Comm comm,
MPI_Info info, NC *nc)
nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
{
hid_t fapl_id = H5P_DEFAULT;
unsigned flags = (mode & NC_WRITE) ?
H5F_ACC_RDWR : H5F_ACC_RDONLY;
int retval;
NC_HDF5_FILE_INFO_T* nc4_info = NULL;
int inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
#ifdef USE_DISKLESS
NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
#endif
#ifdef USE_PARALLEL
NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#endif /* !USE_PARALLEL */
LOG((3, "%s: path %s mode %d", __func__, path, mode));
assert(path && nc);
/* Stop diskless open in its tracks */
if(mode & NC_DISKLESS)
return NC_EDISKLESS;
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_nc4f_list_add(nc, path, mode)))
@ -2179,7 +2208,7 @@ nc4_open_file(const char *path, int mode, MPI_Comm comm,
nc4_info = NC4_DATA(nc);
assert(nc4_info && nc4_info->root_grp);
/* Need this access plist to control how HDF5 handles open onjects
/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to
* fail if there are any open objects in the file. */
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
@ -2204,14 +2233,14 @@ nc4_open_file(const char *path, int mode, MPI_Comm comm,
if (mode & NC_MPIIO) /* MPI/IO */
{
LOG((4, "opening parallel file with MPI/IO"));
if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
if (H5Pset_fapl_mpio(fapl_id, mpiinfo->comm, mpiinfo->info) < 0)
BAIL(NC_EPARINIT);
}
#ifdef USE_PARALLEL_POSIX
else /* MPI/POSIX */
{
LOG((4, "opening parallel file with MPI/posix"));
if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
if (H5Pset_fapl_mpiposix(fapl_id, mpiinfo->comm, 0) < 0)
BAIL(NC_EPARINIT);
}
#else /* USE_PARALLEL_POSIX */
@ -2224,19 +2253,19 @@ nc4_open_file(const char *path, int mode, MPI_Comm comm,
#endif /* USE_PARALLEL_POSIX */
/* Keep copies of the MPI Comm & Info objects */
if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm))
if (MPI_SUCCESS != MPI_Comm_dup(mpiinfo->comm, &nc4_info->comm))
BAIL(NC_EMPI);
comm_duped++;
if (MPI_INFO_NULL != info)
if (MPI_INFO_NULL != mpiinfo->info)
{
if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info))
if (MPI_SUCCESS != MPI_Info_dup(mpiinfo->info, &nc4_info->info))
BAIL(NC_EMPI);
info_duped++;
}
else
{
/* No dup, just copy it. */
nc4_info->info = info;
nc4_info->info = mpiinfo->info;
}
}
#else /* only set cache for non-parallel. */
@ -2250,7 +2279,13 @@ nc4_open_file(const char *path, int mode, MPI_Comm comm,
/* The NetCDF-3.x prototype contains an mode option NC_SHARE for
multiple processes accessing the dataset concurrently. As there
is no HDF5 equivalent, NC_SHARE is treated as NC_NOWRITE. */
if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
if(inmemory) {
if((nc4_info->hdfid = H5LTopen_file_image(meminfo->memory,meminfo->size,
H5LT_FILE_IMAGE_DONT_COPY|H5LT_FILE_IMAGE_DONT_RELEASE
)) < 0)
BAIL(NC_EHDFERR);
nc4_info->no_write = NC_TRUE;
} else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
/* Does the mode specify that this file is read-only? */
@ -2670,24 +2705,25 @@ nc4_open_hdf4_file(const char *path, int mode, NC *nc)
int
NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
int use_parallel, void *mpidata, NC_Dispatch *dispatch, NC *nc_file)
int use_parallel, void *parameters, NC_Dispatch *dispatch, NC *nc_file)
{
int hdf_file = 0;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
int res;
int hdf_file = 0;
#ifdef USE_PARALLEL
NC_MPI_INFO mpidfalt = {MPI_COMM_WORLD, MPI_INFO_NULL};
#endif
#if defined USE_PARALLEL || defined USE_HDF4
int inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY);
#endif
assert(nc_file && path);
LOG((1, "%s: path %s mode %d comm %d info %d",
__func__, path, mode, comm, info));
LOG((1, "%s: path %s mode %d params %x",
__func__, path, mode, parameters));
#ifdef USE_PARALLEL
if (mpidata)
{
comm = ((NC_MPI_INFO *)mpidata)->comm;
info = ((NC_MPI_INFO *)mpidata)->info;
}
if (!inmemory && use_parallel && parameters == NULL)
parameters = &mpidfalt;
#endif /* USE_PARALLEL */
/* If this is our first file, turn off HDF5 error messages. */
@ -2699,12 +2735,13 @@ NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
virgin = 0;
}
/* Check the mode for validity. First make sure only certain bits
* are turned on. Also MPI I/O and MPI POSIX cannot both be
* selected at once. */
if (mode & ~(NC_WRITE | NC_SHARE | NC_MPIIO | NC_MPIPOSIX |
NC_PNETCDF | NC_NOCLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL) ||
(mode & NC_MPIIO && mode & NC_MPIPOSIX))
/* Check the mode for validity */
if((mode & ILLEGAL_OPEN_FLAGS) != 0)
return NC_EINVAL;
/* Cannot have both */
if((mode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
return NC_EINVAL;
#ifndef USE_PARALLEL_POSIX
@ -2718,62 +2755,22 @@ NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
}
#endif /* USE_PARALLEL_POSIX */
/* Figure out if this is a hdf4 or hdf5 file. */
if ((res = nc_check_for_hdf(path, use_parallel, parameters, &hdf_file)))
return res;
/* Depending on the type of file, open it. */
#if 0 /*def USE_PNETCDF*/
if(mode & NC_PNETCDF) {
/* this is not really an hdf file */
int pnetcdf_nvars, i;
NC_HDF5_FILE_INFO_T* nc4_info;
/* Create the fake nc4_info data */
res = nc4_nc4f_list_add(nc_file, path, mode);
nc4_info = NC4_DATA(nc_file);
assert(nc4_info);
res = ncmpi_open(comm, path, mode, info, &(nc_file->int_ncid));
nc4_info->pnetcdf_file++;
/* Default to independent access, like netCDF-4/HDF5 files. */
if (!res)
res = ncmpi_begin_indep_data(nc_file->int_ncid);
/* I need to keep track of the ndims of each var to translate
* start, count, and stride arrays to MPI_Offset type. */
if (!res)
{
res = ncmpi_inq_nvars(nc_file->int_ncid, &pnetcdf_nvars);
for (i = 0; i < pnetcdf_nvars; i++)
res = ncmpi_inq_varndims(nc_file->int_ncid, i,
&(nc4_info->pnetcdf_ndims[i]));
}
} else
#endif
{
/* Figure out if this is a hdf4 or hdf5 file. */
if ((res = nc_check_for_hdf(path, use_parallel, comm, info, &hdf_file)))
return res;
if (hdf_file == NC_HDF5_FILE)
{
nc_file->int_ncid = nc_file->ext_ncid;
res = nc4_open_file(path, mode, comm, info, nc_file);
}
nc_file->int_ncid = nc_file->ext_ncid;
if (hdf_file == NC_HDF5_FILE)
res = nc4_open_file(path, mode, parameters, nc_file);
#ifdef USE_HDF4
else if (hdf_file == NC_HDF4_FILE)
{
nc_file->int_ncid = nc_file->ext_ncid;
res = nc4_open_hdf4_file(path, mode, nc_file);
}
else if (hdf_file == NC_HDF4_FILE && inmemory)
return NC_EDISKLESS;
else if (hdf_file == NC_HDF4_FILE)
res = nc4_open_hdf4_file(path, mode, nc_file);
#endif /* USE_HDF4 */
else /* netcdf */
{
else
assert(0); /* should never happen */
}
}
return res;
}
@ -3207,6 +3204,35 @@ NC4_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp)
return NC_NOERR;
}
#if 0
int
NC4_set_content(int ncid, size_t size, void* memory)
{
int retval = NC_NOERR;
herr_t herr;
NC *nc;
NC_HDF5_FILE_INFO_T *h5;
NC_GRP_INFO_T *grp;
LOG((4,"%s: ncid 0x%x size %ld memory 0x%x", __func__, ncid, size, memory));
/* Find file metadata. */
if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
return retval;
assert(h5 && grp && nc);
#ifdef USE_DISKLESS
herr = H5Pset_file_image(h5->hdfid,memory,size);
if(herr)
BAIL(NC_EHDFERR);
#else
retval = NC_EDISKLESS;
#endif
done:
return retval;
}
#endif
/* This function will do the enddef stuff for a netcdf-4 file. */
int

12
ncdap_test/testauth.sh
View File

@ -104,7 +104,7 @@ cd $srcdir
srcdir=`pwd`
cd ${builddir}
function createrc {
createrc() {
xf
RCP="$1" ; shift
unset NOPWD
@ -143,7 +143,7 @@ function createrc {
fi
}
function createnetrc {
createnetrc() {
xf
NCP="$1" ; shift
unset NOPWD
@ -180,14 +180,14 @@ function createnetrc {
fi
}
function reset {
reset() {
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC $OUTPUT ; do
rm -f ${f}
done
unset DAPRCFILE
}
function restore {
restore() {
reset
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do
if test -f ${f}.save ; then
@ -197,7 +197,7 @@ function restore {
done
}
function save {
save() {
for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do
if test -f $f ; then
if test -f ${f}.save ; then
@ -210,7 +210,7 @@ function save {
done
}
function show {
show() {
if test "x$SHOW" = x1 ; then cat $OUTPUT; fi
if test "x$OUTPUT" != "x"; then rm -f $OUTPUT; fi
}

6
ncdump/Makefile.am
View File

@ -37,6 +37,10 @@ if LARGE_FILE_TESTS
TESTS += tst_iter.sh
endif
if BUILD_DISKLESS
TESTS += tst_inmemory.sh
endif
if USE_NETCDF4
# NetCDF-4 has some extra tests.
check_PROGRAMS += tst_create_files tst_h_rdc0 tst_group_data \
@ -128,7 +132,7 @@ tst_iter.sh tst_mud.sh ref_tst_mud4.cdl ref_tst_mud4-bc.cdl \
ref_tst_mud4_chars.cdl \
ref_tst_ncf213.cdl tst_h_scalar.sh \
tst_formatx3.sh tst_formatx4.sh \
CMakeLists.txt XGetopt.c tst_bom.sh
CMakeLists.txt XGetopt.c tst_bom.sh tst_inmemory.sh
# CDL files and Expected results
SUBDIRS=cdl expected

96
ncdump/ncdump.c
View File

@ -8,9 +8,15 @@ Research/Unidata. See \ref copyright file for more info. */
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
#ifndef _WIN32
#ifdef _MSC_VER /* Microsoft Compilers */
#include <io.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef _MSC_VER
#define snprintf _snprintf
@ -37,6 +43,7 @@ int optind;
#include "indent.h"
#include "isnan.h"
#include "cdl.h"
#include "netcdf_mem.h"
#define int64_t long long
#define uint64_t unsigned long long
@ -277,6 +284,67 @@ kind_string_extended(int kind, int mode)
return text;
}
#ifdef USE_DISKLESS
static int
fileopen(const char* path, void** memp, size_t* sizep)
{
int status = NC_NOERR;
int fd = -1;
int oflags = 0;
size_t size = 0;
void* mem = NULL;
off_t red = 0;
char* pos = NULL;
/* Open the file, but make sure we can write it if needed */
oflags = O_RDONLY;
#ifdef O_BINARY
oflags |= O_BINARY;
#endif
oflags |= O_EXCL;
#ifdef vms
fd = open(path, oflags, 0, "ctx=stm");
#else
fd = open(path, oflags, 0666);
#endif
#ifdef DEBUG
if(fd < 0) {
fprintf(stderr,"open failed: file=%s err=",path);
status = errno;
}
#endif
if(fd < 0) {status = errno; goto done;}
/* get current filesize = max(|file|,initialize)*/
size = lseek(fd,0,SEEK_END);
if(size < 0) {status = errno; goto done;}
/* move pointer back to beginning of file */
(void)lseek(fd,0,SEEK_SET);
mem = malloc(size);
if(mem == NULL) {status = NC_ENOMEM; goto done;}
/* Read the file into memory */
/* We need to do multiple reads because there is no
guarantee that the amount read will be the full amount */
red = size;
pos = (char*)mem;
while(red > 0) {
ssize_t count = read(fd, pos, red);
if(count < 0) {status = errno; goto done;}
if(count == 0) {status = NC_ENOTNC; goto done;}
red -= count;
pos += count;
}
done:
if(fd >= 0) (void)close(fd);
if(status != NC_NOERR && mem != NULL)
free(mem);
else {
if(sizep) *sizep = size;
if(memp) *memp = mem;
}
return status;
}
#endif
/*
* Emit initial line of output for NcML
@ -288,7 +356,6 @@ pr_initx(int ncid, const char *path)
path);
}
/*
* Print attribute string, for text attributes.
*/
@ -1927,7 +1994,7 @@ main(int argc, char *argv[])
exit(EXIT_SUCCESS);
}
while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwK")) != EOF)
while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwKX:")) != EOF)
switch(c) {
case 'h': /* dump header only, no data */
formatting_specs.header_only = true;
@ -2015,6 +2082,16 @@ main(int argc, char *argv[])
case 'w': /* with client-side cache for DAP URLs */
formatting_specs.with_cache = true;
break;
case 'X': /* special options */
switch (tolower((int)optarg[0])) {
case 'm':
formatting_specs.xopt_inmemory = 1;
break;
default:
error("invalid value for -X option: %s", optarg);
break;
}
break;
case '?':
usage();
exit(EXIT_FAILURE);
@ -2057,7 +2134,16 @@ main(int argc, char *argv[])
/* else fall thru and treat like a file path */
}
#endif /*USE_DAP*/
nc_status = nc_open(path, NC_NOWRITE, &ncid);
#ifdef USE_DISKLESS
if(formatting_specs.xopt_inmemory) {
size_t size = 0;
void* mem = NULL;
nc_status = fileopen(path,&mem,&size);
if(nc_status == NC_NOERR)
nc_status = nc_open_mem(path,NC_DISKLESS|NC_INMEMORY,size,mem,&ncid);
} else
#endif
nc_status = nc_open(path, NC_NOWRITE, &ncid);
if (nc_status != NC_NOERR) {
error("%s: %s", path, nc_strerror(nc_status));
}
@ -2100,4 +2186,6 @@ main(int argc, char *argv[])
}
exit(EXIT_SUCCESS);
}
END_OF_MAIN();

2
ncdump/ncdump.h
View File

@ -75,6 +75,8 @@ typedef struct { /* specification for how to format dump */
* on command line.
*/
int nc_mode; /* mode as reported by inq_format_extended */
int xopt_inmemory; /* Use in-memory option; testing only */
} fspec_t;
#endif /*_NCDUMP_H_ */

57
ncdump/tst_inmemory.sh Executable file
View File

@ -0,0 +1,57 @@
#!/bin/sh
verbose=1
set -e
if test "x$builddir" = "x"; then builddir=`pwd`; fi
if test "x$srcdir" = "x"; then srcdir=`dirname $0`; fi
# Make buildir absolute
cd $builddir
builddir=`pwd`
# Make srcdir be absolute
cd $srcdir
srcdir=`pwd`
cd $builddir
# Setup
PASS=1
# Define the .cdl files to test
CLASSIC="small ref_tst_nans ref_tst_utf8"
EXTENDED="ref_nc_test_netcdf4 ref_tst_comp ref_tst_opaque_data"
rm -fr ./results
mkdir ./results
# Dump classic files two ways and compare
dotest() {
K=$1
for f in $2 ; do
echo "Testing ${f}"
${builddir}/../ncgen/ncgen -$K -o ./results/${f}.nc ${srcdir}/${f}.cdl
./ncdump ./results/${f}.nc >./results/${f}.cdl
./ncdump -Xm ./results/${f}.nc >./results/${f}.cdx
diff -w ./results/${f}.cdl ./results/${f}.cdx >& ./results/${f}.diff
if test -s ./results/${f}.diff ; then
echo "***FAIL: $f"
PASS=0
fi
done
}
dotest 3 "$CLASSIC"
dotest 4 "$EXTENDED"
# Cleanup
rm -fr results
if test "x$PASS" = x1 ; then
echo "*** PASS all tests"
CODE=0
else
CODE=1
fi
exit $CODE