mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

moved hdf5 var code to hdf5var.c

Browse Source
This commit is contained in:
Ed Hartnett 2018-07-19 07:23:03 -06:00
commit 3c0abc3d28
25 changed files with 3861 additions and 2742 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CMakeLists.txt
View File

@ -381,14 +381,6 @@ OPTION(ENABLE_MMAP "Use MMAP." ON)
# Option to use examples.
OPTION(ENABLE_EXAMPLES "Build Examples" ON)
# Option Logging, only valid for netcdf4.
OPTION(ENABLE_LOGGING "Enable Logging." OFF)
IF(ENABLE_LOGGING)
ADD_DEFINITIONS(-DLOGGING)
SET(LOGGING ON)
ENDIF()
# Option to automatically build netcdf-fortran.
IF(NOT MSVC)
OPTION(ENABLE_REMOTE_FORTRAN_BOOTSTRAP "Download and build netcdf-fortran automatically (EXPERIMENTAL)." OFF)
@ -533,10 +525,12 @@ IF(ENABLE_LOGGING)
ADD_DEFINITIONS(-DLOGGING)
ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL)
SET(LOGGING ON)
SET(ENABLE_SET_LOG_LEVEL ON)
ENDIF()
OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON)
IF(ENABLE_NETCDF_$ AND ((NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING))
IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL)
SET(ENABLE_SET_LOG_LEVEL ON)
ENDIF()
# Option to allow for strict null file padding.

3
config.h.cmake.in
View File

@ -367,6 +367,9 @@ are set when opening a binary file on Windows. */
/* If true, turn on logging. */
#cmakedefine LOGGING 1
/* If true, define nc_set_log_level. */
#cmakedefine ENABLE_SET_LOG_LEVEL 1
/* max size of the default per-var chunk cache. */
#cmakedefine MAX_DEFAULT_CACHE_SIZE ${MAX_DEFAULT_CACHE_SIZE}

6
include/Makefile.am
View File

@ -17,9 +17,9 @@ include_HEADERS += netcdf_mem.h
noinst_HEADERS = nc_logging.h nc_tests.h fbits.h nc.h nclist.h \
ncuri.h ncutf8.h ncdispatch.h ncdimscale.h netcdf_f.h err_macros.h \
ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h \
ncauth.h ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h \
ncexternl.h ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h hdf5internal.h
if USE_DAP
noinst_HEADERS += ncdap.h

58
include/hdf5internal.h Normal file
View File

@ -0,0 +1,58 @@
/* Copyright 2005-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file This header file contains macros, types, and prototypes for
* the HDF5 code in libhdf5. This header should not be included in
* code outside libhdf5.
*
* @author Ed Hartnett
*/
#ifndef _HDF5INTERNAL_
#define _HDF5INTERNAL_
#include "config.h"
#include "nc4internal.h"
#include "ncdimscale.h"
#include "ncdispatch.h"
#include <nc4dispatch.h>
#include <hdf5.h>
#include <hdf5_hl.h>
#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10)
/* These have to do with creating chuncked datasets in HDF5. */
#define NC_HDF5_UNLIMITED_DIMSIZE (0)
#define NC_HDF5_CHUNKSIZE_FACTOR (10)
#define NC_HDF5_MIN_CHUNK_SIZE (2)
#define NC_EMPTY_SCALE "NC_EMPTY_SCALE"
/* This is an attribute I had to add to handle multidimensional
* coordinate variables. */
#define COORDINATES "_Netcdf4Coordinates"
#define COORDINATES_LEN (NC_MAX_NAME * 5)
/* This is used when the user defines a non-coordinate variable with
* same name as a dimension. */
#define NON_COORD_PREPEND "_nc4_non_coord_"
/* An attribute in the HDF5 root group of this name means that the
* file must follow strict netCDF classic format rules. */
#define NC3_STRICT_ATT_NAME "_nc3_strict"
/* If this attribute is present on a dimscale variable, use the value
* as the netCDF dimid. */
#define NC_DIMID_ATT_NAME "_Netcdf4Dimid"
/** This is the name of the class HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS"
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
/* These functions do HDF5 things. */
int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
#endif /* _HDF5INTERNAL_ */

47
include/nc4internal.h
View File

@ -1,8 +1,10 @@
/* Copyright 2005-2018 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file This header file contains the definitions of structs used to
* hold netCDF file metadata in memory.
* @file This header file contains macros, types and prototypes used
* to build and manipulate the netCDF metadata model.
*
* @author Ed Hartnett, Dennis Heimbigner, Ward Fisher
*/
#ifndef _NC4INTERNAL_
@ -14,7 +16,6 @@
#include <ctype.h>
#include <stdarg.h>
#include <string.h>
#include <hdf5.h>
#include "ncdimscale.h"
#include "nc_logging.h"
@ -37,7 +38,6 @@
typedef enum {GET, PUT} NC_PG_T;
typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT;
#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10)
#define NC_V2_ERR (-1)
/* The name of the root group. */
@ -74,36 +74,6 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT;
#define X_DOUBLE_MAX 1.7976931348623157e+308
#define X_DOUBLE_MIN (-X_DOUBLE_MAX)
/* These have to do with creating chuncked datasets in HDF5. */
#define NC_HDF5_UNLIMITED_DIMSIZE (0)
#define NC_HDF5_CHUNKSIZE_FACTOR (10)
#define NC_HDF5_MIN_CHUNK_SIZE (2)
#define NC_EMPTY_SCALE "NC_EMPTY_SCALE"
/* This is an attribute I had to add to handle multidimensional
* coordinate variables. */
#define COORDINATES "_Netcdf4Coordinates"
#define COORDINATES_LEN (NC_MAX_NAME * 5)
/* This is used when the user defines a non-coordinate variable with
* same name as a dimension. */
#define NON_COORD_PREPEND "_nc4_non_coord_"
/* An attribute in the HDF5 root group of this name means that the
* file must follow strict netCDF classic format rules. */
#define NC3_STRICT_ATT_NAME "_nc3_strict"
/* If this attribute is present on a dimscale variable, use the value
* as the netCDF dimid. */
#define NC_DIMID_ATT_NAME "_Netcdf4Dimid"
/** This is the name of the class HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS"
/** This is the name of the name HDF5 dimension scale attribute. */
#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
/** This is the number of netCDF atomic types. */
#define NUM_ATOMIC_TYPES (NC_MAX_ATOMIC_TYPE + 1)
@ -353,7 +323,6 @@ int nc4_convert_type(const void *src, void *dest, const nc_type src_type,
/* These functions do HDF5 things. */
int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim);
int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset);
int nc4_put_vars(NC *nc, int ncid, int varid, const size_t *startp,
@ -438,6 +407,12 @@ int nc4_check_name(const char *name, char *norm_name);
int nc4_normalize_name(const char *name, char *norm_name);
int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name);
/* Find default fill value. */
int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value);
/* Close the file. */
int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem);
/* HDF5 initialization */
extern int nc4_hdf5_initialized;
extern void nc4_hdf5_initialize(void);
@ -521,4 +496,4 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd
extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
#endif /* _NETCDF4_ */
#endif /* _NC4INTERNAL_ */

4
include/nc_logging.h
View File

@ -42,7 +42,6 @@ void nc_log_hdf5(void);
#else /* LOGGING */
/* These definitions will be used unless LOGGING is defined. */
#define LOG(e)
#define BAIL2(e) \
@ -52,9 +51,12 @@ void nc_log_hdf5(void);
#define BAIL_QUIET BAIL
#ifdef USE_NETCDF_4
#ifndef ENABLE_SET_LOG_LEVEL
/* Define away any calls to nc_set_log_level(), if its not enabled. */
#define nc_set_log_level(e)
#endif /* ENABLE_SET_LOG_LEVEL */
#endif
#endif /* LOGGING */

2
include/ncdimscale.h
View File

@ -8,6 +8,8 @@
#ifndef _NCDIMSCALE_H_
#define _NCDIMSCALE_H_
#include <hdf5.h>
typedef struct hdf5_objid
{
unsigned long fileno[2]; /* file number */

3
libhdf5/CMakeLists.txt
View File

@ -7,7 +7,8 @@
# The source files for the HDF5 dispatch layer.
SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c)
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c
hdf5var.c)
# Build the HDF4 dispatch layer as a library that will be included in
# the netCDF library.

5
libhdf5/Makefile.am
View File

@ -13,8 +13,9 @@ libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS}
noinst_LTLIBRARIES = libnchdf5.la
# The source files.
libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c
libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \
hdf5var.c
# Package this for cmake build.
EXTRA_DIST = CMakeLists.txt

1
libhdf5/hdf5attr.c
View File

@ -13,6 +13,7 @@
#include "nc.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include "ncdispatch.h"

328
libhdf5/hdf5create.c Normal file
View File

@ -0,0 +1,328 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
* @file
* @internal The netCDF-4 file functions relating to file creation.
*
* @author Ed Hartnett
*/
#include "config.h"
#include "hdf5internal.h"
/* From hdf5file.c. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
/** @internal These flags may not be set for create. */
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5);
/* From nc4mem.c */
extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t);
/**
* @internal Create a netCDF-4/HDF5 file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz The proposed initial file size (advisory)
* @param parameters extra parameter info (like MPI communicator)
* @param nc Pointer to an instance of NC.
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @return ::NC_EEXIST File exists and NC_NOCLOBBER used.
* @return ::NC_EHDFERR HDF5 returned error.
* @ingroup netcdf4
* @author Ed Hartnett, Dennis Heimbigner
*/
static int
nc4_create_file(const char *path, int cmode, size_t initialsz,
void* parameters, NC *nc)
{
hid_t fcpl_id, fapl_id = -1;
unsigned flags;
FILE *fp;
int retval = NC_NOERR;
NC_FILE_INFO_T* nc4_info = NULL;
#ifdef USE_PARALLEL4
NC_MPI_INFO *mpiinfo = NULL;
MPI_Comm comm;
MPI_Info info;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#endif /* !USE_PARALLEL4 */
assert(nc && path);
LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode));
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode))))
BAIL(retval);
nc4_info = NC4_DATA(nc);
assert(nc4_info && nc4_info->root_grp);
nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY;
nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS;
nc4_info->mem.created = 1;
nc4_info->mem.initialsize = initialsz;
if(nc4_info->mem.inmemory && parameters)
nc4_info->mem.memio = *(NC_memio*)parameters;
#ifdef USE_PARALLEL4
else if(parameters) {
mpiinfo = (NC_MPI_INFO *)parameters;
comm = mpiinfo->comm;
info = mpiinfo->info;
}
#endif
if(nc4_info->mem.diskless)
flags = H5F_ACC_TRUNC;
else if(cmode & NC_NOCLOBBER)
flags = H5F_ACC_EXCL;
else
flags = H5F_ACC_TRUNC;
/* If this file already exists, and NC_NOCLOBBER is specified,
return an error (unless diskless|inmemory) */
if (nc4_info->mem.diskless) {
if((cmode & NC_WRITE) && (cmode & NC_NOCLOBBER) == 0)
nc4_info->mem.persist = 1;
} else if (nc4_info->mem.inmemory) {
/* ok */
} else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) {
fclose(fp);
BAIL(NC_EEXIST);
}
/* Need this access plist to control how HDF5 handles open objects
* on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to
* fail if there are any open objects in the file. */
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
BAIL(NC_EHDFERR);
#ifdef USE_PARALLEL4
/* If this is a parallel file create, set up the file creation
property list. */
if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) {
nc4_info->parallel = NC_TRUE;
if (cmode & NC_MPIIO) /* MPI/IO */
{
LOG((4, "creating parallel file with MPI/IO"));
if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
BAIL(NC_EPARINIT);
}
#ifdef USE_PARALLEL_POSIX
else /* MPI/POSIX */
{
LOG((4, "creating parallel file with MPI/posix"));
if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
BAIL(NC_EPARINIT);
}
#else /* USE_PARALLEL_POSIX */
/* Should not happen! Code in NC4_create/NC4_open should alias
* the NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not
* available in HDF5. -QAK */
else /* MPI/POSIX */
BAIL(NC_EPARINIT);
#endif /* USE_PARALLEL_POSIX */
/* Keep copies of the MPI Comm & Info objects */
if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm))
BAIL(NC_EMPI);
comm_duped++;
if (MPI_INFO_NULL != info)
{
if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info))
BAIL(NC_EMPI);
info_duped++;
}
else
{
/* No dup, just copy it. */
nc4_info->info = info;
}
}
#else /* only set cache for non-parallel... */
if(cmode & NC_DISKLESS) {
if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist))
BAIL(NC_EDISKLESS);
}
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
nc4_chunk_cache_preemption) < 0)
BAIL(NC_EHDFERR);
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
nc4_chunk_cache_preemption));
#endif /* USE_PARALLEL4 */
#ifdef HDF5_HAS_LIBVER_BOUNDS
#if H5_VERSION_GE(1,10,2)
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0)
#else
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST,
H5F_LIBVER_LATEST) < 0)
#endif
BAIL(NC_EHDFERR);
#endif
/* Create the property list. */
if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
BAIL(NC_EHDFERR);
/* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */
if (H5Pset_obj_track_times(fcpl_id,0)<0)
BAIL(NC_EHDFERR);
/* Set latest_format in access propertly list and
* H5P_CRT_ORDER_TRACKED in the creation property list. This turns
* on HDF5 creation ordering. */
if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0)
BAIL(NC_EHDFERR);
/* Create the file. */
#ifdef HDF5_HAS_COLL_METADATA_OPS
H5Pset_all_coll_metadata_ops(fapl_id, 1 );
H5Pset_coll_metadata_write(fapl_id, 1);
#endif
if(nc4_info->mem.inmemory) {
#if 0
if(nc4_info->mem.memio.size == 0)
nc4_info->memio.size = DEFAULT_CREATE_MEMSIZE; /* last ditch fix */
if(nc4_info->memio.memory == NULL) { /* last ditch fix */
nc4_info->memio.memory = malloc(nc4_info->memio.size);
if(nc4_info->memio.memory == NULL)
BAIL(NC_ENOMEM);
}
assert(nc4_info->memio.size > 0 && nc4_info->memio.memory != NULL);
#endif
retval = NC4_create_image_file(nc4_info,initialsz);
if(retval)
BAIL(retval);
} else if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
/*Change the return error from NC_EFILEMETADATA to
System error EACCES because that is the more likely problem */
BAIL(EACCES);
/* Open the root group. */
if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(nc4_info->hdfid, "/",
H5P_DEFAULT)) < 0)
BAIL(NC_EFILEMETA);
/* Release the property lists. */
if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0)
BAIL(NC_EHDFERR);
/* Define mode gets turned on automatically on create. */
nc4_info->flags |= NC_INDEF;
/* Get the HDF5 superblock and read and parse the special
* _NCProperties attribute. */
if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo)))
BAIL(retval);
/* Write the _NCProperties attribute. */
if ((retval = NC4_put_propattr(nc4_info)))
BAIL(retval);
return NC_NOERR;
exit: /*failure exit*/
#ifdef USE_PARALLEL4
if (comm_duped) MPI_Comm_free(&nc4_info->comm);
if (info_duped) MPI_Info_free(&nc4_info->info);
#endif
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if(!nc4_info) return retval;
nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort */
return retval;
}
/**
* @internal Create a netCDF-4/HDF5 file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz Ignored by this function.
* @param basepe Ignored by this function.
* @param chunksizehintp Ignored by this function.
* @param use_parallel 0 for sequential, non-zero for parallel I/O.
* @param parameters pointer to struct holding extra data (e.g. for
* parallel I/O) layer. Ignored if NULL.
* @param dispatch Pointer to the dispatch table for this file.
* @param nc_file Pointer to an instance of NC.
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @ingroup netcdf4
* @author Ed Hartnett
*/
int
NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
size_t *chunksizehintp, int use_parallel, void *parameters,
NC_Dispatch *dispatch, NC* nc_file)
{
int res;
assert(nc_file && path);
LOG((1, "%s: path %s cmode 0x%x parameters %p",
__func__, path, cmode, parameters));
/* If this is our first file, turn off HDF5 error messages. */
if (!nc4_hdf5_initialized)
nc4_hdf5_initialize();
/* Check the cmode for validity. */
if((cmode & ILLEGAL_CREATE_FLAGS) != 0)
{res = NC_EINVAL; goto done;}
/* Cannot have both */
if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
{res = NC_EINVAL; goto done;}
/* Currently no parallel diskless io */
if((cmode & (NC_MPIIO | NC_MPIPOSIX)) && (cmode & NC_DISKLESS))
{res = NC_EINVAL; goto done;}
#ifndef USE_PARALLEL_POSIX
/* If the HDF5 library has been compiled without the MPI-POSIX VFD, alias
* the NC_MPIPOSIX flag to NC_MPIIO. -QAK
*/
if(cmode & NC_MPIPOSIX)
{
cmode &= ~NC_MPIPOSIX;
cmode |= NC_MPIIO;
}
#endif /* USE_PARALLEL_POSIX */
cmode |= NC_NETCDF4;
/* Apply default create format. */
if (nc_get_default_format() == NC_FORMAT_CDF5)
cmode |= NC_CDF5;
else if (nc_get_default_format() == NC_FORMAT_64BIT_OFFSET)
cmode |= NC_64BIT_OFFSET;
else if (nc_get_default_format() == NC_FORMAT_NETCDF4_CLASSIC)
cmode |= NC_CLASSIC_MODEL;
LOG((2, "cmode after applying default format: 0x%x", cmode));
nc_file->int_ncid = nc_file->ext_ncid;
res = nc4_create_file(path, cmode, initialsz, parameters, nc_file);
done:
return res;
}

1
libhdf5/hdf5dim.c
View File

@ -12,6 +12,7 @@
*/
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**

2526
libhdf5/hdf5file.c
View File

File diff suppressed because it is too large Load Diff

1
libhdf5/hdf5grp.c
View File

@ -12,6 +12,7 @@
*/
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**

1
libhdf5/hdf5internal.c
View File

@ -15,6 +15,7 @@
*/
#include "config.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc.h" /* from libsrc */
#include "ncdispatch.h" /* from libdispatch */
#include "ncutf8.h"

2115
libhdf5/hdf5open.c Normal file
View File

File diff suppressed because it is too large Load Diff

1
libhdf5/hdf5type.c
View File

@ -12,6 +12,7 @@
* @author Ed Hartnett
*/
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**

1250
libhdf5/hdf5var.c Normal file
View File

File diff suppressed because it is too large Load Diff

1
libhdf5/nc4hdf.c
View File

@ -15,6 +15,7 @@
#include "config.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include <H5DSpublic.h>
#include <math.h>

1
libhdf5/nc4info.c
View File

@ -12,6 +12,7 @@
#include <hdf5.h>
#include "netcdf.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#define HDF5_MAX_NAME 1024 /**< HDF5 max name. */

1
libsrc4/nc4attr.c
View File

@ -18,6 +18,7 @@
#include "nc.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include "ncdispatch.h"

230
libsrc4/nc4var.c
View File

@ -10,23 +10,12 @@
* @author Ed Hartnett, Dennis Heimbigner, Ward Fisher
*/
#include "config.h"
#include <nc4internal.h>
#include "nc4dispatch.h"
#include "hdf5internal.h"
#include <math.h>
/* Szip options. */
#define NC_SZIP_EC_OPTION_MASK 4 /**< @internal SZIP EC option mask. */
#define NC_SZIP_NN_OPTION_MASK 32 /**< @internal SZIP NN option mask. */
#define NC_SZIP_MAX_PIXELS_PER_BLOCK 32 /**< @internal SZIP max pixels per block. */
/** @internal Default size for unlimited dim chunksize */
#define DEFAULT_1D_UNLIM_SIZE (4096)
#define NC_ARRAY_GROWBY 4 /**< @internal Amount to grow array. */
extern int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info,
void *fill_value);
/**
* @internal Set chunk cache size for a variable. This is the internal
* function called by nc_set_var_chunk_cache().
@ -201,165 +190,6 @@ nc_get_var_chunk_cache_ints(int ncid, int varid, int *sizep,
return NC_NOERR;
}
/**
* @internal Check a set of chunksizes to see if they specify a chunk
* that is too big.
*
* @param grp Pointer to the group info.
* @param var Pointer to the var info.
* @param chunksizes Array of chunksizes to check.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @returns ::NC_EBADCHUNK Bad chunksize.
*/
static int
check_chunksizes(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, const size_t *chunksizes)
{
double dprod;
size_t type_len;
int d;
int retval;
if ((retval = nc4_get_typelen_mem(grp->nc4_info, var->type_info->hdr.id, &type_len)))
return retval;
if (var->type_info->nc_type_class == NC_VLEN)
dprod = (double)sizeof(hvl_t);
else
dprod = (double)type_len;
for (d = 0; d < var->ndims; d++)
dprod *= (double)chunksizes[d];
if (dprod > (double) NC_MAX_UINT)
return NC_EBADCHUNK;
return NC_NOERR;
}
/**
* @internal Determine some default chunksizes for a variable.
*
* @param grp Pointer to the group info.
* @param var Pointer to the var info.
*
* @returns ::NC_NOERR for success
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @author Ed Hartnett
*/
static int
nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
{
int d;
size_t type_size;
float num_values = 1, num_unlim = 0;
int retval;
size_t suggested_size;
#ifdef LOGGING
double total_chunk_size;
#endif
if (var->type_info->nc_type_class == NC_STRING)
type_size = sizeof(char *);
else
type_size = var->type_info->size;
#ifdef LOGGING
/* Later this will become the total number of bytes in the default
* chunk. */
total_chunk_size = (double) type_size;
#endif
/* How many values in the variable (or one record, if there are
* unlimited dimensions). */
for (d = 0; d < var->ndims; d++)
{
assert(var->dim[d]);
if (! var->dim[d]->unlimited)
num_values *= (float)var->dim[d]->len;
else {
num_unlim++;
var->chunksizes[d] = 1; /* overwritten below, if all dims are unlimited */
}
}
/* Special case to avoid 1D vars with unlim dim taking huge amount
of space (DEFAULT_CHUNK_SIZE bytes). Instead we limit to about
4KB */
if (var->ndims == 1 && num_unlim == 1) {
if (DEFAULT_CHUNK_SIZE / type_size <= 0)
suggested_size = 1;
else if (DEFAULT_CHUNK_SIZE / type_size > DEFAULT_1D_UNLIM_SIZE)
suggested_size = DEFAULT_1D_UNLIM_SIZE;
else
suggested_size = DEFAULT_CHUNK_SIZE / type_size;
var->chunksizes[0] = suggested_size / type_size;
LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d "
"chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[0]));
}
if (var->ndims > 1 && var->ndims == num_unlim) { /* all dims unlimited */
suggested_size = pow((double)DEFAULT_CHUNK_SIZE/type_size, 1.0/(double)(var->ndims));
for (d = 0; d < var->ndims; d++)
{
var->chunksizes[d] = suggested_size ? suggested_size : 1;
LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d "
"chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d]));
}
}
/* Pick a chunk length for each dimension, if one has not already
* been picked above. */
for (d = 0; d < var->ndims; d++)
if (!var->chunksizes[d])
{
suggested_size = (pow((double)DEFAULT_CHUNK_SIZE/(num_values * type_size),
1.0/(double)(var->ndims - num_unlim)) * var->dim[d]->len - .5);
if (suggested_size > var->dim[d]->len)
suggested_size = var->dim[d]->len;
var->chunksizes[d] = suggested_size ? suggested_size : 1;
LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d "
"chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d]));
}
#ifdef LOGGING
/* Find total chunk size. */
for (d = 0; d < var->ndims; d++)
total_chunk_size *= (double) var->chunksizes[d];
LOG((4, "total_chunk_size %f", total_chunk_size));
#endif
/* But did this result in a chunk that is too big? */
retval = check_chunksizes(grp, var, var->chunksizes);
if (retval)
{
/* Other error? */
if (retval != NC_EBADCHUNK)
return retval;
/* Chunk is too big! Reduce each dimension by half and try again. */
for ( ; retval == NC_EBADCHUNK; retval = check_chunksizes(grp, var, var->chunksizes))
for (d = 0; d < var->ndims; d++)
var->chunksizes[d] = var->chunksizes[d]/2 ? var->chunksizes[d]/2 : 1;
}
/* Do we have any big data overhangs? They can be dangerous to
* babies, the elderly, or confused campers who have had too much
* beer. */
for (d = 0; d < var->ndims; d++)
{
size_t num_chunks;
size_t overhang;
assert(var->chunksizes[d] > 0);
num_chunks = (var->dim[d]->len + var->chunksizes[d] - 1) / var->chunksizes[d];
if(num_chunks > 0) {
overhang = (num_chunks * var->chunksizes[d]) - var->dim[d]->len;
var->chunksizes[d] -= overhang / num_chunks;
}
}
return NC_NOERR;
}
/**
* @internal Get all the information about a variable. Pass NULL for
* whatever you don't care about. This is the internal function called
@ -605,62 +435,6 @@ nc_inq_var_chunking_ints(int ncid, int varid, int *contiguousp, int *chunksizesp
return retval;
}
/**
* @internal Define chunking stuff for a var. This is called by
* the fortran API.
*
* @param ncid File ID.
* @param varid Variable ID.
* @param contiguous Pointer to contiguous setting.
* @param chunksizesp Array of chunksizes.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is
* not netCDF-4/HDF5.
* @returns ::NC_ELATEDEF Too late to change settings for this variable.
* @returns ::NC_ENOTINDEFINE Not in define mode.
* @returns ::NC_EINVAL Invalid input
* @returns ::NC_EBADCHUNK Bad chunksize.
* @author Ed Hartnett
*/
int
nc_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp)
{
NC *nc;
NC_GRP_INFO_T *grp;
NC_VAR_INFO_T *var;
NC_FILE_INFO_T *h5;
size_t *cs = NULL;
int i, retval;
/* Find this ncid's file info. */
if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
return retval;
assert(nc);
/* Find var cause I need the number of dims. */
if ((retval = nc4_find_g_var_nc(nc, ncid, varid, &grp, &var)))
return retval;
/* Allocate space for the size_t copy of the chunksizes array. */
if (var->ndims)
if (!(cs = malloc(var->ndims * sizeof(size_t))))
return NC_ENOMEM;
/* Copy to size_t array. */
for (i = 0; i < var->ndims; i++)
cs[i] = chunksizesp[i];
retval = nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL,
&contiguous, cs, NULL, NULL, NULL);
if (var->ndims)
free(cs);
return retval;
}
/**
* @internal Find the ID of a variable, from the name. This function
* is called by nc_inq_varid().

3
nc_test/tst_inmemory.c
View File

@ -605,9 +605,8 @@ main(int argc, char **argv)
NC_memio filedata3;
NC_memio filedata4;
nc_set_log_level(0);
#ifdef USE_NETCDF4
nc_set_log_level(0);
H5Eprint1(stderr);
#endif

2
nc_test4/tst_atts.c
View File

@ -10,7 +10,7 @@
#include <config.h>
#include <nc_tests.h>
#include "err_macros.h"
#include "nc4internal.h"
#include "hdf5internal.h"
/* The data file we will create. */
#define FILE_NAME "tst_atts.nc"

1
ncdump/tst_dimsizes.c
View File

@ -34,7 +34,6 @@ main(int argc, char **argv)
size_t dimsize;
int dimid;
int stat = NC_NOERR;
nc_set_log_level(5);
printf("\n*** Testing Max Dimension Sizes\n");
printf("\n|size_t|=%lu\n",(unsigned long)sizeof(size_t));