mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-12-27 08:49:16 +08:00
Code Issues Packages Projects Releases Wiki Activity
388bbf4e73
netcdf-c/libhdf5/hdf5create.c
Scot Breitenfeld 388bbf4e73 Allow for the HDF5 file format to be compatible with features greater 
than HDF5 1.8. This will allow for HDF5 features (VDS, SWMR, new references, etc...)
which may require to have a superblock greater than v2.

See for discussion Ref: Update HDF5 format compatibility Unidata#951
2021-01-11 09:48:04 -06:00

348 lines
11 KiB
C
Raw Blame History

/* Copyright 2003-2018, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
* @file
* @internal The netCDF-4 file functions relating to file creation.
*
* @author Ed Hartnett
*/
#include "config.h"
#include "hdf5internal.h"
/* From hdf5file.c. */
extern size_t nc4_chunk_cache_size;
extern size_t nc4_chunk_cache_nelems;
extern float nc4_chunk_cache_preemption;
/** @internal These flags may not be set for create. */
static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5);
/* From nc4mem.c */
extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t);
#ifdef _WIN32
static hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);
#else
#define nc4_H5Fcreate H5Fcreate
#endif
/**
* @internal Create a netCDF-4/HDF5 file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz The proposed initial file size (advisory, for
* in-memory netCDF-4/HDF5 files only).
* @param parameters extra parameter info (like MPI communicator).
* @param ncid The already-assigned ncid for this file (aka ext_ncid).
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @return ::NC_EEXIST File exists and NC_NOCLOBBER used.
* @return ::NC_EHDFERR HDF5 returned error.
* @ingroup netcdf4
* @author Ed Hartnett, Dennis Heimbigner
*/
static int
nc4_create_file(const char *path, int cmode, size_t initialsz,
void* parameters, int ncid)
{
hid_t fcpl_id, fapl_id = -1;
unsigned flags;
FILE *fp;
int retval = NC_NOERR;
NC_FILE_INFO_T *nc4_info;
NC_HDF5_FILE_INFO_T *hdf5_info;
NC_HDF5_GRP_INFO_T *hdf5_grp;
#ifdef USE_PARALLEL4
NC_MPI_INFO *mpiinfo = NULL;
MPI_Comm comm;
MPI_Info info;
int comm_duped = 0; /* Whether the MPI Communicator was duplicated */
int info_duped = 0; /* Whether the MPI Info object was duplicated */
#endif /* !USE_PARALLEL4 */
assert(path);
LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode));
/* Add necessary structs to hold netcdf-4 file data. */
if ((retval = nc4_file_list_add(ncid, path, NC_WRITE | cmode,
(void **)&nc4_info)))
BAIL(retval);
assert(nc4_info && nc4_info->root_grp);
nc4_info->root_grp->atts_read = 1;
/* Add struct to hold HDF5-specific file metadata. */
if (!(nc4_info->format_file_info = calloc(1, sizeof(NC_HDF5_FILE_INFO_T))))
BAIL(NC_ENOMEM);
hdf5_info = (NC_HDF5_FILE_INFO_T *)nc4_info->format_file_info;
/* Add struct to hold HDF5-specific group info. */
if (!(nc4_info->root_grp->format_grp_info = calloc(1, sizeof(NC_HDF5_GRP_INFO_T))))
return NC_ENOMEM;
hdf5_grp = (NC_HDF5_GRP_INFO_T *)nc4_info->root_grp->format_grp_info;
nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY;
nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS;
nc4_info->mem.persist = (cmode & NC_PERSIST) == NC_PERSIST;
nc4_info->mem.created = 1;
nc4_info->mem.initialsize = initialsz;
/* diskless => !inmemory */
if(nc4_info->mem.inmemory && nc4_info->mem.diskless)
BAIL(NC_EINTERNAL);
if(nc4_info->mem.inmemory && parameters)
nc4_info->mem.memio = *(NC_memio*)parameters;
#ifdef USE_PARALLEL4
else if(parameters) {
mpiinfo = (NC_MPI_INFO *)parameters;
comm = mpiinfo->comm;
info = mpiinfo->info;
}
#endif
if(nc4_info->mem.diskless)
flags = H5F_ACC_TRUNC;
else if(cmode & NC_NOCLOBBER)
flags = H5F_ACC_EXCL;
else
flags = H5F_ACC_TRUNC;
/* If this file already exists, and NC_NOCLOBBER is specified,
return an error (unless diskless|inmemory) */
if (!nc4_info->mem.diskless && !nc4_info->mem.inmemory) {
if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) {
fclose(fp);
BAIL(NC_EEXIST);
}
}
/* Need this access plist to control how HDF5 handles open objects
* on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to
* fail if there are any open objects in the file. */
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
BAIL(NC_EHDFERR);
#ifdef USE_PARALLEL4
/* If this is a parallel file create, set up the file creation
property list. */
if (mpiinfo != NULL) {
nc4_info->parallel = NC_TRUE;
LOG((4, "creating parallel file with MPI/IO"));
if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
BAIL(NC_EPARINIT);
/* Keep copies of the MPI Comm & Info objects */
if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm))
BAIL(NC_EMPI);
comm_duped++;
if (MPI_INFO_NULL != info)
{
if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info))
BAIL(NC_EMPI);
info_duped++;
}
else
{
/* No dup, just copy it. */
nc4_info->info = info;
}
}
#endif /* USE_PARALLEL4 */
/* Only set cache for non-parallel creates. */
if (!nc4_info->parallel)
{
if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
nc4_chunk_cache_preemption) < 0)
BAIL(NC_EHDFERR);
LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
__func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems,
nc4_chunk_cache_preemption));
}
#ifdef HAVE_H5PSET_LIBVER_BOUNDS
#if H5_VERSION_LT(1,10,0)
/* all HDF5 1.8 lib versions */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0);
#else
#if H5_VERSION_LE(1,10,1)
/* lib versions 1.10.0, 1.10.1 */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0);
#else
/* lib versions 1.10.2 and higher */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0);
#endif
BAIL(NC_EHDFERR);
#endif
/* Create the property list. */
if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
BAIL(NC_EHDFERR);
/* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */
if (H5Pset_obj_track_times(fcpl_id,0)<0)
BAIL(NC_EHDFERR);
/* Set latest_format in access propertly list and
* H5P_CRT_ORDER_TRACKED in the creation property list. This turns
* on HDF5 creation ordering. */
if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0)
BAIL(NC_EHDFERR);
#ifdef HDF5_HAS_COLL_METADATA_OPS
/* If HDF5 supports collective metadata operations, turn them
* on. This is only relevant for parallel I/O builds of HDF5. */
if (H5Pset_all_coll_metadata_ops(fapl_id, 1) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_coll_metadata_write(fapl_id, 1) < 0)
BAIL(NC_EHDFERR);
#endif
if(nc4_info->mem.inmemory) {
retval = NC4_create_image_file(nc4_info,initialsz);
if(retval)
BAIL(retval);
}
else
if(nc4_info->mem.diskless) {
size_t alloc_incr; /* Buffer allocation increment */
size_t min_incr = 65536; /* Minimum buffer increment */
double buf_prcnt = 0.1f; /* Percentage of buffer size to set as increment */
/* set allocation increment to a percentage of the supplied buffer size, or
* a pre-defined minimum increment value, whichever is larger
*/
if ((buf_prcnt * initialsz) > min_incr)
alloc_incr = (size_t)(buf_prcnt * initialsz);
else
alloc_incr = min_incr;
/* Configure FAPL to use the core file driver */
if (H5Pset_fapl_core(fapl_id, alloc_incr, (nc4_info->mem.persist?1:0)) < 0)
BAIL(NC_EHDFERR);
if ((hdf5_info->hdfid = nc4_H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
BAIL(EACCES);
}
else /* Normal file */
{
/* Create the HDF5 file. */
if ((hdf5_info->hdfid = nc4_H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
BAIL(EACCES);
}
/* Open the root group. */
if ((hdf5_grp->hdf_grpid = H5Gopen2(hdf5_info->hdfid, "/", H5P_DEFAULT)) < 0)
BAIL(NC_EFILEMETA);
/* Release the property lists. */
if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0)
BAIL(NC_EHDFERR);
/* Define mode gets turned on automatically on create. */
nc4_info->flags |= NC_INDEF;
/* Save the HDF5 superblock number and set the _NCProperties attribute. */
if ((retval = NC4_new_provenance(nc4_info)))
BAIL(retval);
return NC_NOERR;
exit: /*failure exit*/
#ifdef USE_PARALLEL4
if (comm_duped) MPI_Comm_free(&nc4_info->comm);
if (info_duped) MPI_Info_free(&nc4_info->info);
#endif
if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
if(!nc4_info) return retval;
nc4_close_hdf5_file(nc4_info, 1, NULL); /* treat like abort */
return retval;
}
/**
* @internal Create a netCDF-4/HDF5 file.
*
* @param path The file name of the new file.
* @param cmode The creation mode flag.
* @param initialsz Ignored by this function.
* @param basepe Ignored by this function.
* @param chunksizehintp Ignored by this function.
* @param parameters pointer to struct holding extra data (e.g. for
* parallel I/O) layer. Ignored if NULL.
* @param dispatch Pointer to the dispatch table for this file.
* @param ncid The ncid that has been assigned by the dispatch layer
* (aka ext_ncid).
*
* @return ::NC_NOERR No error.
* @return ::NC_EINVAL Invalid input (check cmode).
* @ingroup netcdf4
* @author Ed Hartnett
*/
int
NC4_create(const char* path, int cmode, size_t initialsz, int basepe,
size_t *chunksizehintp, void *parameters,
const NC_Dispatch *dispatch, int ncid)
{
int res;
assert(path);
LOG((1, "%s: path %s cmode 0x%x parameters %p",
__func__, path, cmode, parameters));
/* If this is our first file, turn off HDF5 error messages. */
if (!nc4_hdf5_initialized)
nc4_hdf5_initialize();
#ifdef LOGGING
/* If nc logging level has changed, see if we need to turn on
* HDF5's error messages. */
hdf5_set_log_level();
#endif /* LOGGING */
/* Check the cmode for validity. Checking parallel against
* NC_DISKLESS already done in NC_create(). */
if (cmode & ILLEGAL_CREATE_FLAGS)
return NC_EINVAL;
/* Create the netCDF-4/HDF5 file. */
res = nc4_create_file(path, cmode, initialsz, parameters, ncid);
return res;
}
#ifdef _WIN32
/**
* Wrapper function for H5Fcreate.
* Converts the filename from ANSI to UTF-8 as needed before calling H5Fcreate.
*
* @param filename The filename encoded ANSI to access.
* @param flags File access flags.
* @param fcpl_id File creation property list identifier.
* @param fapl_id File access property list identifier.
* @return A file identifier if succeeded. A negative value if failed.
*/
static hid_t
nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
{
pathbuf_t pb;
hid_t hid;
filename = nc4_ndf5_ansi_to_utf8(&pb, filename);
if (!filename)
return H5I_INVALID_HID;
hid = H5Fcreate(filename, flags, fcpl_id, fapl_id);
nc4_hdf5_free_pathbuf(&pb);
return hid;
}
#endif /* _WIN32 */
Reference in New Issue View Git Blame Copy Permalink