diff --git a/CMakeLists.txt b/CMakeLists.txt index f578edebd..5d59b024c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -381,14 +381,6 @@ OPTION(ENABLE_MMAP "Use MMAP." ON) # Option to use examples. OPTION(ENABLE_EXAMPLES "Build Examples" ON) - -# Option Logging, only valid for netcdf4. -OPTION(ENABLE_LOGGING "Enable Logging." OFF) -IF(ENABLE_LOGGING) - ADD_DEFINITIONS(-DLOGGING) - SET(LOGGING ON) -ENDIF() - # Option to automatically build netcdf-fortran. IF(NOT MSVC) OPTION(ENABLE_REMOTE_FORTRAN_BOOTSTRAP "Download and build netcdf-fortran automatically (EXPERIMENTAL)." OFF) @@ -533,10 +525,12 @@ IF(ENABLE_LOGGING) ADD_DEFINITIONS(-DLOGGING) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) SET(LOGGING ON) + SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) -IF(ENABLE_NETCDF_$ AND ((NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING)) +IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) + SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() # Option to allow for strict null file padding. diff --git a/config.h.cmake.in b/config.h.cmake.in index 611ceb7a0..9313fc7cd 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -367,6 +367,9 @@ are set when opening a binary file on Windows. */ /* If true, turn on logging. */ #cmakedefine LOGGING 1 +/* If true, define nc_set_log_level. */ +#cmakedefine ENABLE_SET_LOG_LEVEL 1 + /* max size of the default per-var chunk cache. */ #cmakedefine MAX_DEFAULT_CACHE_SIZE ${MAX_DEFAULT_CACHE_SIZE} diff --git a/include/Makefile.am b/include/Makefile.am index dceffa5dc..af0059951 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -17,9 +17,9 @@ include_HEADERS += netcdf_mem.h noinst_HEADERS = nc_logging.h nc_tests.h fbits.h nc.h nclist.h \ ncuri.h ncutf8.h ncdispatch.h ncdimscale.h netcdf_f.h err_macros.h \ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \ -nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h \ -ncauth.h ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h \ -ncexternl.h ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h +nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ +ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ +ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h hdf5internal.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h new file mode 100644 index 000000000..ee27eec43 --- /dev/null +++ b/include/hdf5internal.h @@ -0,0 +1,58 @@ +/* Copyright 2005-2018 University Corporation for Atmospheric + Research/Unidata. */ +/** + * @file This header file contains macros, types, and prototypes for + * the HDF5 code in libhdf5. This header should not be included in + * code outside libhdf5. + * + * @author Ed Hartnett +*/ + +#ifndef _HDF5INTERNAL_ +#define _HDF5INTERNAL_ + +#include "config.h" +#include "nc4internal.h" +#include "ncdimscale.h" +#include "ncdispatch.h" +#include +#include +#include + +#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) + +/* These have to do with creating chuncked datasets in HDF5. */ +#define NC_HDF5_UNLIMITED_DIMSIZE (0) +#define NC_HDF5_CHUNKSIZE_FACTOR (10) +#define NC_HDF5_MIN_CHUNK_SIZE (2) + +#define NC_EMPTY_SCALE "NC_EMPTY_SCALE" + +/* This is an attribute I had to add to handle multidimensional + * coordinate variables. */ +#define COORDINATES "_Netcdf4Coordinates" +#define COORDINATES_LEN (NC_MAX_NAME * 5) + +/* This is used when the user defines a non-coordinate variable with + * same name as a dimension. */ +#define NON_COORD_PREPEND "_nc4_non_coord_" + +/* An attribute in the HDF5 root group of this name means that the + * file must follow strict netCDF classic format rules. */ +#define NC3_STRICT_ATT_NAME "_nc3_strict" + +/* If this attribute is present on a dimscale variable, use the value + * as the netCDF dimid. */ +#define NC_DIMID_ATT_NAME "_Netcdf4Dimid" + +/** This is the name of the class HDF5 dimension scale attribute. */ +#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS" + +/** This is the name of the name HDF5 dimension scale attribute. */ +#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME" + +/* These functions do HDF5 things. */ +int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); +int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); + +#endif /* _HDF5INTERNAL_ */ diff --git a/include/nc4internal.h b/include/nc4internal.h index e9aea2c40..439ee2c3a 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -1,8 +1,10 @@ /* Copyright 2005-2018 University Corporation for Atmospheric Research/Unidata. */ /** - * @file This header file contains the definitions of structs used to - * hold netCDF file metadata in memory. + * @file This header file contains macros, types and prototypes used + * to build and manipulate the netCDF metadata model. + * + * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher */ #ifndef _NC4INTERNAL_ @@ -14,7 +16,6 @@ #include #include #include -#include #include "ncdimscale.h" #include "nc_logging.h" @@ -37,7 +38,6 @@ typedef enum {GET, PUT} NC_PG_T; typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; -#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) #define NC_V2_ERR (-1) /* The name of the root group. */ @@ -74,36 +74,6 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; #define X_DOUBLE_MAX 1.7976931348623157e+308 #define X_DOUBLE_MIN (-X_DOUBLE_MAX) -/* These have to do with creating chuncked datasets in HDF5. */ -#define NC_HDF5_UNLIMITED_DIMSIZE (0) -#define NC_HDF5_CHUNKSIZE_FACTOR (10) -#define NC_HDF5_MIN_CHUNK_SIZE (2) - -#define NC_EMPTY_SCALE "NC_EMPTY_SCALE" - -/* This is an attribute I had to add to handle multidimensional - * coordinate variables. */ -#define COORDINATES "_Netcdf4Coordinates" -#define COORDINATES_LEN (NC_MAX_NAME * 5) - -/* This is used when the user defines a non-coordinate variable with - * same name as a dimension. */ -#define NON_COORD_PREPEND "_nc4_non_coord_" - -/* An attribute in the HDF5 root group of this name means that the - * file must follow strict netCDF classic format rules. */ -#define NC3_STRICT_ATT_NAME "_nc3_strict" - -/* If this attribute is present on a dimscale variable, use the value - * as the netCDF dimid. */ -#define NC_DIMID_ATT_NAME "_Netcdf4Dimid" - -/** This is the name of the class HDF5 dimension scale attribute. */ -#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS" - -/** This is the name of the name HDF5 dimension scale attribute. */ -#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME" - /** This is the number of netCDF atomic types. */ #define NUM_ATOMIC_TYPES (NC_MAX_ATOMIC_TYPE + 1) @@ -353,7 +323,6 @@ int nc4_convert_type(const void *src, void *dest, const nc_type src_type, /* These functions do HDF5 things. */ int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); -int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim); int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset); int nc4_put_vars(NC *nc, int ncid, int varid, const size_t *startp, @@ -438,6 +407,12 @@ int nc4_check_name(const char *name, char *norm_name); int nc4_normalize_name(const char *name, char *norm_name); int nc4_check_dup_name(NC_GRP_INFO_T *grp, char *norm_name); +/* Find default fill value. */ +int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value); + +/* Close the file. */ +int nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem); + /* HDF5 initialization */ extern int nc4_hdf5_initialized; extern void nc4_hdf5_initialize(void); @@ -521,4 +496,4 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ -#endif /* _NETCDF4_ */ +#endif /* _NC4INTERNAL_ */ diff --git a/include/nc_logging.h b/include/nc_logging.h index 6c17152fb..c4735b71f 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -42,7 +42,6 @@ void nc_log_hdf5(void); #else /* LOGGING */ /* These definitions will be used unless LOGGING is defined. */ - #define LOG(e) #define BAIL2(e) \ @@ -52,9 +51,12 @@ void nc_log_hdf5(void); #define BAIL_QUIET BAIL +#ifdef USE_NETCDF_4 #ifndef ENABLE_SET_LOG_LEVEL +/* Define away any calls to nc_set_log_level(), if its not enabled. */ #define nc_set_log_level(e) #endif /* ENABLE_SET_LOG_LEVEL */ +#endif #endif /* LOGGING */ diff --git a/include/ncdimscale.h b/include/ncdimscale.h index 3738bb165..cd88d3ec3 100644 --- a/include/ncdimscale.h +++ b/include/ncdimscale.h @@ -8,6 +8,8 @@ #ifndef _NCDIMSCALE_H_ #define _NCDIMSCALE_H_ +#include + typedef struct hdf5_objid { unsigned long fileno[2]; /* file number */ diff --git a/libhdf5/CMakeLists.txt b/libhdf5/CMakeLists.txt index ff9ce739e..bfeb8d0ef 100644 --- a/libhdf5/CMakeLists.txt +++ b/libhdf5/CMakeLists.txt @@ -7,7 +7,8 @@ # The source files for the HDF5 dispatch layer. SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c -hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c) +hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c +hdf5var.c) # Build the HDF4 dispatch layer as a library that will be included in # the netCDF library. diff --git a/libhdf5/Makefile.am b/libhdf5/Makefile.am index e1f95a70e..645db023e 100644 --- a/libhdf5/Makefile.am +++ b/libhdf5/Makefile.am @@ -13,8 +13,9 @@ libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS} noinst_LTLIBRARIES = libnchdf5.la # The source files. -libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \ -hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c +libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \ +hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \ +hdf5var.c # Package this for cmake build. EXTRA_DIST = CMakeLists.txt diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 2ffbd6398..9ed2d7f87 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -13,6 +13,7 @@ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "ncdispatch.h" diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c new file mode 100644 index 000000000..145be8e1e --- /dev/null +++ b/libhdf5/hdf5create.c @@ -0,0 +1,328 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See COPYRIGHT file for copying and redistribution + * conditions. */ +/** + * @file + * @internal The netCDF-4 file functions relating to file creation. + * + * @author Ed Hartnett + */ + +#include "config.h" +#include "hdf5internal.h" + +/* From hdf5file.c. */ +extern size_t nc4_chunk_cache_size; +extern size_t nc4_chunk_cache_nelems; +extern float nc4_chunk_cache_preemption; + +/** @internal These flags may not be set for create. */ +static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5); + +/* From nc4mem.c */ +extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t); + +/** + * @internal Create a netCDF-4/HDF5 file. + * + * @param path The file name of the new file. + * @param cmode The creation mode flag. + * @param initialsz The proposed initial file size (advisory) + * @param parameters extra parameter info (like MPI communicator) + * @param nc Pointer to an instance of NC. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid input (check cmode). + * @return ::NC_EEXIST File exists and NC_NOCLOBBER used. + * @return ::NC_EHDFERR HDF5 returned error. + * @ingroup netcdf4 + * @author Ed Hartnett, Dennis Heimbigner + */ +static int +nc4_create_file(const char *path, int cmode, size_t initialsz, + void* parameters, NC *nc) +{ + hid_t fcpl_id, fapl_id = -1; + unsigned flags; + FILE *fp; + int retval = NC_NOERR; + NC_FILE_INFO_T* nc4_info = NULL; + +#ifdef USE_PARALLEL4 + NC_MPI_INFO *mpiinfo = NULL; + MPI_Comm comm; + MPI_Info info; + int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ + int info_duped = 0; /* Whether the MPI Info object was duplicated */ +#endif /* !USE_PARALLEL4 */ + + assert(nc && path); + LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode)); + + /* Add necessary structs to hold netcdf-4 file data. */ + if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode)))) + BAIL(retval); + + nc4_info = NC4_DATA(nc); + assert(nc4_info && nc4_info->root_grp); + + nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY; + nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS; + nc4_info->mem.created = 1; + nc4_info->mem.initialsize = initialsz; + + if(nc4_info->mem.inmemory && parameters) + nc4_info->mem.memio = *(NC_memio*)parameters; +#ifdef USE_PARALLEL4 + else if(parameters) { + mpiinfo = (NC_MPI_INFO *)parameters; + comm = mpiinfo->comm; + info = mpiinfo->info; + } +#endif + if(nc4_info->mem.diskless) + flags = H5F_ACC_TRUNC; + else if(cmode & NC_NOCLOBBER) + flags = H5F_ACC_EXCL; + else + flags = H5F_ACC_TRUNC; + + /* If this file already exists, and NC_NOCLOBBER is specified, + return an error (unless diskless|inmemory) */ + if (nc4_info->mem.diskless) { + if((cmode & NC_WRITE) && (cmode & NC_NOCLOBBER) == 0) + nc4_info->mem.persist = 1; + } else if (nc4_info->mem.inmemory) { + /* ok */ + } else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) { + fclose(fp); + BAIL(NC_EEXIST); + } + + /* Need this access plist to control how HDF5 handles open objects + * on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to + * fail if there are any open objects in the file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + BAIL(NC_EHDFERR); + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) + BAIL(NC_EHDFERR); + +#ifdef USE_PARALLEL4 + /* If this is a parallel file create, set up the file creation + property list. */ + if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) { + nc4_info->parallel = NC_TRUE; + if (cmode & NC_MPIIO) /* MPI/IO */ + { + LOG((4, "creating parallel file with MPI/IO")); + if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0) + BAIL(NC_EPARINIT); + } +#ifdef USE_PARALLEL_POSIX + else /* MPI/POSIX */ + { + LOG((4, "creating parallel file with MPI/posix")); + if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0) + BAIL(NC_EPARINIT); + } +#else /* USE_PARALLEL_POSIX */ + /* Should not happen! Code in NC4_create/NC4_open should alias + * the NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not + * available in HDF5. -QAK */ + else /* MPI/POSIX */ + BAIL(NC_EPARINIT); +#endif /* USE_PARALLEL_POSIX */ + + /* Keep copies of the MPI Comm & Info objects */ + if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm)) + BAIL(NC_EMPI); + comm_duped++; + if (MPI_INFO_NULL != info) + { + if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info)) + BAIL(NC_EMPI); + info_duped++; + } + else + { + /* No dup, just copy it. */ + nc4_info->info = info; + } + } +#else /* only set cache for non-parallel... */ + if(cmode & NC_DISKLESS) { + if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist)) + BAIL(NC_EDISKLESS); + } + if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, + nc4_chunk_cache_preemption) < 0) + BAIL(NC_EHDFERR); + LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", + __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, + nc4_chunk_cache_preemption)); +#endif /* USE_PARALLEL4 */ + +#ifdef HDF5_HAS_LIBVER_BOUNDS +#if H5_VERSION_GE(1,10,2) + if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0) +#else + if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, + H5F_LIBVER_LATEST) < 0) +#endif + BAIL(NC_EHDFERR); +#endif + + /* Create the property list. */ + if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) + BAIL(NC_EHDFERR); + + /* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */ + if (H5Pset_obj_track_times(fcpl_id,0)<0) + BAIL(NC_EHDFERR); + + /* Set latest_format in access propertly list and + * H5P_CRT_ORDER_TRACKED in the creation property list. This turns + * on HDF5 creation ordering. */ + if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | + H5P_CRT_ORDER_INDEXED)) < 0) + BAIL(NC_EHDFERR); + if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | + H5P_CRT_ORDER_INDEXED)) < 0) + BAIL(NC_EHDFERR); + + /* Create the file. */ +#ifdef HDF5_HAS_COLL_METADATA_OPS + H5Pset_all_coll_metadata_ops(fapl_id, 1 ); + H5Pset_coll_metadata_write(fapl_id, 1); +#endif + + if(nc4_info->mem.inmemory) { +#if 0 + if(nc4_info->mem.memio.size == 0) + nc4_info->memio.size = DEFAULT_CREATE_MEMSIZE; /* last ditch fix */ + if(nc4_info->memio.memory == NULL) { /* last ditch fix */ + nc4_info->memio.memory = malloc(nc4_info->memio.size); + if(nc4_info->memio.memory == NULL) + BAIL(NC_ENOMEM); + } + assert(nc4_info->memio.size > 0 && nc4_info->memio.memory != NULL); +#endif + retval = NC4_create_image_file(nc4_info,initialsz); + if(retval) + BAIL(retval); + } else if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0) + /*Change the return error from NC_EFILEMETADATA to + System error EACCES because that is the more likely problem */ + BAIL(EACCES); + + /* Open the root group. */ + if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(nc4_info->hdfid, "/", + H5P_DEFAULT)) < 0) + BAIL(NC_EFILEMETA); + + /* Release the property lists. */ + if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0) + BAIL(NC_EHDFERR); + + /* Define mode gets turned on automatically on create. */ + nc4_info->flags |= NC_INDEF; + + /* Get the HDF5 superblock and read and parse the special + * _NCProperties attribute. */ + if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo))) + BAIL(retval); + + /* Write the _NCProperties attribute. */ + if ((retval = NC4_put_propattr(nc4_info))) + BAIL(retval); + + return NC_NOERR; + +exit: /*failure exit*/ +#ifdef USE_PARALLEL4 + if (comm_duped) MPI_Comm_free(&nc4_info->comm); + if (info_duped) MPI_Info_free(&nc4_info->info); +#endif + if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); + if(!nc4_info) return retval; + nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort */ + return retval; +} + +/** + * @internal Create a netCDF-4/HDF5 file. + * + * @param path The file name of the new file. + * @param cmode The creation mode flag. + * @param initialsz Ignored by this function. + * @param basepe Ignored by this function. + * @param chunksizehintp Ignored by this function. + * @param use_parallel 0 for sequential, non-zero for parallel I/O. + * @param parameters pointer to struct holding extra data (e.g. for + * parallel I/O) layer. Ignored if NULL. + * @param dispatch Pointer to the dispatch table for this file. + * @param nc_file Pointer to an instance of NC. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid input (check cmode). + * @ingroup netcdf4 + * @author Ed Hartnett + */ +int +NC4_create(const char* path, int cmode, size_t initialsz, int basepe, + size_t *chunksizehintp, int use_parallel, void *parameters, + NC_Dispatch *dispatch, NC* nc_file) +{ + int res; + + assert(nc_file && path); + + LOG((1, "%s: path %s cmode 0x%x parameters %p", + __func__, path, cmode, parameters)); + + /* If this is our first file, turn off HDF5 error messages. */ + if (!nc4_hdf5_initialized) + nc4_hdf5_initialize(); + + /* Check the cmode for validity. */ + if((cmode & ILLEGAL_CREATE_FLAGS) != 0) + {res = NC_EINVAL; goto done;} + + /* Cannot have both */ + if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX)) + {res = NC_EINVAL; goto done;} + + /* Currently no parallel diskless io */ + if((cmode & (NC_MPIIO | NC_MPIPOSIX)) && (cmode & NC_DISKLESS)) + {res = NC_EINVAL; goto done;} + +#ifndef USE_PARALLEL_POSIX +/* If the HDF5 library has been compiled without the MPI-POSIX VFD, alias + * the NC_MPIPOSIX flag to NC_MPIIO. -QAK + */ + if(cmode & NC_MPIPOSIX) + { + cmode &= ~NC_MPIPOSIX; + cmode |= NC_MPIIO; + } +#endif /* USE_PARALLEL_POSIX */ + + cmode |= NC_NETCDF4; + + /* Apply default create format. */ + if (nc_get_default_format() == NC_FORMAT_CDF5) + cmode |= NC_CDF5; + else if (nc_get_default_format() == NC_FORMAT_64BIT_OFFSET) + cmode |= NC_64BIT_OFFSET; + else if (nc_get_default_format() == NC_FORMAT_NETCDF4_CLASSIC) + cmode |= NC_CLASSIC_MODEL; + + LOG((2, "cmode after applying default format: 0x%x", cmode)); + + nc_file->int_ncid = nc_file->ext_ncid; + + res = nc4_create_file(path, cmode, initialsz, parameters, nc_file); + +done: + return res; +} diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index ec109a904..b052985c3 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -12,6 +12,7 @@ */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index a892b4b79..61968f738 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -15,6 +15,7 @@ #include /* netcdf functions sometimes return system errors */ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "netcdf_mem.h" #ifdef USE_HDF4 @@ -25,63 +26,29 @@ extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); /* From nc4mem.c */ -extern int NC4_open_image_file(NC_FILE_INFO_T* h5); -extern int NC4_create_image_file(NC_FILE_INFO_T* h5, size_t); extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); /** @internal When we have open objects at file close, should we log them or print to stdout. Default is to log. */ #define LOGOPEN 1 -#define CD_NELEMS_ZLIB 1 /**< Number of parameters needed for ZLIB filter. */ - -/** - * @internal Wrap HDF5 allocated memory free operations - * - * @param memory Pointer to memory to be freed. - * - * @return ::NC_NOERR No error. - * @author Dennis Heimbigner -*/ -static void -hdf5free(void* memory) -{ -#ifndef JNA - /* On Windows using the microsoft runtime, it is an error - for one library to free memory allocated by a different library.*/ -#ifdef HDF5_HAS_H5FREE - if(memory != NULL) H5free_memory(memory); -#else -#ifndef _MSC_VER - if(memory != NULL) free(memory); -#endif -#endif -#endif -} - -/* Custom iteration callback data */ -typedef struct { - NC_GRP_INFO_T *grp; - NC_VAR_INFO_T *var; -} att_iter_info; - /* Define the table of names and properties of attributes that are reserved. */ #define NRESERVED 11 /*|NC_reservedatt|*/ /* Must be in sorted order for binary search */ static const NC_reservedatt NC_reserved[NRESERVED] = { -{NC_ATT_CLASS, READONLYFLAG|DIMSCALEFLAG}, /*CLASS*/ -{NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/ -{NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/ -{NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/ -{NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ -{ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ -{NCPROPS, READONLYFLAG|NAMEONLYFLAG}, /*_NCProperties*/ -{NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Coordinates*/ -{NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Dimid*/ -{SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/ -{NC3_STRICT_ATT_NAME, READONLYFLAG}, /*_nc3_strict*/ + {NC_ATT_CLASS, READONLYFLAG|DIMSCALEFLAG}, /*CLASS*/ + {NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/ + {NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/ + {NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/ + {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/ + {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/ + {NCPROPS, READONLYFLAG|NAMEONLYFLAG}, /*_NCProperties*/ + {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Coordinates*/ + {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*_Netcdf4Dimid*/ + {SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/ + {NC3_STRICT_ATT_NAME, READONLYFLAG}, /*_nc3_strict*/ }; /** @@ -92,442 +59,27 @@ static const NC_reservedatt NC_reserved[NRESERVED] = { const NC_reservedatt* NC_findreserved(const char* name) { - int n = NRESERVED; - int L = 0; - int R = (n - 1); - for(;;) { - if(L > R) break; - int m = (L + R) / 2; - const NC_reservedatt* p = &NC_reserved[m]; - int cmp = strcmp(p->name,name); - if(cmp == 0) return p; - if(cmp < 0) - L = (m + 1); - else /*cmp > 0*/ - R = (m - 1); - } - return NULL; + int n = NRESERVED; + int L = 0; + int R = (n - 1); + for(;;) { + if(L > R) break; + int m = (L + R) / 2; + const NC_reservedatt* p = &NC_reserved[m]; + int cmp = strcmp(p->name,name); + if(cmp == 0) return p; + if(cmp < 0) + L = (m + 1); + else /*cmp > 0*/ + R = (m - 1); + } + return NULL; } -/** - * @internal Given an HDF5 type, set a pointer to netcdf type. - * - * @param h5 Pointer to HDF5 file info struct. - * @param native_typeid HDF5 type ID. - * @param xtype Pointer that gets netCDF type. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_EHDFERR HDF5 returned error. - * @return ::NC_EBADTYPID Type not found. - * @author Ed Hartnett -*/ -static int -get_netcdf_type(NC_FILE_INFO_T *h5, hid_t native_typeid, - nc_type *xtype) -{ - NC_TYPE_INFO_T *type; - H5T_class_t class; - htri_t is_str, equal = 0; - - assert(h5 && xtype); - - if ((class = H5Tget_class(native_typeid)) < 0) - return NC_EHDFERR; - - /* H5Tequal doesn't work with H5T_C_S1 for some reason. But - * H5Tget_class will return H5T_STRING if this is a string. */ - if (class == H5T_STRING) - { - if ((is_str = H5Tis_variable_str(native_typeid)) < 0) - return NC_EHDFERR; - if (is_str) - *xtype = NC_STRING; - else - *xtype = NC_CHAR; - return NC_NOERR; - } - else if (class == H5T_INTEGER || class == H5T_FLOAT) - { - /* For integers and floats, we don't have to worry about - * endianness if we compare native types. */ - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_BYTE; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_SHORT; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_INT; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_FLOAT; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_DOUBLE; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_UBYTE; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_USHORT; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_UINT; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_INT64; - return NC_NOERR; - } - if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0) - return NC_EHDFERR; - if (equal) - { - *xtype = NC_UINT64; - return NC_NOERR; - } - } - - /* Maybe we already know about this type. */ - if (!equal) - if((type = nc4_rec_find_hdf_type(h5, native_typeid))) - { - *xtype = type->hdr.id; - return NC_NOERR; - } - - *xtype = NC_NAT; - return NC_EBADTYPID; -} - -/** - * @internal Read an attribute. This is called by att_read_var_callbk(). - * - * @param grp Pointer to group info struct. - * @param attid Attribute ID. - * @param att Pointer that gets att info struct. - * - * @return ::NC_NOERR No error. - * @return ::NC_EHDFERR HDF5 returned error. - * @author Ed Hartnett -*/ -static int -read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) -{ - hid_t spaceid = 0, file_typeid = 0; - hsize_t dims[1] = {0}; /* netcdf attributes always 1-D. */ - int retval = NC_NOERR; - size_t type_size; - int att_ndims; - hssize_t att_npoints; - H5T_class_t att_class; - int fixed_len_string = 0; - size_t fixed_size = 0; - - assert(att->hdr.name); - LOG((5, "%s: att->hdr.id %d att->hdr.name %s att->nc_typeid %d att->len %d", - __func__, att->hdr.id, att->hdr.name, (int)att->nc_typeid, att->len)); - - /* Get type of attribute in file. */ - if ((file_typeid = H5Aget_type(attid)) < 0) - return NC_EATTMETA; - if ((att->native_hdf_typeid = H5Tget_native_type(file_typeid, H5T_DIR_DEFAULT)) < 0) - BAIL(NC_EHDFERR); - if ((att_class = H5Tget_class(att->native_hdf_typeid)) < 0) - BAIL(NC_EATTMETA); - if (att_class == H5T_STRING && !H5Tis_variable_str(att->native_hdf_typeid)) - { - fixed_len_string++; - if (!(fixed_size = H5Tget_size(att->native_hdf_typeid))) - BAIL(NC_EATTMETA); - } - if ((retval = get_netcdf_type(grp->nc4_info, att->native_hdf_typeid, &(att->nc_typeid)))) - BAIL(retval); - - - /* Get len. */ - if ((spaceid = H5Aget_space(attid)) < 0) - BAIL(NC_EATTMETA); - if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0) - BAIL(NC_EATTMETA); - if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) - BAIL(NC_EATTMETA); - - /* If both att_ndims and att_npoints are zero, then this is a - * zero length att. */ - if (att_ndims == 0 && att_npoints == 0) - dims[0] = 0; - else if (att->nc_typeid == NC_STRING) - dims[0] = att_npoints; - else if (att->nc_typeid == NC_CHAR) - { - /* NC_CHAR attributes are written as a scalar in HDF5, of type - * H5T_C_S1, of variable length. */ - if (att_ndims == 0) - { - if (!(dims[0] = H5Tget_size(file_typeid))) - BAIL(NC_EATTMETA); - } - else - { - /* This is really a string type! */ - att->nc_typeid = NC_STRING; - dims[0] = att_npoints; - } - } - else - { - H5S_class_t space_class; - - /* All netcdf attributes are scalar or 1-D only. */ - if (att_ndims > 1) - BAIL(NC_EATTMETA); - - /* Check class of HDF5 dataspace */ - if ((space_class = H5Sget_simple_extent_type(spaceid)) < 0) - BAIL(NC_EATTMETA); - - /* Check for NULL HDF5 dataspace class (should be weeded out earlier) */ - if (H5S_NULL == space_class) - BAIL(NC_EATTMETA); - - /* check for SCALAR HDF5 dataspace class */ - if (H5S_SCALAR == space_class) - dims[0] = 1; - else /* Must be "simple" dataspace */ - { - /* Read the size of this attribute. */ - if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0) - BAIL(NC_EATTMETA); - } - } - - /* Tell the user what the length if this attribute is. */ - att->len = dims[0]; - - /* Allocate some memory if the len is not zero, and read the - attribute. */ - if (dims[0]) - { - if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->nc_typeid, &type_size))) - return retval; - if (att_class == H5T_VLEN) - { - if (!(att->vldata = malloc((unsigned int)(att->len * sizeof(hvl_t))))) - BAIL(NC_ENOMEM); - if (H5Aread(attid, att->native_hdf_typeid, att->vldata) < 0) - BAIL(NC_EATTMETA); - } - else if (att->nc_typeid == NC_STRING) - { - if (!(att->stdata = calloc(att->len, sizeof(char *)))) - BAIL(NC_ENOMEM); - /* For a fixed length HDF5 string, the read requires - * contiguous memory. Meanwhile, the netCDF API requires that - * nc_free_string be called on string arrays, which would not - * work if one contiguous memory block were used. So here I - * convert the contiguous block of strings into an array of - * malloced strings (each string with its own malloc). Then I - * copy the data and free the contiguous memory. This - * involves copying the data, which is bad, but this only - * occurs for fixed length string attributes, and presumably - * these are small. (And netCDF-4 does not create them - it - * always uses variable length strings. */ - if (fixed_len_string) - { - int i; - char *contig_buf, *cur; - - /* Alloc space for the contiguous memory read. */ - if (!(contig_buf = malloc(att->len * fixed_size * sizeof(char)))) - BAIL(NC_ENOMEM); - - /* Read the fixed-len strings as one big block. */ - if (H5Aread(attid, att->native_hdf_typeid, contig_buf) < 0) { - free(contig_buf); - BAIL(NC_EATTMETA); - } - - /* Copy strings, one at a time, into their new home. Alloc - space for each string. The user will later free this - space with nc_free_string. */ - cur = contig_buf; - for (i = 0; i < att->len; i++) - { - if (!(att->stdata[i] = malloc(fixed_size))) { - free(contig_buf); - BAIL(NC_ENOMEM); - } - strncpy(att->stdata[i], cur, fixed_size); - cur += fixed_size; - } - - /* Free contiguous memory buffer. */ - free(contig_buf); - } - else - { - /* Read variable-length string atts. */ - if (H5Aread(attid, att->native_hdf_typeid, att->stdata) < 0) - BAIL(NC_EATTMETA); - } - } - else - { - if (!(att->data = malloc((unsigned int)(att->len * type_size)))) - BAIL(NC_ENOMEM); - if (H5Aread(attid, att->native_hdf_typeid, att->data) < 0) - BAIL(NC_EATTMETA); - } - } - - if (H5Tclose(file_typeid) < 0) - BAIL(NC_EHDFERR); - if (H5Sclose(spaceid) < 0) - return NC_EHDFERR; - - return NC_NOERR; - -exit: - if (H5Tclose(file_typeid) < 0) - BAIL2(NC_EHDFERR); - if (spaceid > 0 && H5Sclose(spaceid) < 0) - BAIL2(NC_EHDFERR); - return retval; -} - -/** - * @internal Callback function for reading attributes. This is used by read_var(). - * - * @param loc_id HDF5 attribute ID. - * @param att_name Name of the attrigute. - * @param ainfo HDF5 info struct for attribute. - * @param att_data The attribute data. - * - * @return ::NC_NOERR No error. - * @return ::NC_EHDFERR HDF5 returned error. - * @return ::NC_ENOMEM Out of memory. - * @return ::NC_EATTMETA HDF5 can't open attribute. - * @return ::NC_EBADTYPID Can't read attribute type. - */ -static herr_t -att_read_var_callbk(hid_t loc_id, const char *att_name, const H5A_info_t *ainfo, void *att_data) -{ - - hid_t attid = 0; - int retval = NC_NOERR; - NC_ATT_INFO_T *att; - att_iter_info *att_info = (att_iter_info *)att_data; - - /* Should we ignore this attribute? */ - const NC_reservedatt* ra = NC_findreserved(att_name); - if(ra != NULL) goto exit; /* ignore */ - - /* Add to the end of the list of atts for this var. */ - if ((retval = nc4_att_list_add(att_info->var->att, att_name, &att))) - BAIL(retval); - - /* Open the att by name. */ - if ((attid = H5Aopen(loc_id, att_name, H5P_DEFAULT)) < 0) - BAIL(NC_EATTMETA); - LOG((4, "%s:: att_name %s", __func__, att_name)); - - /* Read the rest of the info about the att, - * including its values. */ - if ((retval = read_hdf5_att(att_info->grp, attid, att))) - BAIL(retval); - - if (att) - att->created = NC_TRUE; - - if (attid > 0 && H5Aclose(attid) < 0) - BAIL2(NC_EHDFERR); - - return NC_NOERR; - -exit: - if(retval) { - if (retval == NC_EBADTYPID) { - /* NC_EBADTYPID will be normally converted to NC_NOERR so that - the parent iterator does not fail. */ - retval = nc4_att_list_del(att_info->var->att,att); - att = NULL; - } - } - if (attid > 0 && H5Aclose(attid) < 0) - retval = NC_EHDFERR; - return retval; -} - -/** @internal These flags may not be set for open mode. */ -static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP|NC_64BIT_OFFSET); - -/** @internal These flags may not be set for create. */ -static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_64BIT_OFFSET|NC_CDF5); - extern void reportopenobjects(int log, hid_t); -/** - * @internal Struct to track information about objects in a group, for - * nc4_rec_read_metadata() -*/ -typedef struct NC4_rec_read_metadata_obj_info -{ - hid_t oid; /* HDF5 object ID */ - char oname[NC_MAX_NAME + 1]; /* Name of object */ - H5G_stat_t statbuf; /* Information about the object */ - struct NC4_rec_read_metadata_obj_info *next; /* Pointer to next node in list */ -} NC4_rec_read_metadata_obj_info_t; - -/** - * @internal User data struct for call to H5Literate() in - * nc4_rec_read_metadata(). Tracks the groups, named datatypes and - * datasets in the group, for later use. -*/ -typedef struct NC4_rec_read_metadata_ud -{ - NClist* grps; /* NClist */ - NC_GRP_INFO_T *grp; /* Pointer to parent group */ -} NC4_rec_read_metadata_ud_t; - /* Forward */ static int NC4_enddef(int ncid); -static int nc4_rec_read_metadata(NC_GRP_INFO_T *grp); static void dumpopenobjects(NC_FILE_INFO_T* h5); /** @@ -538,7 +90,7 @@ static void dumpopenobjects(NC_FILE_INFO_T* h5); * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ static int sync_netcdf4_file(NC_FILE_INFO_T *h5) { @@ -597,9 +149,9 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ -static int -close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) + */ +int +nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) { int retval = NC_NOERR; @@ -649,7 +201,7 @@ close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, int extractmem) retval = NC4_extract_file_image(h5); } } - + if (H5Fclose(h5->hdfid) < 0) { dumpopenobjects(h5); @@ -665,64 +217,40 @@ exit: static void dumpopenobjects(NC_FILE_INFO_T* h5) { - int nobjs; + int nobjs; - nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL); - /* Apparently we can get an error even when nobjs == 0 */ - if(nobjs < 0) { - return; - } else if(nobjs > 0) { - char msg[1024]; - int logit = 0; - /* If the close doesn't work, probably there are still some HDF5 - * objects open, which means there's a bug in the library. So - * print out some info on to help the poor programmer figure it - * out. */ - snprintf(msg,sizeof(msg),"There are %d HDF5 objects open!", nobjs); + nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL); + /* Apparently we can get an error even when nobjs == 0 */ + if(nobjs < 0) { + return; + } else if(nobjs > 0) { + char msg[1024]; + int logit = 0; + /* If the close doesn't work, probably there are still some HDF5 + * objects open, which means there's a bug in the library. So + * print out some info on to help the poor programmer figure it + * out. */ + snprintf(msg,sizeof(msg),"There are %d HDF5 objects open!", nobjs); #ifdef LOGGING #ifdef LOGOPEN - LOG((0, msg)); - logit = 1; + LOG((0, msg)); + logit = 1; #endif #else - fprintf(stdout,"%s\n",msg); - logit = 0; + fprintf(stdout,"%s\n",msg); + logit = 0; #endif - reportopenobjects(logit,h5->hdfid); - fflush(stderr); - } + reportopenobjects(logit,h5->hdfid); + fflush(stderr); + } - return; + return; } size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */ size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */ float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */ -#define NUM_TYPES 12 /**< Number of netCDF atomic types. */ - -/** @internal Native HDF5 constants for atomic types. For performance, - * fill this array only the first time, and keep it in global memory - * for each further use. */ -static hid_t h5_native_type_constant_g[NUM_TYPES]; - -/** @internal NetCDF atomic type names. */ -static const char nc_type_name_g[NUM_TYPES][NC_MAX_NAME + 1] = {"char", "byte", "short", - "int", "float", "double", "ubyte", - "ushort", "uint", "int64", - "uint64", "string"}; - -/** @internal NetCDF atomic types. */ -static const nc_type nc_type_constant_g[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, - NC_INT, NC_FLOAT, NC_DOUBLE, NC_UBYTE, - NC_USHORT, NC_UINT, NC_INT64, - NC_UINT64, NC_STRING}; - -/** @internal NetCDF atomic type sizes. */ -static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof(short), - sizeof(int), sizeof(float), sizeof(double), sizeof(unsigned char), - sizeof(unsigned short), sizeof(unsigned int), sizeof(long long), - sizeof(unsigned long long), sizeof(char *)}; /** * Set chunk cache size. Only affects files opened/created *after* it @@ -849,1924 +377,6 @@ nc4typelen(nc_type type) return -1; } -/** - * @internal Create a netCDF-4/HDF5 file. - * - * @param path The file name of the new file. - * @param cmode The creation mode flag. - * @param initialsz The proposed initial file size (advisory) - * @param parameters extra parameter info (like MPI communicator) - * @param nc Pointer to an instance of NC. - * - * @return ::NC_NOERR No error. - * @return ::NC_EINVAL Invalid input (check cmode). - * @return ::NC_EEXIST File exists and NC_NOCLOBBER used. - * @return ::NC_EHDFERR HDF5 returned error. - * @ingroup netcdf4 - * @author Ed Hartnett, Dennis Heimbigner - */ -static int -nc4_create_file(const char *path, int cmode, size_t initialsz, void* parameters, NC *nc) -{ - hid_t fcpl_id, fapl_id = -1; - unsigned flags; - FILE *fp; - int retval = NC_NOERR; - NC_FILE_INFO_T* nc4_info = NULL; - -#ifdef USE_PARALLEL4 - NC_MPI_INFO* mpiinfo = NULL; - MPI_Comm comm; - MPI_Info info; - int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ - int info_duped = 0; /* Whether the MPI Info object was duplicated */ -#endif /* !USE_PARALLEL4 */ - - assert(nc && path); - LOG((3, "%s: path %s mode 0x%x", __func__, path, cmode)); - - /* Add necessary structs to hold netcdf-4 file data. */ - if ((retval = nc4_nc4f_list_add(nc, path, (NC_WRITE | cmode)))) - BAIL(retval); - - nc4_info = NC4_DATA(nc); - assert(nc4_info && nc4_info->root_grp); - - nc4_info->mem.inmemory = (cmode & NC_INMEMORY) == NC_INMEMORY; - nc4_info->mem.diskless = (cmode & NC_DISKLESS) == NC_DISKLESS; - nc4_info->mem.created = 1; - nc4_info->mem.initialsize = initialsz; - - if(nc4_info->mem.inmemory && parameters) - nc4_info->mem.memio = *(NC_memio*)parameters; -#ifdef USE_PARALLEL4 - else if(parameters) { - mpiinfo = (NC_MPI_INFO *)parameters; - comm = mpiinfo->comm; - info = mpiinfo->info; - } -#endif - if(nc4_info->mem.diskless) - flags = H5F_ACC_TRUNC; - else if(cmode & NC_NOCLOBBER) - flags = H5F_ACC_EXCL; - else - flags = H5F_ACC_TRUNC; - - /* If this file already exists, and NC_NOCLOBBER is specified, - return an error (unless diskless|inmemory) */ - if (nc4_info->mem.diskless) { - if((cmode & NC_WRITE) && (cmode & NC_NOCLOBBER) == 0) - nc4_info->mem.persist = 1; - } else if (nc4_info->mem.inmemory) { - /* ok */ - } else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path, "r"))) { - fclose(fp); - BAIL(NC_EEXIST); - } - - /* Need this access plist to control how HDF5 handles open objects - * on file close. Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file. */ - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - BAIL(NC_EHDFERR); - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) - BAIL(NC_EHDFERR); - -#ifdef USE_PARALLEL4 - /* If this is a parallel file create, set up the file creation - property list. */ - if ((cmode & NC_MPIIO) || (cmode & NC_MPIPOSIX)) { - nc4_info->parallel = NC_TRUE; - if (cmode & NC_MPIIO) /* MPI/IO */ - { - LOG((4, "creating parallel file with MPI/IO")); - if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0) - BAIL(NC_EPARINIT); - } -#ifdef USE_PARALLEL_POSIX - else /* MPI/POSIX */ - { - LOG((4, "creating parallel file with MPI/posix")); - if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0) - BAIL(NC_EPARINIT); - } -#else /* USE_PARALLEL_POSIX */ - /* Should not happen! Code in NC4_create/NC4_open should alias the - * NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not - * available in HDF5. -QAK - */ - else /* MPI/POSIX */ - BAIL(NC_EPARINIT); -#endif /* USE_PARALLEL_POSIX */ - - /* Keep copies of the MPI Comm & Info objects */ - if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm)) - BAIL(NC_EMPI); - comm_duped++; - if (MPI_INFO_NULL != info) - { - if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info)) - BAIL(NC_EMPI); - info_duped++; - } - else - { - /* No dup, just copy it. */ - nc4_info->info = info; - } - } -#else /* only set cache for non-parallel... */ - if(cmode & NC_DISKLESS) { - if (H5Pset_fapl_core(fapl_id, 4096, nc4_info->mem.persist)) - BAIL(NC_EDISKLESS); - } - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) - BAIL(NC_EHDFERR); - LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption)); -#endif /* USE_PARALLEL4 */ - -#ifdef HDF5_HAS_LIBVER_BOUNDS -#if H5_VERSION_GE(1,10,2) - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0) -#else - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) -#endif - BAIL(NC_EHDFERR); -#endif - - /* Create the property list. */ - if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0) - BAIL(NC_EHDFERR); - - /* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */ - if (H5Pset_obj_track_times(fcpl_id,0)<0) - BAIL(NC_EHDFERR); - - /* Set latest_format in access propertly list and - * H5P_CRT_ORDER_TRACKED in the creation property list. This turns - * on HDF5 creation ordering. */ - if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | - H5P_CRT_ORDER_INDEXED)) < 0) - BAIL(NC_EHDFERR); - if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED | - H5P_CRT_ORDER_INDEXED)) < 0) - BAIL(NC_EHDFERR); - - /* Create the file. */ -#ifdef HDF5_HAS_COLL_METADATA_OPS - H5Pset_all_coll_metadata_ops(fapl_id, 1 ); - H5Pset_coll_metadata_write(fapl_id, 1); -#endif - - if(nc4_info->mem.inmemory) { -#if 0 - if(nc4_info->mem.memio.size == 0) - nc4_info->memio.size = DEFAULT_CREATE_MEMSIZE; /* last ditch fix */ - if(nc4_info->memio.memory == NULL) { /* last ditch fix */ - nc4_info->memio.memory = malloc(nc4_info->memio.size); - if(nc4_info->memio.memory == NULL) - BAIL(NC_ENOMEM); - } - assert(nc4_info->memio.size > 0 && nc4_info->memio.memory != NULL); -#endif - retval = NC4_create_image_file(nc4_info,initialsz); - if(retval) - BAIL(retval); - } else if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0) - /*Change the return error from NC_EFILEMETADATA to - System error EACCES because that is the more likely problem */ - BAIL(EACCES); - - /* Open the root group. */ - if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(nc4_info->hdfid, "/", - H5P_DEFAULT)) < 0) - BAIL(NC_EFILEMETA); - - /* Release the property lists. */ - if (H5Pclose(fapl_id) < 0 || H5Pclose(fcpl_id) < 0) - BAIL(NC_EHDFERR); - - /* Define mode gets turned on automatically on create. */ - nc4_info->flags |= NC_INDEF; - - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo))) - BAIL(retval); - - /* Write the _NCProperties attribute. */ - if ((retval = NC4_put_propattr(nc4_info))) - BAIL(retval); - - return NC_NOERR; - -exit: /*failure exit*/ -#ifdef USE_PARALLEL4 - if (comm_duped) MPI_Comm_free(&nc4_info->comm); - if (info_duped) MPI_Info_free(&nc4_info->info); -#endif - if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); - if(!nc4_info) return retval; - close_netcdf4_file(nc4_info,1,0); /* treat like abort */ - return retval; -} - -/** - * @internal Create a netCDF-4/HDF5 file. - * - * @param path The file name of the new file. - * @param cmode The creation mode flag. - * @param initialsz Ignored by this function. - * @param basepe Ignored by this function. - * @param chunksizehintp Ignored by this function. - * @param use_parallel 0 for sequential, non-zero for parallel I/O. - * @param parameters pointer to struct holding extra data (e.g. for parallel I/O) - * layer. Ignored if NULL. - * @param dispatch Pointer to the dispatch table for this file. - * @param nc_file Pointer to an instance of NC. - * - * @return ::NC_NOERR No error. - * @return ::NC_EINVAL Invalid input (check cmode). - * @ingroup netcdf4 - * @author Ed Hartnett - */ -int -NC4_create(const char* path, int cmode, size_t initialsz, int basepe, - size_t *chunksizehintp, int use_parallel, void *parameters, - NC_Dispatch *dispatch, NC* nc_file) -{ - int res; - - assert(nc_file && path); - - LOG((1, "%s: path %s cmode 0x%x parameters %p", - __func__, path, cmode, parameters)); - - /* If this is our first file, turn off HDF5 error messages. */ - if (!nc4_hdf5_initialized) - nc4_hdf5_initialize(); - - /* Check the cmode for validity. */ - if((cmode & ILLEGAL_CREATE_FLAGS) != 0) - {res = NC_EINVAL; goto done;} - - /* Cannot have both */ - if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX)) - {res = NC_EINVAL; goto done;} - - /* Currently no parallel diskless io */ - if((cmode & (NC_MPIIO | NC_MPIPOSIX)) && (cmode & NC_DISKLESS)) - {res = NC_EINVAL; goto done;} - -#ifndef USE_PARALLEL_POSIX -/* If the HDF5 library has been compiled without the MPI-POSIX VFD, alias - * the NC_MPIPOSIX flag to NC_MPIIO. -QAK - */ - if(cmode & NC_MPIPOSIX) - { - cmode &= ~NC_MPIPOSIX; - cmode |= NC_MPIIO; - } -#endif /* USE_PARALLEL_POSIX */ - - cmode |= NC_NETCDF4; - - /* Apply default create format. */ - if (nc_get_default_format() == NC_FORMAT_CDF5) - cmode |= NC_CDF5; - else if (nc_get_default_format() == NC_FORMAT_64BIT_OFFSET) - cmode |= NC_64BIT_OFFSET; - else if (nc_get_default_format() == NC_FORMAT_NETCDF4_CLASSIC) - cmode |= NC_CLASSIC_MODEL; - - LOG((2, "cmode after applying default format: 0x%x", cmode)); - - nc_file->int_ncid = nc_file->ext_ncid; - - res = nc4_create_file(path, cmode, initialsz, parameters, nc_file); - -done: - return res; -} - -/** - * @internal This function is called by read_dataset when a dimension - * scale dataset is encountered. It reads in the dimension data - * (creating a new NC_DIM_INFO_T object), and also checks to see if - * this is a dimension without a variable - that is, a coordinate - * dimension which does not have any coordinate data. - * - * @param grp Pointer to group info struct. - * @param datasetid The HDF5 dataset ID. - * @param obj_name - * @param statbuf - * @param scale_size Size of dimension scale. - * @param max_scale_size Maximum size of dim scale. - * @param dim Pointer to pointer that gets new dim info struct. - * - * @returns ::NC_NOERR No error. - * @return ::NC_EHDFERR HDF5 returned error. - * @author Ed Hartnett - */ -static int -read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, - const H5G_stat_t *statbuf, hsize_t scale_size, hsize_t max_scale_size, - NC_DIM_INFO_T **dim) -{ - NC_DIM_INFO_T *new_dim; /* Dimension added to group */ - char dimscale_name_att[NC_MAX_NAME + 1] = ""; /* Dimscale name, for checking if dim without var */ - htri_t attr_exists = -1; /* Flag indicating hidden attribute exists */ - hid_t attid = -1; /* ID of hidden attribute (to store dim ID) */ - int dimscale_created = 0; /* Remember if a dimension was created (for error recovery) */ - short initial_next_dimid = grp->nc4_info->next_dimid;/* Retain for error recovery */ - int retval; - size_t len = 0; - int too_long = NC_FALSE; - int assigned_id = -1; - - /* Does this dataset have a hidden attribute that tells us its - * dimid? If so, read it. */ - if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0) - BAIL(NC_EHDFERR); - if (attr_exists) - { - if ((attid = H5Aopen_name(datasetid, NC_DIMID_ATT_NAME)) < 0) - BAIL(NC_EHDFERR); - - if (H5Aread(attid, H5T_NATIVE_INT, &assigned_id) < 0) - BAIL(NC_EHDFERR); - - /* Check if scale's dimid should impact the group's next dimid */ - if (assigned_id >= grp->nc4_info->next_dimid) - grp->nc4_info->next_dimid = assigned_id + 1; - } - - if (SIZEOF_SIZE_T < 8 && scale_size > NC_MAX_UINT) - { - len = NC_MAX_UINT; - too_long = NC_TRUE; - } - else - len = scale_size; - - /* Create the dimension for this scale. */ - if ((retval = nc4_dim_list_add(grp, obj_name, len, assigned_id, &new_dim))) - BAIL(retval); - - new_dim->too_long = too_long; - - dimscale_created++; - - new_dim->hdf5_objid.fileno[0] = statbuf->fileno[0]; - new_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; - new_dim->hdf5_objid.objno[0] = statbuf->objno[0]; - new_dim->hdf5_objid.objno[1] = statbuf->objno[1]; - - /* If the dimscale has an unlimited dimension, then this dimension - * is unlimited. */ - if (max_scale_size == H5S_UNLIMITED) - new_dim->unlimited = NC_TRUE; - - /* If the scale name is set to DIM_WITHOUT_VARIABLE, then this is a - * dimension, but not a variable. (If get_scale_name returns an - * error, just move on, there's no NAME.) */ - if (H5DSget_scale_name(datasetid, dimscale_name_att, NC_MAX_NAME) >= 0) - { - if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE, - strlen(DIM_WITHOUT_VARIABLE))) - { - if (new_dim->unlimited) - { - size_t len = 0, *lenp = &len; - - if ((retval = nc4_find_dim_len(grp, new_dim->hdr.id, &lenp))) - BAIL(retval); - new_dim->len = *lenp; - } - - /* Hold open the dataset, since the dimension doesn't have a coordinate variable */ - new_dim->hdf_dimscaleid = datasetid; - H5Iinc_ref(new_dim->hdf_dimscaleid); /* Increment number of objects using ID */ - } - } - - /* Set the dimension created */ - *dim = new_dim; - -exit: - /* Close the hidden attribute, if it was opened (error, or no error) */ - if (attid > 0 && H5Aclose(attid) < 0) - BAIL2(NC_EHDFERR); - - /* On error, undo any dimscale creation */ - if (retval < 0 && dimscale_created) - { - /* free the dimension */ - if ((retval = nc4_dim_list_del(grp, new_dim))) - BAIL2(retval); - - /* Reset the group's information */ - grp->nc4_info->next_dimid = initial_next_dimid; - } - - return retval; -} - -/** - * @internal This function reads the hacked in coordinates attribute I - * use for multi-dimensional coordinates. - * - * @param grp Group info pointer. - * @param var Var info pointer. - * - * @return NC_NOERR No error. - * @author Ed Hartnett -*/ -static int -read_coord_dimids(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) -{ - hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1; - hssize_t npoints; - int ret = 0; - int d; - - /* There is a hidden attribute telling us the ids of the - * dimensions that apply to this multi-dimensional coordinate - * variable. Read it. */ - if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++; - if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++; - - /* How many dimensions are there? */ - if (!ret && (spaceid = H5Aget_space(coord_attid)) < 0) ret++; - if (!ret && (npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) ret++; - - /* Check that the number of points is the same as the number of dimensions - * for the variable */ - if (!ret && npoints != var->ndims) ret++; - - if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++; - LOG((4, "dimscale %s is multidimensional and has coords", var->hdr.name)); - - /* Update var->dim field based on the var->dimids */ - for (d = 0; d < var->ndims; d++) { - /* Ok if does not find a dim at this time, but if found set it */ - nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL); - } - - /* Set my HDF5 IDs free! */ - if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++; - if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++; - if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++; - return ret ? NC_EATTMETA : NC_NOERR; -} - -/** - * @internal This function is called when reading a file's metadata - * for each dimension scale attached to a variable. - * - * @param did HDF5 ID for dimscale. - * @param dim - * @param dsid - * @param dimscale_hdf5_objids - * - * @return 0 for success, -1 for error. - * @author Ed Hartnett -*/ -static herr_t -dimscale_visitor(hid_t did, unsigned dim, hid_t dsid, - void *dimscale_hdf5_objids) -{ - H5G_stat_t statbuf; - - /* Get more info on the dimscale object.*/ - if (H5Gget_objinfo(dsid, ".", 1, &statbuf) < 0) - return -1; - - /* Pass this information back to caller. */ - (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0]; - (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1]; - (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0]; - (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1]; - return 0; -} - -/** - * @internal Given an HDF5 type, set a pointer to netcdf type_info struct, - * either an existing one (for user-defined types) or a newly created - * one. - * - * @param h5 Pointer to HDF5 file info struct. - * @param datasetid HDF5 dataset ID. - * @param type_info Pointer to pointer that gets type info struct. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_EHDFERR HDF5 returned error. - * @return ::NC_EBADTYPID Type not found. - * @author Ed Hartnett -*/ -static int -get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, - NC_TYPE_INFO_T **type_info) -{ - htri_t is_str, equal = 0; - H5T_class_t class; - hid_t native_typeid, hdf_typeid; - H5T_order_t order; - int t; - - assert(h5 && type_info); - - /* Because these N5T_NATIVE_* constants are actually function calls - * (!) in H5Tpublic.h, I can't initialize this array in the usual - * way, because at least some C compilers (like Irix) complain - * about calling functions when defining constants. So I have to do - * it like this. Note that there's no native types for char or - * string. Those are handled later. */ - if (!h5_native_type_constant_g[1]) - { - h5_native_type_constant_g[1] = H5T_NATIVE_SCHAR; - h5_native_type_constant_g[2] = H5T_NATIVE_SHORT; - h5_native_type_constant_g[3] = H5T_NATIVE_INT; - h5_native_type_constant_g[4] = H5T_NATIVE_FLOAT; - h5_native_type_constant_g[5] = H5T_NATIVE_DOUBLE; - h5_native_type_constant_g[6] = H5T_NATIVE_UCHAR; - h5_native_type_constant_g[7] = H5T_NATIVE_USHORT; - h5_native_type_constant_g[8] = H5T_NATIVE_UINT; - h5_native_type_constant_g[9] = H5T_NATIVE_LLONG; - h5_native_type_constant_g[10] = H5T_NATIVE_ULLONG; - } - - /* Get the HDF5 typeid - we'll need it later. */ - if ((hdf_typeid = H5Dget_type(datasetid)) < 0) - return NC_EHDFERR; - - /* Get the native typeid. Will be equivalent to hdf_typeid when - * creating but not necessarily when reading, a variable. */ - if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0) - return NC_EHDFERR; - - /* Is this type an integer, string, compound, or what? */ - if ((class = H5Tget_class(native_typeid)) < 0) - return NC_EHDFERR; - - /* Is this an atomic type? */ - if (class == H5T_STRING || class == H5T_INTEGER || class == H5T_FLOAT) - { - /* Allocate a phony NC_TYPE_INFO_T struct to hold type info. */ - if (!(*type_info = calloc(1, sizeof(NC_TYPE_INFO_T)))) - return NC_ENOMEM; - - /* H5Tequal doesn't work with H5T_C_S1 for some reason. But - * H5Tget_class will return H5T_STRING if this is a string. */ - if (class == H5T_STRING) - { - if ((is_str = H5Tis_variable_str(native_typeid)) < 0) - return NC_EHDFERR; - /* Make sure fixed-len strings will work like variable-len strings */ - if (is_str || H5Tget_size(hdf_typeid) > 1) - { - /* Set a class for the type */ - t = NUM_TYPES - 1; - (*type_info)->nc_type_class = NC_STRING; - } - else - { - /* Set a class for the type */ - t = 0; - (*type_info)->nc_type_class = NC_CHAR; - } - } - else - { - for (t = 1; t < NUM_TYPES - 1; t++) - { - if ((equal = H5Tequal(native_typeid, h5_native_type_constant_g[t])) < 0) - return NC_EHDFERR; - if (equal) - break; - } - - /* Find out about endianness. - * As of HDF 1.8.6, this works with all data types - * Not just H5T_INTEGER. - * - * See https://www.hdfgroup.org/HDF5/doc/RM/RM_H5T.html#Datatype-GetOrder - */ - if((order = H5Tget_order(hdf_typeid)) < 0) - return NC_EHDFERR; - - if(order == H5T_ORDER_LE) - (*type_info)->endianness = NC_ENDIAN_LITTLE; - else if(order == H5T_ORDER_BE) - (*type_info)->endianness = NC_ENDIAN_BIG; - else - return NC_EBADTYPE; - - if(class == H5T_INTEGER) - (*type_info)->nc_type_class = NC_INT; - else - (*type_info)->nc_type_class = NC_FLOAT; - } - (*type_info)->hdr.id = nc_type_constant_g[t]; - (*type_info)->size = nc_type_size_g[t]; - if (!((*type_info)->hdr.name = strdup(nc_type_name_g[t]))) - return NC_ENOMEM; - (*type_info)->hdf_typeid = hdf_typeid; - (*type_info)->native_hdf_typeid = native_typeid; - return NC_NOERR; - } - else - { - NC_TYPE_INFO_T *type; - - /* This is a user-defined type. */ - if((type = nc4_rec_find_hdf_type(h5, native_typeid))) - *type_info = type; - - /* The type entry in the array of user-defined types already has - * an open data typeid (and native typeid), so close the ones we - * opened above. */ - if (H5Tclose(native_typeid) < 0) - return NC_EHDFERR; - if (H5Tclose(hdf_typeid) < 0) - return NC_EHDFERR; - - if (type) - return NC_NOERR; - } - - return NC_EBADTYPID; -} - -/** - * @internal Read information about a user defined type from the HDF5 - * file, and stash it in the group's list of types. - * - * @param grp Pointer to group info struct. - * @param hdf_typeid HDF5 type ID. - * @param type_name Pointer that gets the type name. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_EHDFERR HDF5 returned error. - * @return ::NC_EBADTYPID Type not found. - * @author Ed Hartnett -*/ -static int -read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) -{ - NC_TYPE_INFO_T *type; - H5T_class_t class; - hid_t native_typeid; - size_t type_size; - int retval = NC_NOERR; - int nmembers; - - assert(grp && type_name); - - LOG((4, "%s: type_name %s grp->hdr.name %s", __func__, type_name, grp->hdr.name)); - - /* What is the native type for this platform? */ - if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0) - return NC_EHDFERR; - - /* What is the size of this type on this platform. */ - if (!(type_size = H5Tget_size(native_typeid))) - return NC_EHDFERR; - LOG((5, "type_size %d", type_size)); - - /* Add to the list for this new type, and get a local pointer to it. */ - if ((retval = nc4_type_list_add(grp, type_size, type_name, &type))) - return retval; - - /* Remember common info about this type. */ - type->committed = NC_TRUE; - type->hdf_typeid = hdf_typeid; - H5Iinc_ref(type->hdf_typeid); /* Increment number of objects using ID */ - type->native_hdf_typeid = native_typeid; - - /* What is the class of this type, compound, vlen, etc. */ - if ((class = H5Tget_class(hdf_typeid)) < 0) - return NC_EHDFERR; - switch (class) - { - case H5T_STRING: - type->nc_type_class = NC_STRING; - break; - - case H5T_COMPOUND: - { - int nmembers; - unsigned int m; - char* member_name = NULL; -#ifdef JNA - char jna[1001]; -#endif - - type->nc_type_class = NC_COMPOUND; - - if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0) - return NC_EHDFERR; - LOG((5, "compound type has %d members", nmembers)); - type->u.c.field = nclistnew(); - nclistsetalloc(type->u.c.field,nmembers); - - for (m = 0; m < nmembers; m++) - { - hid_t member_hdf_typeid; - hid_t member_native_typeid; - size_t member_offset; - H5T_class_t mem_class; - nc_type member_xtype; - - /* Get the typeid and native typeid of this member of the - * compound type. */ - if ((member_hdf_typeid = H5Tget_member_type(type->native_hdf_typeid, m)) < 0) - return NC_EHDFERR; - - if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, H5T_DIR_DEFAULT)) < 0) - return NC_EHDFERR; - - /* Get the name of the member.*/ - member_name = H5Tget_member_name(type->native_hdf_typeid, m); - if (!member_name || strlen(member_name) > NC_MAX_NAME) { - retval = NC_EBADNAME; - break; - } -#ifdef JNA - else { - strncpy(jna,member_name,1000); - member_name = jna; - } -#endif - - /* Offset in bytes on *this* platform. */ - member_offset = H5Tget_member_offset(type->native_hdf_typeid, m); - - /* Get dimensional data if this member is an array of something. */ - if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0) - return NC_EHDFERR; - if (mem_class == H5T_ARRAY) - { - int ndims, dim_size[NC_MAX_VAR_DIMS]; - hsize_t dims[NC_MAX_VAR_DIMS]; - int d; - - if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0) { - retval = NC_EHDFERR; - break; - } - if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims) { - retval = NC_EHDFERR; - break; - } - for (d = 0; d < ndims; d++) - dim_size[d] = dims[d]; - - /* What is the netCDF typeid of this member? */ - if ((retval = get_netcdf_type(grp->nc4_info, H5Tget_super(member_hdf_typeid), - &member_xtype))) - break; - - /* Add this member to our list of fields in this compound type. */ - if ((retval = nc4_field_list_add(type, member_name, - member_offset, H5Tget_super(member_hdf_typeid), - H5Tget_super(member_native_typeid), - member_xtype, ndims, dim_size))) - break; - } - else - { - /* What is the netCDF typeid of this member? */ - if ((retval = get_netcdf_type(grp->nc4_info, member_native_typeid, - &member_xtype))) - break; - - /* Add this member to our list of fields in this compound type. */ - if ((retval = nc4_field_list_add(type, member_name, - member_offset, member_hdf_typeid, member_native_typeid, - member_xtype, 0, NULL))) - break; - } - - hdf5free(member_name); - member_name = NULL; - } - hdf5free(member_name); - member_name = NULL; - if(retval) /* error exit from loop */ - return retval; - } - break; - - case H5T_VLEN: - { - htri_t ret; - - /* For conveninence we allow user to pass vlens of strings - * with null terminated strings. This means strings are - * treated slightly differently by the API, although they are - * really just VLENs of characters. */ - if ((ret = H5Tis_variable_str(hdf_typeid)) < 0) - return NC_EHDFERR; - if (ret) - type->nc_type_class = NC_STRING; - else - { - hid_t base_hdf_typeid; - nc_type base_nc_type = NC_NAT; - - type->nc_type_class = NC_VLEN; - - /* Find the base type of this vlen (i.e. what is this a - * vlen of?) */ - if (!(base_hdf_typeid = H5Tget_super(native_typeid))) - return NC_EHDFERR; - - /* What size is this type? */ - if (!(type_size = H5Tget_size(base_hdf_typeid))) - return NC_EHDFERR; - - /* What is the netcdf corresponding type. */ - if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid, - &base_nc_type))) - return retval; - LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d", - base_hdf_typeid, type_size, base_nc_type)); - - /* Remember the base types for this vlen */ - type->u.v.base_nc_typeid = base_nc_type; - type->u.v.base_hdf_typeid = base_hdf_typeid; - } - } - break; - - case H5T_OPAQUE: - type->nc_type_class = NC_OPAQUE; - break; - - case H5T_ENUM: - { - hid_t base_hdf_typeid; - nc_type base_nc_type = NC_NAT; - void *value; - int i; - char *member_name = NULL; -#ifdef JNA - char jna[1001]; -#endif - - type->nc_type_class = NC_ENUM; - - /* Find the base type of this enum (i.e. what is this a - * enum of?) */ - if (!(base_hdf_typeid = H5Tget_super(hdf_typeid))) - return NC_EHDFERR; - /* What size is this type? */ - if (!(type_size = H5Tget_size(base_hdf_typeid))) - return NC_EHDFERR; - /* What is the netcdf corresponding type. */ - if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid, - &base_nc_type))) - return retval; - LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d", - base_hdf_typeid, type_size, base_nc_type)); - - /* Remember the base types for this enum */ - type->u.e.base_nc_typeid = base_nc_type; - type->u.e.base_hdf_typeid = base_hdf_typeid; - - /* Find out how many member are in the enum. */ - if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0) - return NC_EHDFERR; - type->u.e.enum_member = nclistnew(); - nclistsetalloc(type->u.e.enum_member,nmembers); - - /* Allocate space for one value. */ - if (!(value = calloc(1, type_size))) - return NC_ENOMEM; - - /* Read each name and value defined in the enum. */ - for (i = 0; i < nmembers; i++) - { - /* Get the name and value from HDF5. */ - if (!(member_name = H5Tget_member_name(hdf_typeid, i))) - { - retval = NC_EHDFERR; - break; - } -#ifdef JNA - strncpy(jna,member_name,1000); - member_name = jna; -#endif - - if (strlen(member_name) > NC_MAX_NAME) - { - retval = NC_EBADNAME; - break; - } - if (H5Tget_member_value(hdf_typeid, i, value) < 0) - { - retval = NC_EHDFERR; - break; - } - - /* Insert new field into this type's list of fields. */ - if ((retval = nc4_enum_member_add(type, type->size, - member_name, value))) - { - break; - } - - hdf5free(member_name); - member_name = NULL; - } - hdf5free(member_name); - member_name = NULL; - if(value) free(value); - if(retval) /* error exit from loop */ - return retval; - } - break; - - default: - LOG((0, "unknown class")); - return NC_EBADCLASS; - } - return retval; -} - -/** - * @internal This function reads all the attributes of a variable. - * - * @param grp Pointer to the group info. - * @param var Pointer to the var info. - * - * @return NC_NOERR No error. - * @author Ed Hartnett - */ -int -nc4_read_var_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) -{ - att_iter_info att_info; /* Custom iteration information */ - - /* Check inputs. */ - assert(grp && var); - - /* Assign var and grp in struct. */ - att_info.var = var; - att_info.grp = grp; - - /* Now read all the attributes of this variable, ignoring the - ones that hold HDF5 dimension scale information. */ - if ((H5Aiterate2(var->hdf_datasetid, H5_INDEX_CRT_ORDER, H5_ITER_INC, NULL, - att_read_var_callbk, &att_info)) < 0) - return NC_EATTMETA; - - /* Remember that we have read the atts for this var. */ - var->atts_not_read = 0; - - return NC_NOERR; -} - -/** - * @internal This function is called by read_dataset(), (which is called - * by nc4_rec_read_metadata()) when a netCDF variable is found in the - * file. This function reads in all the metadata about the var, - * including the attributes. - * - * @param grp Pointer to group info struct. - * @param datasetid HDF5 dataset ID. - * @param obj_name Name of the HDF5 object to read. - * @param ndims Number of dimensions. - * @param dim - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_EHDFERR HDF5 returned error. - * @author Ed Hartnett -*/ -static int -read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, - size_t ndims, NC_DIM_INFO_T *dim) -{ - NC_VAR_INFO_T *var = NULL; - hid_t access_pid = 0; - int incr_id_rc = 0; /* Whether the dataset ID's ref count has been incremented */ - int d; - H5Z_filter_t filter; - int num_filters; - unsigned int cd_values_zip[CD_NELEMS_ZLIB]; - size_t cd_nelems = CD_NELEMS_ZLIB; - hid_t propid = 0; - H5D_fill_value_t fill_status; - H5D_layout_t layout; - hsize_t chunksize[NC_MAX_VAR_DIMS] = {0}; - int retval = NC_NOERR; - double rdcc_w0; - int f; - char* finalname = NULL; - - assert(obj_name && grp); - LOG((4, "%s: obj_name %s", __func__, obj_name)); - - /* Check for a weird case: a non-coordinate variable that has the - * same name as a dimension. It's legal in netcdf, and requires - * that the HDF5 dataset name be changed. */ - if (strlen(obj_name) > strlen(NON_COORD_PREPEND) && - !strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND))) - { - /* Allocate space for the name. */ - if(finalname) {free(finalname); finalname = NULL;} - if (!(finalname = malloc(((strlen(obj_name) - strlen(NON_COORD_PREPEND))+ 1) * sizeof(char)))) - BAIL(NC_ENOMEM); - strcpy(finalname, &obj_name[strlen(NON_COORD_PREPEND)]); - } else - finalname = strdup(obj_name); - - /* Add a variable to the end of the group's var list. */ - if ((retval = nc4_var_list_add(grp,finalname,ndims,&var))) - BAIL(retval); - - /* Fill in what we already know. */ - var->hdf_datasetid = datasetid; - H5Iinc_ref(var->hdf_datasetid); /* Increment number of objects using ID */ - incr_id_rc++; /* Indicate that we've incremented the ref. count (for errors) */ - var->created = NC_TRUE; - - /* Get the current chunk cache settings. */ - if ((access_pid = H5Dget_access_plist(datasetid)) < 0) - BAIL(NC_EVARMETA); - - /* Learn about current chunk cache settings. */ - if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems), - &(var->chunk_cache_size), &rdcc_w0)) < 0) - BAIL(NC_EHDFERR); - var->chunk_cache_preemption = rdcc_w0; - - /* Find out what filters are applied to this HDF5 dataset, - * fletcher32, deflate, and/or shuffle. All other filters are - * just dumped */ - if ((propid = H5Dget_create_plist(datasetid)) < 0) - BAIL(NC_EHDFERR); - - /* Get the chunking info for non-scalar vars. */ - if ((layout = H5Pget_layout(propid)) < -1) - BAIL(NC_EHDFERR); - if (layout == H5D_CHUNKED) - { - if (H5Pget_chunk(propid, NC_MAX_VAR_DIMS, chunksize) < 0) - BAIL(NC_EHDFERR); - if (!(var->chunksizes = malloc(var->ndims * sizeof(size_t)))) - BAIL(NC_ENOMEM); - for (d = 0; d < var->ndims; d++) - var->chunksizes[d] = chunksize[d]; - } - else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT) - var->contiguous = NC_TRUE; - - /* The possible values of filter (which is just an int) can be - * found in H5Zpublic.h. */ - if ((num_filters = H5Pget_nfilters(propid)) < 0) - BAIL(NC_EHDFERR); - for (f = 0; f < num_filters; f++) - { - if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - cd_values_zip, 0, NULL, NULL)) < 0) - BAIL(NC_EHDFERR); - switch (filter) - { - case H5Z_FILTER_SHUFFLE: - var->shuffle = NC_TRUE; - break; - - case H5Z_FILTER_FLETCHER32: - var->fletcher32 = NC_TRUE; - break; - - case H5Z_FILTER_DEFLATE: - var->deflate = NC_TRUE; - if (cd_nelems != CD_NELEMS_ZLIB || cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL) - BAIL(NC_EHDFERR); - var->deflate_level = cd_values_zip[0]; - break; - - default: - var->filterid = filter; - var->nparams = cd_nelems; - if(cd_nelems == 0) - var->params = NULL; - else { - /* We have to re-read the parameters based on actual nparams */ - var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*var->nparams); - if(var->params == NULL) - BAIL(NC_ENOMEM); - if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, - var->params, 0, NULL, NULL)) < 0) - BAIL(NC_EHDFERR); - } - break; - } - } - - /* Learn all about the type of this variable. */ - if ((retval = get_type_info2(grp->nc4_info, datasetid, - &var->type_info))) - BAIL(retval); - - /* Indicate that the variable has a pointer to the type */ - var->type_info->rc++; - - /* Is there a fill value associated with this dataset? */ - if (H5Pfill_value_defined(propid, &fill_status) < 0) - BAIL(NC_EHDFERR); - - /* Get the fill value, if there is one defined. */ - if (fill_status == H5D_FILL_VALUE_USER_DEFINED) - { - /* Allocate space to hold the fill value. */ - if (!var->fill_value) - { - if (var->type_info->nc_type_class == NC_VLEN) - { - if (!(var->fill_value = malloc(sizeof(nc_vlen_t)))) - BAIL(NC_ENOMEM); - } - else if (var->type_info->nc_type_class == NC_STRING) - { - if (!(var->fill_value = malloc(sizeof(char *)))) - BAIL(NC_ENOMEM); - } - else - { - assert(var->type_info->size); - if (!(var->fill_value = malloc(var->type_info->size))) - BAIL(NC_ENOMEM); - } - } - - /* Get the fill value from the HDF5 property lust. */ - if (H5Pget_fill_value(propid, var->type_info->native_hdf_typeid, - var->fill_value) < 0) - BAIL(NC_EHDFERR); - } - else - var->no_fill = NC_TRUE; - - /* If it's a scale, mark it as such. */ - if (dim) - { - assert(ndims); - var->dimscale = NC_TRUE; - if (var->ndims > 1) - { - if ((retval = read_coord_dimids(grp, var))) - BAIL(retval); - } - else - { - /* sanity check */ - assert(0 == strcmp(var->hdr.name, dim->hdr.name)); - - var->dimids[0] = dim->hdr.id; - var->dim[0] = dim; - } - dim->coord_var = var; - } - /* If this is not a scale, but has scales, iterate - * through them. (i.e. this is a variable that is not a - * coordinate variable) */ - else - { - int num_scales = 0; - - /* Find out how many scales are attached to this - * dataset. H5DSget_num_scales returns an error if there are no - * scales, so convert a negative return value to zero. */ - num_scales = H5DSget_num_scales(datasetid, 0); - if (num_scales < 0) - num_scales = 0; - - if (num_scales && ndims) - { - /* Allocate space to remember whether the dimscale has been attached - * for each dimension. */ - if (NULL == (var->dimscale_attached = calloc(ndims, sizeof(nc_bool_t)))) - BAIL(NC_ENOMEM); - - /* Store id information allowing us to match hdf5 - * dimscales to netcdf dimensions. */ - if (NULL == (var->dimscale_hdf5_objids = malloc(ndims * sizeof(struct hdf5_objid)))) - BAIL(NC_ENOMEM); - for (d = 0; d < var->ndims; d++) - { - if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor, - &(var->dimscale_hdf5_objids[d])) < 0) - BAIL(NC_EHDFERR); - var->dimscale_attached[d] = NC_TRUE; - } - } - } - - /* Read variable attributes. */ - var->atts_not_read = 1; - /* if ((retval = nc4_read_var_atts(grp, var))) */ - /* BAIL(retval); */ - - /* Is this a deflated variable with a chunksize greater than the - * current cache size? */ - if ((retval = nc4_adjust_var_cache(grp, var))) - BAIL(retval); - -exit: - if(finalname) free(finalname); - if (retval) - { - if (incr_id_rc && H5Idec_ref(datasetid) < 0) - BAIL2(NC_EHDFERR); - if (var != NULL) { - nc4_var_list_del(grp,var); - } - } - if (access_pid && H5Pclose(access_pid) < 0) - BAIL2(NC_EHDFERR); - if (propid > 0 && H5Pclose(propid) < 0) - BAIL2(NC_EHDFERR); - return retval; -} - -/** - * @internal This function is called by nc4_rec_read_metadata to read - * all the group level attributes (the NC_GLOBAL atts for this - * group). - * - * @param grp Pointer to group info struct. - * - * @return ::NC_NOERR No error. - * @return ::NC_EHDFERR HDF5 returned error. - * @author Ed Hartnett -*/ -int -nc4_read_grp_atts(NC_GRP_INFO_T *grp) -{ - hid_t attid = -1; - hsize_t num_obj, i; - NC_ATT_INFO_T *att; - NC_TYPE_INFO_T *type; - char obj_name[NC_MAX_HDF5_NAME + 1]; - int retval = NC_NOERR; - int hidden = 0; - - num_obj = H5Aget_num_attrs(grp->hdf_grpid); - for (i = 0; i < num_obj; i++) - { - if ((attid = H5Aopen_idx(grp->hdf_grpid, (unsigned int)i)) < 0) - BAIL(NC_EATTMETA); - if (H5Aget_name(attid, NC_MAX_NAME + 1, obj_name) < 0) - BAIL(NC_EATTMETA); - LOG((3, "reading attribute of _netCDF group, named %s", obj_name)); - - /* See if this a hidden, global attribute */ - hidden = 0; /* default */ - if(grp->nc4_info->root_grp == grp) { - const NC_reservedatt* ra = NC_findreserved(obj_name); - if(ra != NULL && (ra->flags & NAMEONLYFLAG)) - hidden = 1; - } - - /* This may be an attribute telling us that strict netcdf-3 - * rules are in effect. If so, we will make note of the fact, - * but not add this attribute to the metadata. It's not a user - * attribute, but an internal netcdf-4 one. */ - if(strcmp(obj_name, NC3_STRICT_ATT_NAME)==0) - grp->nc4_info->cmode |= NC_CLASSIC_MODEL; - else if(!hidden) { - /* Add an att struct at the end of the list, and then go to it. */ - if ((retval = nc4_att_list_add(grp->att, obj_name, &att))) - BAIL(retval); - retval = read_hdf5_att(grp, attid, att); - if(retval == NC_EBADTYPID) { - if((retval = nc4_att_list_del(grp->att, att))) - BAIL(retval); - } else if(retval) { - BAIL(retval); - } else { - att->created = NC_TRUE; - if ((retval = nc4_find_type(grp->nc4_info, att->nc_typeid, &type))) - BAIL(retval); - } - } - /* Unconditionally close the open attribute */ - H5Aclose(attid); - attid = -1; - } - - /* Remember that we have read the atts for this group. */ - grp->atts_not_read = 0; - -exit: - if (attid > 0) { - if(H5Aclose(attid) < 0) - BAIL2(NC_EHDFERR); - } - return retval; -} - -/** - * @internal This function is called when nc4_rec_read_metadata - * encounters an HDF5 dataset when reading a file. - * - * @param grp Pointer to group info struct. - * @param datasetid HDF5 dataset ID. - * @param obj_name Object name. - * @param statbuf HDF5 status buffer. - * - * @return ::NC_NOERR No error. - * @return ::NC_EBADID Bad ncid. - * @return ::NC_EHDFERR HDF5 returned error. - * @author Ed Hartnett -*/ -static int -read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, - const H5G_stat_t *statbuf) -{ - NC_DIM_INFO_T *dim = NULL; /* Dimension created for scales */ - hid_t spaceid = 0; - int ndims; - htri_t is_scale; - int retval = NC_NOERR; - - /* Get the dimension information for this dataset. */ - if ((spaceid = H5Dget_space(datasetid)) < 0) - BAIL(NC_EHDFERR); - if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0) - BAIL(NC_EHDFERR); - - /* Is this a dimscale? */ - if ((is_scale = H5DSis_scale(datasetid)) < 0) - BAIL(NC_EHDFERR); - if (is_scale) - { - hsize_t dims[H5S_MAX_RANK]; - hsize_t max_dims[H5S_MAX_RANK]; - - /* Query the scale's size & max. size */ - if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0) - BAIL(NC_EHDFERR); - - /* Read the scale information. */ - if ((retval = read_scale(grp, datasetid, obj_name, statbuf, dims[0], - max_dims[0], &dim))) - BAIL(retval); - } - - /* Add a var to the linked list, and get its metadata, - * unless this is one of those funny dimscales that are a - * dimension in netCDF but not a variable. (Spooky!) */ - if (NULL == dim || (dim && !dim->hdf_dimscaleid)) - if ((retval = read_var(grp, datasetid, obj_name, ndims, dim))) - BAIL(retval); - -exit: - if (spaceid && H5Sclose(spaceid) <0) - BAIL2(retval); - - return retval; -} - -/** - * @internal Add callback function to list. - * - * @param udata - the callback state - * @param oinfo The object info. - * - * @return ::NC_NOERR No error. - * @return ::NC_ENOMEM Out of memory. - * @author Ed Hartnett - */ -static int -nc4_rec_read_metadata_cb_list_add(NC4_rec_read_metadata_ud_t* udata, - const NC4_rec_read_metadata_obj_info_t *oinfo) -{ - NC4_rec_read_metadata_obj_info_t *new_oinfo; /* Pointer to info for object */ - - /* Allocate memory for the object's info */ - if (!(new_oinfo = calloc(1, sizeof(*new_oinfo)))) - return NC_ENOMEM; - - /* Make a copy of the object's info */ - memcpy(new_oinfo, oinfo, sizeof(*oinfo)); - - nclistpush(udata->grps,new_oinfo); - return (NC_NOERR); -} - -/** - * @internal Callback function called from nc4_rec_read_metadata(). - * - * @param grpid HDF5 group ID. - * @param name Name of object. - * @param info Info struct for object. - * @param _op_data Pointer to data. - * - * @return ::NC_NOERR No error. - * @return H5_ITER_ERROR HDF5 error. - * @author Ed Hartnett - */ -static int -nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, - void *_op_data) -{ - NC4_rec_read_metadata_ud_t *udata = (NC4_rec_read_metadata_ud_t *)_op_data; /* Pointer to user data for callback */ - NC4_rec_read_metadata_obj_info_t oinfo; /* Pointer to info for object */ - int retval = H5_ITER_CONT; - - /* Reset the memory for the object's info */ - memset(&oinfo, 0, sizeof(oinfo)); - - /* Open this critter. */ - if ((oinfo.oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0) - BAIL(H5_ITER_ERROR); - - /* Get info about the object.*/ - if (H5Gget_objinfo(oinfo.oid, ".", 1, &oinfo.statbuf) < 0) - BAIL(H5_ITER_ERROR); - - strncpy(oinfo.oname, name, NC_MAX_NAME); - - /* Add object to list, for later */ - switch(oinfo.statbuf.type) - { - case H5G_GROUP: - LOG((3, "found group %s", oinfo.oname)); - - /* Defer descending into child group immediately, so that the types - * in the current group can be processed and be ready for use by - * vars in the child group(s). - */ - if (nc4_rec_read_metadata_cb_list_add(udata, &oinfo)) - BAIL(H5_ITER_ERROR); - break; - - case H5G_DATASET: - LOG((3, "found dataset %s", oinfo.oname)); - - /* Learn all about this dataset, which may be a dimscale - * (i.e. dimension metadata), or real data. */ - if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, &oinfo.statbuf))) - { - /* Allow NC_EBADTYPID to transparently skip over datasets - * which have a datatype that netCDF-4 doesn't undertand - * (currently), but break out of iteration for other - * errors. - */ - if(NC_EBADTYPID != retval) - BAIL(H5_ITER_ERROR); - else - retval = H5_ITER_CONT; - } - - /* Close the object */ - if (H5Oclose(oinfo.oid) < 0) - BAIL(H5_ITER_ERROR); - break; - - case H5G_TYPE: - LOG((3, "found datatype %s", oinfo.oname)); - - /* Process the named datatype */ - if (read_type(udata->grp, oinfo.oid, oinfo.oname)) - BAIL(H5_ITER_ERROR); - - /* Close the object */ - if (H5Oclose(oinfo.oid) < 0) - BAIL(H5_ITER_ERROR); - break; - - default: - LOG((0, "Unknown object class %d in %s!", oinfo.statbuf.type, __func__)); - BAIL(H5_ITER_ERROR); - } - -exit: - if (retval) - { - if (oinfo.oid > 0 && H5Oclose(oinfo.oid) < 0) - BAIL2(H5_ITER_ERROR); - } - - return (retval); -} - -/** - * @internal This is the main function to recursively read all the - * metadata for the file. The links in the 'grp' are iterated over - * and added to the file's metadata information. Note that child - * groups are not immediately processed, but are deferred until all - * the other links in the group are handled (so that vars in the child - * groups are guaranteed to have types that they use in a parent group - * in place). - * - * @param grp Pointer to a group. - * - * @return ::NC_NOERR No error. - * @author Ed Hartnett - */ -static int -nc4_rec_read_metadata(NC_GRP_INFO_T *grp) -{ - NC4_rec_read_metadata_ud_t udata; /* User data for iteration */ - NC4_rec_read_metadata_obj_info_t *oinfo; /* Pointer to info for object */ - hsize_t idx=0; - hid_t pid = 0; - unsigned crt_order_flags = 0; - H5_index_t iter_index; - int i, retval = NC_NOERR; /* everything worked! */ - - assert(grp && grp->hdr.name); - LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name)); - - /* Portably initialize user data for later */ - memset(&udata, 0, sizeof(udata)); - - /* Open this HDF5 group and retain its grpid. It will remain open - * with HDF5 until this file is nc_closed. */ - if (!grp->hdf_grpid) - { - if (grp->parent) - { - if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid, - grp->hdr.name, H5P_DEFAULT)) < 0) - BAIL(NC_EHDFERR); - } - else - { - if ((grp->hdf_grpid = H5Gopen2(grp->nc4_info->hdfid, - "/", H5P_DEFAULT)) < 0) - BAIL(NC_EHDFERR); - } - } - assert(grp->hdf_grpid > 0); - - /* Get the group creation flags, to check for creation ordering */ - pid = H5Gget_create_plist(grp->hdf_grpid); - H5Pget_link_creation_order(pid, &crt_order_flags); - if (H5Pclose(pid) < 0) - BAIL(NC_EHDFERR); - - /* Set the iteration index to use */ - if (crt_order_flags & H5P_CRT_ORDER_TRACKED) - iter_index = H5_INDEX_CRT_ORDER; - else - { - NC_FILE_INFO_T *h5 = grp->nc4_info; - - /* Without creation ordering, file must be read-only. */ - if (!h5->no_write) - BAIL(NC_ECANTWRITE); - - iter_index = H5_INDEX_NAME; - } - - /* Set user data for iteration */ - udata.grp = grp; - udata.grps = nclistnew(); - - /* Iterate over links in this group, building lists for the types, - * datasets and groups encountered - */ - if (H5Literate(grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, - nc4_rec_read_metadata_cb, (void *)&udata) < 0) - BAIL(NC_EHDFERR); - - /* Process the child groups found */ - /* (Deferred until now, so that the types in the current group get - * processed and are available for vars in the child group(s).) - */ - for(i=0;ioname, &child_grp))) - BAIL(retval); - - /* Recursively read the child group's metadata */ - if ((retval = nc4_rec_read_metadata(child_grp))) - BAIL(retval); - - /* Close the object */ - if (H5Oclose(oinfo->oid) < 0) - BAIL(NC_EHDFERR); - } - - /* Defer the reading of global atts until someone asks for one. */ - grp->atts_not_read = 1; - /* if ((retval = nc4_read_grp_atts(grp))) */ - /* return retval; */ - - /* when exiting define mode, mark all variable written */ - for (i=0; ivars); i++) { - NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i); - if(var == NULL) continue; - var->written_to = NC_TRUE; - } - -exit: - /* Clean up local information, if anything remains */ - for(i=0;ioid) < 0) - BAIL2(NC_EHDFERR); - } - free(oinfo); - } - nclistfree(udata.grps); - udata.grps = NULL; - - return retval; -} - -/** - * @internal Check for the attribute that indicates that netcdf - * classic model is in use. - * - * @param root_grp pointer to the group info for the root group of the - * @param is_classic store 1 if this is a classic file. - * file. - * - * @return NC_NOERR No error. - * @author Ed Hartnett - */ -static int -check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) -{ - htri_t attr_exists = -1; - - /* Check inputs. */ - assert(!root_grp->parent && is_classic); - - /* If this attribute exists in the root group, then classic model - * is in effect. */ - if ((attr_exists = H5Aexists(root_grp->hdf_grpid, NC3_STRICT_ATT_NAME)) < 0) - return NC_EHDFERR; - *is_classic = attr_exists ? 1 : 0; - - return NC_NOERR; -} - -/** - * @internal Open a netcdf-4 file. Things have already been kicked off - * in ncfunc.c in nc_open, but here the netCDF-4 part of opening a - * file is handled. - * - * @param path The file name of the new file. - * @param mode The open mode flag. - * @param parameters File parameters. - * @param nc Pointer to NC file info. - * - * @return ::NC_NOERR No error. - * @author Ed Hartnett, Dennis Heimbigner -*/ -static int -nc4_open_file(const char *path, int mode, void* parameters, NC *nc) -{ - hid_t fapl_id = H5P_DEFAULT; - int retval; - unsigned flags; - NC_FILE_INFO_T *nc4_info = NULL; - int is_classic; - -#ifdef USE_PARALLEL4 - NC_MPI_INFO* mpiinfo = NULL; - int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ - int info_duped = 0; /* Whether the MPI Info object was duplicated */ -#endif - - LOG((3, "%s: path %s mode %d", __func__, path, mode)); - assert(path && nc); - - flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY; - - /* Add necessary structs to hold netcdf-4 file data. */ - if ((retval = nc4_nc4f_list_add(nc, path, mode))) - BAIL(retval); - nc4_info = NC4_DATA(nc); - assert(nc4_info && nc4_info->root_grp); - - nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); - nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); - if(nc4_info->mem.inmemory) { - NC_memio* memparams = NULL; - if(parameters == NULL) - BAIL(NC_EINMEMORY); - memparams = (NC_memio*)parameters; - nc4_info->mem.memio = *memparams; /* keep local copy */ - /* As a safeguard, if !locked and NC_WRITE is set, - then we must take control of the incoming memory */ - nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; - if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) { - memparams->memory = NULL; - } -#ifdef USE_PARALLEL4 - } else { - mpiinfo = (NC_MPI_INFO*)parameters; -#endif /* !USE_PARALLEL4 */ - } - - /* Need this access plist to control how HDF5 handles open objects - * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to - * fail if there are any open objects in the file. */ - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) - BAIL(NC_EHDFERR); - - if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) - BAIL(NC_EHDFERR); - -#ifdef USE_PARALLEL4 - /* If this is a parallel file create, set up the file creation - property list. */ - if (mode & NC_MPIIO || mode & NC_MPIPOSIX) - { - nc4_info->parallel = NC_TRUE; - if (mode & NC_MPIIO) /* MPI/IO */ - { - LOG((4, "opening parallel file with MPI/IO")); - if (H5Pset_fapl_mpio(fapl_id, mpiinfo->comm, mpiinfo->info) < 0) - BAIL(NC_EPARINIT); - } -#ifdef USE_PARALLEL_POSIX - else /* MPI/POSIX */ - { - LOG((4, "opening parallel file with MPI/posix")); - if (H5Pset_fapl_mpiposix(fapl_id, mpiinfo->comm, 0) < 0) - BAIL(NC_EPARINIT); - } -#else /* USE_PARALLEL_POSIX */ - /* Should not happen! Code in NC4_create/NC4_open should alias the - * NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not - * available in HDF5. -QAK - */ - else /* MPI/POSIX */ - BAIL(NC_EPARINIT); -#endif /* USE_PARALLEL_POSIX */ - - /* Keep copies of the MPI Comm & Info objects */ - if (MPI_SUCCESS != MPI_Comm_dup(mpiinfo->comm, &nc4_info->comm)) - BAIL(NC_EMPI); - comm_duped++; - if (MPI_INFO_NULL != mpiinfo->info) - { - if (MPI_SUCCESS != MPI_Info_dup(mpiinfo->info, &nc4_info->info)) - BAIL(NC_EMPI); - info_duped++; - } - else - { - /* No dup, just copy it. */ - nc4_info->info = mpiinfo->info; - } - } -#else /* only set cache for non-parallel. */ - if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, - nc4_chunk_cache_preemption) < 0) - BAIL(NC_EHDFERR); - LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", - __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption)); -#endif /* USE_PARALLEL4 */ - - /* The NetCDF-3.x prototype contains an mode option NC_SHARE for - multiple processes accessing the dataset concurrently. As there - is no HDF5 equivalent, NC_SHARE is treated as NC_NOWRITE. */ -#ifdef HDF5_HAS_COLL_METADATA_OPS - H5Pset_all_coll_metadata_ops(fapl_id, 1 ); -#endif - - /* Does the mode specify that this file is read-only? */ - if ((mode & NC_WRITE) == 0) - nc4_info->no_write = NC_TRUE; - - if(nc4_info->mem.inmemory) { - /* validate */ - if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL) - BAIL(NC_INMEMORY); - retval = NC4_open_image_file(nc4_info); - if(retval) - BAIL(NC_EHDFERR); - } else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0) - BAIL(NC_EHDFERR); - - /* Now read in all the metadata. Some types and dimscale - * information may be difficult to resolve here, if, for example, a - * dataset of user-defined type is encountered before the - * definition of that type. */ - if ((retval = nc4_rec_read_metadata(nc4_info->root_grp))) - BAIL(retval); - - /* Check for classic model attribute. */ - if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) - BAIL(retval); - if (is_classic) - nc4_info->cmode |= NC_CLASSIC_MODEL; - - /* Now figure out which netCDF dims are indicated by the dimscale - * information. */ - if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp))) - BAIL(retval); - -#ifdef LOGGING - /* This will print out the names, types, lens, etc of the vars and - atts in the file, if the logging level is 2 or greater. */ - log_metadata_nc(nc); -#endif - - /* Close the property list. */ - if (H5Pclose(fapl_id) < 0) - BAIL(NC_EHDFERR); - - /* Get the HDF5 superblock and read and parse the special - * _NCProperties attribute. */ - if ((retval = NC4_get_fileinfo(nc4_info, NULL))) - BAIL(retval); - - return NC_NOERR; - -exit: -#ifdef USE_PARALLEL4 - if (comm_duped) MPI_Comm_free(&nc4_info->comm); - if (info_duped) MPI_Info_free(&nc4_info->info); -#endif - if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); - if (!nc4_info) return retval; - close_netcdf4_file(nc4_info,1,0); /* treat like abort*/ - return retval; -} - -/** - * @internal Open a netCDF-4 file. - * - * @param path The file name of the new file. - * @param mode The open mode flag. - * @param basepe Ignored by this function. - * @param chunksizehintp Ignored by this function. - * @param use_parallel 0 for sequential, non-zero for parallel I/O. - * @param parameters pointer to struct holding extra data (e.g. for parallel I/O) - * layer. Ignored if NULL. - * @param dispatch Pointer to the dispatch table for this file. - * @param nc_file Pointer to an instance of NC. - * - * @return ::NC_NOERR No error. - * @return ::NC_EINVAL Invalid inputs. - * @author Ed Hartnett - */ -int -NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp, - int use_parallel, void *parameters, NC_Dispatch *dispatch, NC *nc_file) -{ - assert(nc_file && path && dispatch && nc_file && - nc_file->model == NC_FORMATX_NC4); - - LOG((1, "%s: path %s mode %d params %x", - __func__, path, mode, parameters)); - -#ifdef USE_PARALLEL4 - /* User must provide MPI communicator for parallel I/O. */ - if (use_parallel && !parameters) - return NC_EINVAL; - -#ifndef USE_PARALLEL_POSIX - /* If the HDF5 library has been compiled without the MPI-POSIX VFD, - * alias the NC_MPIPOSIX flag to NC_MPIIO. -QAK */ - if (mode & NC_MPIPOSIX) - { - mode &= ~NC_MPIPOSIX; - mode |= NC_MPIIO; - } -#endif /* USE_PARALLEL_POSIX */ -#endif /* USE_PARALLEL4 */ - - /* Check the mode for validity */ - if (mode & ILLEGAL_OPEN_FLAGS) - return NC_EINVAL; - - /* If this is our first file, initialize HDF5. */ - if (!nc4_hdf5_initialized) - nc4_hdf5_initialize(); - - nc_file->int_ncid = nc_file->ext_ncid; - - /* Open the file. */ - return nc4_open_file(path, mode, parameters, nc_file); -} /** * @internal Unfortunately HDF only allows specification of fill value @@ -2781,7 +391,7 @@ NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp, * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4_set_fill(int ncid, int fillmode, int *old_modep) { @@ -2820,7 +430,7 @@ NC4_set_fill(int ncid, int fillmode, int *old_modep) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4_redef(int ncid) { @@ -2863,7 +473,7 @@ NC4_redef(int ncid) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t r_align) @@ -2882,7 +492,7 @@ NC4__enddef(int ncid, size_t h_minfree, size_t v_align, * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ static int NC4_enddef(int ncid) { NC *nc; @@ -2918,7 +528,7 @@ static int NC4_enddef(int ncid) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4_sync(int ncid) { @@ -2983,7 +593,7 @@ NC4_abort(int ncid) /* Free any resources the netcdf-4 library has for this file's * metadata. */ - if ((retval = close_netcdf4_file(nc4_info, 1, 0))) + if ((retval = nc4_close_netcdf4_file(nc4_info, 1, 0))) return retval; /* Delete the file, if we should. */ @@ -3002,7 +612,7 @@ NC4_abort(int ncid) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4_close(int ncid, void* params) { @@ -3027,11 +637,11 @@ NC4_close(int ncid, void* params) inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY); /* Call the nc4 close. */ - if ((retval = close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0)))) + if ((retval = nc4_close_netcdf4_file(grp->nc4_info, 0, (inmemory?1:0)))) return retval; if(inmemory && params != NULL) { - NC_memio* memio = (NC_memio*)params; - *memio = h5->mem.memio; + NC_memio* memio = (NC_memio*)params; + *memio = h5->mem.memio; } return NC_NOERR; @@ -3053,7 +663,7 @@ NC4_close(int ncid, void* params) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int NC4_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) { @@ -3100,9 +710,9 @@ NC4_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) back in xtendimp. */ /* Note that this code is inconsistent with nc_inq_unlimid() */ for(i=0;idim);i++) { - NC_DIM_INFO_T* d = (NC_DIM_INFO_T*)ncindexith(grp->dim,i); - if(d == NULL) continue; - if(d->unlimited) { + NC_DIM_INFO_T* d = (NC_DIM_INFO_T*)ncindexith(grp->dim,i); + if(d == NULL) continue; + if(d->unlimited) { *unlimdimidp = d->hdr.id; break; } @@ -3119,7 +729,7 @@ NC4_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) * * @return ::NC_NOERR No error. * @author Ed Hartnett -*/ + */ int nc4_enddef_netcdf4_file(NC_FILE_INFO_T *h5) { diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index 17f39f074..dd0eae1d5 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -12,6 +12,7 @@ */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 8828687d3..92175db6a 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -15,6 +15,7 @@ */ #include "config.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc.h" /* from libsrc */ #include "ncdispatch.h" /* from libdispatch */ #include "ncutf8.h" diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c new file mode 100644 index 000000000..77c9fe0ef --- /dev/null +++ b/libhdf5/hdf5open.c @@ -0,0 +1,2115 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See COPYRIGHT file for copying and redistribution + * conditions. */ +/** + * @file + * @internal This file contains functions that are used in file + * opens, both fast and slow. + * + * @author Ed Hartnett + */ + +#include "config.h" +#include "hdf5internal.h" + +#define NUM_TYPES 12 /**< Number of netCDF atomic types. */ +#define CD_NELEMS_ZLIB 1 /**< Number of parameters needed for ZLIB filter. */ + +/** @internal Native HDF5 constants for atomic types. For performance, + * fill this array only the first time, and keep it in global memory + * for each further use. */ +static hid_t h5_native_type_constant_g[NUM_TYPES]; + +/** @internal NetCDF atomic type names. */ +static const char nc_type_name_g[NUM_TYPES][NC_MAX_NAME + 1] = {"char", "byte", "short", + "int", "float", "double", "ubyte", + "ushort", "uint", "int64", + "uint64", "string"}; + +/** @internal NetCDF atomic types. */ +static const nc_type nc_type_constant_g[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, + NC_INT, NC_FLOAT, NC_DOUBLE, NC_UBYTE, + NC_USHORT, NC_UINT, NC_INT64, + NC_UINT64, NC_STRING}; + +/** @internal NetCDF atomic type sizes. */ +static const int nc_type_size_g[NUM_TYPES] = {sizeof(char), sizeof(char), sizeof(short), + sizeof(int), sizeof(float), sizeof(double), sizeof(unsigned char), + sizeof(unsigned short), sizeof(unsigned int), sizeof(long long), + sizeof(unsigned long long), sizeof(char *)}; + +/** @internal These flags may not be set for open mode. */ +static const int ILLEGAL_OPEN_FLAGS = (NC_MMAP|NC_64BIT_OFFSET); + +/* From hdf5file.c. */ +extern size_t nc4_chunk_cache_size; +extern size_t nc4_chunk_cache_nelems; +extern float nc4_chunk_cache_preemption; + +/* From nc4mem.c */ +extern int NC4_open_image_file(NC_FILE_INFO_T* h5); + +/* Defined later in this file. */ +static int nc4_rec_read_metadata(NC_GRP_INFO_T *grp); + +/** + * @internal Struct to track information about objects in a group, for + * nc4_rec_read_metadata() + */ +typedef struct NC4_rec_read_metadata_obj_info +{ + hid_t oid; /* HDF5 object ID */ + char oname[NC_MAX_NAME + 1]; /* Name of object */ + H5G_stat_t statbuf; /* Information about the object */ + struct NC4_rec_read_metadata_obj_info *next; /* Pointer to next node in list */ +} NC4_rec_read_metadata_obj_info_t; + +/** + * @internal User data struct for call to H5Literate() in + * nc4_rec_read_metadata(). Tracks the groups, named datatypes and + * datasets in the group, for later use. + */ +typedef struct NC4_rec_read_metadata_ud +{ + NClist* grps; /* NClist */ + NC_GRP_INFO_T *grp; /* Pointer to parent group */ +} NC4_rec_read_metadata_ud_t; + +/* Custom iteration callback data */ +typedef struct { + NC_GRP_INFO_T *grp; + NC_VAR_INFO_T *var; +} att_iter_info; + +/** + * @internal Given an HDF5 type, set a pointer to netcdf type_info + * struct, either an existing one (for user-defined types) or a newly + * created one. + * + * @param h5 Pointer to HDF5 file info struct. + * @param datasetid HDF5 dataset ID. + * @param type_info Pointer to pointer that gets type info struct. + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EHDFERR HDF5 returned error. + * @return ::NC_EBADTYPID Type not found. + * @author Ed Hartnett + */ +static int +get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, + NC_TYPE_INFO_T **type_info) +{ + htri_t is_str, equal = 0; + H5T_class_t class; + hid_t native_typeid, hdf_typeid; + H5T_order_t order; + int t; + + assert(h5 && type_info); + + /* Because these N5T_NATIVE_* constants are actually function calls + * (!) in H5Tpublic.h, I can't initialize this array in the usual + * way, because at least some C compilers (like Irix) complain + * about calling functions when defining constants. So I have to do + * it like this. Note that there's no native types for char or + * string. Those are handled later. */ + if (!h5_native_type_constant_g[1]) + { + h5_native_type_constant_g[1] = H5T_NATIVE_SCHAR; + h5_native_type_constant_g[2] = H5T_NATIVE_SHORT; + h5_native_type_constant_g[3] = H5T_NATIVE_INT; + h5_native_type_constant_g[4] = H5T_NATIVE_FLOAT; + h5_native_type_constant_g[5] = H5T_NATIVE_DOUBLE; + h5_native_type_constant_g[6] = H5T_NATIVE_UCHAR; + h5_native_type_constant_g[7] = H5T_NATIVE_USHORT; + h5_native_type_constant_g[8] = H5T_NATIVE_UINT; + h5_native_type_constant_g[9] = H5T_NATIVE_LLONG; + h5_native_type_constant_g[10] = H5T_NATIVE_ULLONG; + } + + /* Get the HDF5 typeid - we'll need it later. */ + if ((hdf_typeid = H5Dget_type(datasetid)) < 0) + return NC_EHDFERR; + + /* Get the native typeid. Will be equivalent to hdf_typeid when + * creating but not necessarily when reading, a variable. */ + if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0) + return NC_EHDFERR; + + /* Is this type an integer, string, compound, or what? */ + if ((class = H5Tget_class(native_typeid)) < 0) + return NC_EHDFERR; + + /* Is this an atomic type? */ + if (class == H5T_STRING || class == H5T_INTEGER || class == H5T_FLOAT) + { + /* Allocate a phony NC_TYPE_INFO_T struct to hold type info. */ + if (!(*type_info = calloc(1, sizeof(NC_TYPE_INFO_T)))) + return NC_ENOMEM; + + /* H5Tequal doesn't work with H5T_C_S1 for some reason. But + * H5Tget_class will return H5T_STRING if this is a string. */ + if (class == H5T_STRING) + { + if ((is_str = H5Tis_variable_str(native_typeid)) < 0) + return NC_EHDFERR; + /* Make sure fixed-len strings will work like variable-len + * strings */ + if (is_str || H5Tget_size(hdf_typeid) > 1) + { + /* Set a class for the type */ + t = NUM_TYPES - 1; + (*type_info)->nc_type_class = NC_STRING; + } + else + { + /* Set a class for the type */ + t = 0; + (*type_info)->nc_type_class = NC_CHAR; + } + } + else + { + for (t = 1; t < NUM_TYPES - 1; t++) + { + if ((equal = H5Tequal(native_typeid, + h5_native_type_constant_g[t])) < 0) + return NC_EHDFERR; + if (equal) + break; + } + + /* Find out about endianness. As of HDF 1.8.6, this works + * with all data types Not just H5T_INTEGER. See + * https://www.hdfgroup.org/HDF5/doc/RM/RM_H5T.html#Datatype-GetOrder */ + if((order = H5Tget_order(hdf_typeid)) < 0) + return NC_EHDFERR; + + if(order == H5T_ORDER_LE) + (*type_info)->endianness = NC_ENDIAN_LITTLE; + else if(order == H5T_ORDER_BE) + (*type_info)->endianness = NC_ENDIAN_BIG; + else + return NC_EBADTYPE; + + if(class == H5T_INTEGER) + (*type_info)->nc_type_class = NC_INT; + else + (*type_info)->nc_type_class = NC_FLOAT; + } + (*type_info)->hdr.id = nc_type_constant_g[t]; + (*type_info)->size = nc_type_size_g[t]; + if (!((*type_info)->hdr.name = strdup(nc_type_name_g[t]))) + return NC_ENOMEM; + (*type_info)->hdf_typeid = hdf_typeid; + (*type_info)->native_hdf_typeid = native_typeid; + return NC_NOERR; + } + else + { + NC_TYPE_INFO_T *type; + + /* This is a user-defined type. */ + if((type = nc4_rec_find_hdf_type(h5, native_typeid))) + *type_info = type; + + /* The type entry in the array of user-defined types already has + * an open data typeid (and native typeid), so close the ones we + * opened above. */ + if (H5Tclose(native_typeid) < 0) + return NC_EHDFERR; + if (H5Tclose(hdf_typeid) < 0) + return NC_EHDFERR; + + if (type) + return NC_NOERR; + } + + return NC_EBADTYPID; +} +/** + * @internal This function reads the hacked in coordinates attribute I + * use for multi-dimensional coordinates. + * + * @param grp Group info pointer. + * @param var Var info pointer. + * + * @return NC_NOERR No error. + * @author Ed Hartnett + */ +static int +read_coord_dimids(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) +{ + hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1; + hssize_t npoints; + int ret = 0; + int d; + + /* There is a hidden attribute telling us the ids of the + * dimensions that apply to this multi-dimensional coordinate + * variable. Read it. */ + if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++; + if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++; + + /* How many dimensions are there? */ + if (!ret && (spaceid = H5Aget_space(coord_attid)) < 0) ret++; + if (!ret && (npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) ret++; + + /* Check that the number of points is the same as the number of dimensions + * for the variable */ + if (!ret && npoints != var->ndims) ret++; + + if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++; + LOG((4, "dimscale %s is multidimensional and has coords", var->hdr.name)); + + /* Update var->dim field based on the var->dimids */ + for (d = 0; d < var->ndims; d++) { + /* Ok if does not find a dim at this time, but if found set it */ + nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL); + } + + /* Set my HDF5 IDs free! */ + if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++; + if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++; + if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++; + return ret ? NC_EATTMETA : NC_NOERR; +} + +/** + * @internal This function is called when reading a file's metadata + * for each dimension scale attached to a variable. + * + * @param did HDF5 ID for dimscale. + * @param dim + * @param dsid + * @param dimscale_hdf5_objids + * + * @return 0 for success, -1 for error. + * @author Ed Hartnett + */ +static herr_t +dimscale_visitor(hid_t did, unsigned dim, hid_t dsid, + void *dimscale_hdf5_objids) +{ + H5G_stat_t statbuf; + + /* Get more info on the dimscale object.*/ + if (H5Gget_objinfo(dsid, ".", 1, &statbuf) < 0) + return -1; + + /* Pass this information back to caller. */ + (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0]; + (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1]; + (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0]; + (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1]; + return 0; +} + +/** + * @internal Check for the attribute that indicates that netcdf + * classic model is in use. + * + * @param root_grp pointer to the group info for the root group of the + * @param is_classic store 1 if this is a classic file. + * file. + * + * @return NC_NOERR No error. + * @author Ed Hartnett + */ +static int +check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) +{ + htri_t attr_exists = -1; + + /* Check inputs. */ + assert(!root_grp->parent && is_classic); + + /* If this attribute exists in the root group, then classic model + * is in effect. */ + if ((attr_exists = H5Aexists(root_grp->hdf_grpid, NC3_STRICT_ATT_NAME)) < 0) + return NC_EHDFERR; + *is_classic = attr_exists ? 1 : 0; + + return NC_NOERR; +} + +/** + * @internal Open a netcdf-4 file. Things have already been kicked off + * in ncfunc.c in nc_open, but here the netCDF-4 part of opening a + * file is handled. + * + * @param path The file name of the new file. + * @param mode The open mode flag. + * @param parameters File parameters. + * @param nc Pointer to NC file info. + * + * @return ::NC_NOERR No error. + * @author Ed Hartnett, Dennis Heimbigner + */ +static int +nc4_open_file(const char *path, int mode, void* parameters, NC *nc) +{ + hid_t fapl_id = H5P_DEFAULT; + int retval; + unsigned flags; + NC_FILE_INFO_T *nc4_info = NULL; + int is_classic; + +#ifdef USE_PARALLEL4 + NC_MPI_INFO* mpiinfo = NULL; + int comm_duped = 0; /* Whether the MPI Communicator was duplicated */ + int info_duped = 0; /* Whether the MPI Info object was duplicated */ +#endif + + LOG((3, "%s: path %s mode %d", __func__, path, mode)); + assert(path && nc); + + flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY; + + /* Add necessary structs to hold netcdf-4 file data. */ + if ((retval = nc4_nc4f_list_add(nc, path, mode))) + BAIL(retval); + nc4_info = NC4_DATA(nc); + assert(nc4_info && nc4_info->root_grp); + + nc4_info->mem.inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); + nc4_info->mem.diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); + if(nc4_info->mem.inmemory) { + NC_memio* memparams = NULL; + if(parameters == NULL) + BAIL(NC_EINMEMORY); + memparams = (NC_memio*)parameters; + nc4_info->mem.memio = *memparams; /* keep local copy */ + /* As a safeguard, if !locked and NC_WRITE is set, + then we must take control of the incoming memory */ + nc4_info->mem.locked = (nc4_info->mem.memio.flags & NC_MEMIO_LOCKED) == NC_MEMIO_LOCKED; + if(!nc4_info->mem.locked && ((mode & NC_WRITE) == NC_WRITE)) { + memparams->memory = NULL; + } +#ifdef USE_PARALLEL4 + } else { + mpiinfo = (NC_MPI_INFO*)parameters; +#endif /* !USE_PARALLEL4 */ + } + + /* Need this access plist to control how HDF5 handles open objects + * on file close. (Setting H5F_CLOSE_SEMI will cause H5Fclose to + * fail if there are any open objects in the file. */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + BAIL(NC_EHDFERR); + + if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) + BAIL(NC_EHDFERR); + +#ifdef USE_PARALLEL4 + /* If this is a parallel file create, set up the file creation + property list. */ + if (mode & NC_MPIIO || mode & NC_MPIPOSIX) + { + nc4_info->parallel = NC_TRUE; + if (mode & NC_MPIIO) /* MPI/IO */ + { + LOG((4, "opening parallel file with MPI/IO")); + if (H5Pset_fapl_mpio(fapl_id, mpiinfo->comm, mpiinfo->info) < 0) + BAIL(NC_EPARINIT); + } +#ifdef USE_PARALLEL_POSIX + else /* MPI/POSIX */ + { + LOG((4, "opening parallel file with MPI/posix")); + if (H5Pset_fapl_mpiposix(fapl_id, mpiinfo->comm, 0) < 0) + BAIL(NC_EPARINIT); + } +#else /* USE_PARALLEL_POSIX */ + /* Should not happen! Code in NC4_create/NC4_open should alias the + * NC_MPIPOSIX flag to NC_MPIIO, if the MPI-POSIX VFD is not + * available in HDF5. -QAK + */ + else /* MPI/POSIX */ + BAIL(NC_EPARINIT); +#endif /* USE_PARALLEL_POSIX */ + + /* Keep copies of the MPI Comm & Info objects */ + if (MPI_SUCCESS != MPI_Comm_dup(mpiinfo->comm, &nc4_info->comm)) + BAIL(NC_EMPI); + comm_duped++; + if (MPI_INFO_NULL != mpiinfo->info) + { + if (MPI_SUCCESS != MPI_Info_dup(mpiinfo->info, &nc4_info->info)) + BAIL(NC_EMPI); + info_duped++; + } + else + { + /* No dup, just copy it. */ + nc4_info->info = mpiinfo->info; + } + } +#else /* only set cache for non-parallel. */ + if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size, + nc4_chunk_cache_preemption) < 0) + BAIL(NC_EHDFERR); + LOG((4, "%s: set HDF raw chunk cache to size %d nelems %d preemption %f", + __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, + nc4_chunk_cache_preemption)); +#endif /* USE_PARALLEL4 */ + + /* The NetCDF-3.x prototype contains an mode option NC_SHARE for + multiple processes accessing the dataset concurrently. As there + is no HDF5 equivalent, NC_SHARE is treated as NC_NOWRITE. */ +#ifdef HDF5_HAS_COLL_METADATA_OPS + H5Pset_all_coll_metadata_ops(fapl_id, 1 ); +#endif + + /* Does the mode specify that this file is read-only? */ + if ((mode & NC_WRITE) == 0) + nc4_info->no_write = NC_TRUE; + + if(nc4_info->mem.inmemory) { + /* validate */ + if(nc4_info->mem.memio.size == 0 || nc4_info->mem.memio.memory == NULL) + BAIL(NC_INMEMORY); + retval = NC4_open_image_file(nc4_info); + if(retval) + BAIL(NC_EHDFERR); + } else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0) + BAIL(NC_EHDFERR); + + /* Now read in all the metadata. Some types and dimscale + * information may be difficult to resolve here, if, for example, a + * dataset of user-defined type is encountered before the + * definition of that type. */ + if ((retval = nc4_rec_read_metadata(nc4_info->root_grp))) + BAIL(retval); + + /* Check for classic model attribute. */ + if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) + BAIL(retval); + if (is_classic) + nc4_info->cmode |= NC_CLASSIC_MODEL; + + /* Now figure out which netCDF dims are indicated by the dimscale + * information. */ + if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp))) + BAIL(retval); + +#ifdef LOGGING + /* This will print out the names, types, lens, etc of the vars and + atts in the file, if the logging level is 2 or greater. */ + log_metadata_nc(nc); +#endif + + /* Close the property list. */ + if (H5Pclose(fapl_id) < 0) + BAIL(NC_EHDFERR); + + /* Get the HDF5 superblock and read and parse the special + * _NCProperties attribute. */ + if ((retval = NC4_get_fileinfo(nc4_info, NULL))) + BAIL(retval); + + return NC_NOERR; + +exit: +#ifdef USE_PARALLEL4 + if (comm_duped) MPI_Comm_free(&nc4_info->comm); + if (info_duped) MPI_Info_free(&nc4_info->info); +#endif + if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id); + if (!nc4_info) return retval; + nc4_close_netcdf4_file(nc4_info,1,0); /* treat like abort*/ + return retval; +} + +/** + * @internal Open a netCDF-4 file. + * + * @param path The file name of the new file. + * @param mode The open mode flag. + * @param basepe Ignored by this function. + * @param chunksizehintp Ignored by this function. + * @param use_parallel 0 for sequential, non-zero for parallel I/O. + * @param parameters pointer to struct holding extra data (e.g. for parallel I/O) + * layer. Ignored if NULL. + * @param dispatch Pointer to the dispatch table for this file. + * @param nc_file Pointer to an instance of NC. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid inputs. + * @author Ed Hartnett + */ +int +NC4_open(const char *path, int mode, int basepe, size_t *chunksizehintp, + int use_parallel, void *parameters, NC_Dispatch *dispatch, NC *nc_file) +{ + assert(nc_file && path && dispatch && nc_file && + nc_file->model == NC_FORMATX_NC4); + + LOG((1, "%s: path %s mode %d params %x", + __func__, path, mode, parameters)); + +#ifdef USE_PARALLEL4 + /* User must provide MPI communicator for parallel I/O. */ + if (use_parallel && !parameters) + return NC_EINVAL; + +#ifndef USE_PARALLEL_POSIX + /* If the HDF5 library has been compiled without the MPI-POSIX VFD, + * alias the NC_MPIPOSIX flag to NC_MPIIO. -QAK */ + if (mode & NC_MPIPOSIX) + { + mode &= ~NC_MPIPOSIX; + mode |= NC_MPIIO; + } +#endif /* USE_PARALLEL_POSIX */ +#endif /* USE_PARALLEL4 */ + + /* Check the mode for validity */ + if (mode & ILLEGAL_OPEN_FLAGS) + return NC_EINVAL; + + /* If this is our first file, initialize HDF5. */ + if (!nc4_hdf5_initialized) + nc4_hdf5_initialize(); + + nc_file->int_ncid = nc_file->ext_ncid; + + /* Open the file. */ + return nc4_open_file(path, mode, parameters, nc_file); +} + +/** + * @internal This function is called by read_dataset(), (which is called + * by nc4_rec_read_metadata()) when a netCDF variable is found in the + * file. This function reads in all the metadata about the var, + * including the attributes. + * + * @param grp Pointer to group info struct. + * @param datasetid HDF5 dataset ID. + * @param obj_name Name of the HDF5 object to read. + * @param ndims Number of dimensions. + * @param dim + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EHDFERR HDF5 returned error. + * @author Ed Hartnett + */ +static int +read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, + size_t ndims, NC_DIM_INFO_T *dim) +{ + NC_VAR_INFO_T *var = NULL; + hid_t access_pid = 0; + int incr_id_rc = 0; /* Whether dataset ID's ref count has been incremented */ + int d; + H5Z_filter_t filter; + int num_filters; + unsigned int cd_values_zip[CD_NELEMS_ZLIB]; + size_t cd_nelems = CD_NELEMS_ZLIB; + hid_t propid = 0; + H5D_fill_value_t fill_status; + H5D_layout_t layout; + hsize_t chunksize[NC_MAX_VAR_DIMS] = {0}; + int retval = NC_NOERR; + double rdcc_w0; + int f; + char* finalname = NULL; + + assert(obj_name && grp); + LOG((4, "%s: obj_name %s", __func__, obj_name)); + + /* Check for a weird case: a non-coordinate variable that has the + * same name as a dimension. It's legal in netcdf, and requires + * that the HDF5 dataset name be changed. */ + if (strlen(obj_name) > strlen(NON_COORD_PREPEND) && + !strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND))) + { + /* Allocate space for the name. */ + if(finalname) {free(finalname); finalname = NULL;} + if (!(finalname = malloc(((strlen(obj_name) - + strlen(NON_COORD_PREPEND))+ 1) * sizeof(char)))) + BAIL(NC_ENOMEM); + strcpy(finalname, &obj_name[strlen(NON_COORD_PREPEND)]); + } else + finalname = strdup(obj_name); + + /* Add a variable to the end of the group's var list. */ + if ((retval = nc4_var_list_add(grp,finalname,ndims,&var))) + BAIL(retval); + + /* Fill in what we already know. */ + var->hdf_datasetid = datasetid; + H5Iinc_ref(var->hdf_datasetid); /* Increment number of objects using ID */ + incr_id_rc++; /* Indicate that we've incremented the ref. count (for errors) */ + var->created = NC_TRUE; + + /* Get the current chunk cache settings. */ + if ((access_pid = H5Dget_access_plist(datasetid)) < 0) + BAIL(NC_EVARMETA); + + /* Learn about current chunk cache settings. */ + if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems), + &(var->chunk_cache_size), &rdcc_w0)) < 0) + BAIL(NC_EHDFERR); + var->chunk_cache_preemption = rdcc_w0; + + /* Find out what filters are applied to this HDF5 dataset, + * fletcher32, deflate, and/or shuffle. All other filters are + * just dumped */ + if ((propid = H5Dget_create_plist(datasetid)) < 0) + BAIL(NC_EHDFERR); + + /* Get the chunking info for non-scalar vars. */ + if ((layout = H5Pget_layout(propid)) < -1) + BAIL(NC_EHDFERR); + if (layout == H5D_CHUNKED) + { + if (H5Pget_chunk(propid, NC_MAX_VAR_DIMS, chunksize) < 0) + BAIL(NC_EHDFERR); + if (!(var->chunksizes = malloc(var->ndims * sizeof(size_t)))) + BAIL(NC_ENOMEM); + for (d = 0; d < var->ndims; d++) + var->chunksizes[d] = chunksize[d]; + } + else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT) + var->contiguous = NC_TRUE; + + /* The possible values of filter (which is just an int) can be + * found in H5Zpublic.h. */ + if ((num_filters = H5Pget_nfilters(propid)) < 0) + BAIL(NC_EHDFERR); + for (f = 0; f < num_filters; f++) + { + if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, + cd_values_zip, 0, NULL, NULL)) < 0) + BAIL(NC_EHDFERR); + switch (filter) + { + case H5Z_FILTER_SHUFFLE: + var->shuffle = NC_TRUE; + break; + + case H5Z_FILTER_FLETCHER32: + var->fletcher32 = NC_TRUE; + break; + + case H5Z_FILTER_DEFLATE: + var->deflate = NC_TRUE; + if (cd_nelems != CD_NELEMS_ZLIB || + cd_values_zip[0] > NC_MAX_DEFLATE_LEVEL) + BAIL(NC_EHDFERR); + var->deflate_level = cd_values_zip[0]; + break; + + default: + var->filterid = filter; + var->nparams = cd_nelems; + if(cd_nelems == 0) + var->params = NULL; + else { + /* We have to re-read the parameters based on actual nparams */ + var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*var->nparams); + if(var->params == NULL) + BAIL(NC_ENOMEM); + if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems, + var->params, 0, NULL, NULL)) < 0) + BAIL(NC_EHDFERR); + } + break; + } + } + + /* Learn all about the type of this variable. */ + if ((retval = get_type_info2(grp->nc4_info, datasetid, + &var->type_info))) + BAIL(retval); + + /* Indicate that the variable has a pointer to the type */ + var->type_info->rc++; + + /* Is there a fill value associated with this dataset? */ + if (H5Pfill_value_defined(propid, &fill_status) < 0) + BAIL(NC_EHDFERR); + + /* Get the fill value, if there is one defined. */ + if (fill_status == H5D_FILL_VALUE_USER_DEFINED) + { + /* Allocate space to hold the fill value. */ + if (!var->fill_value) + { + if (var->type_info->nc_type_class == NC_VLEN) + { + if (!(var->fill_value = malloc(sizeof(nc_vlen_t)))) + BAIL(NC_ENOMEM); + } + else if (var->type_info->nc_type_class == NC_STRING) + { + if (!(var->fill_value = malloc(sizeof(char *)))) + BAIL(NC_ENOMEM); + } + else + { + assert(var->type_info->size); + if (!(var->fill_value = malloc(var->type_info->size))) + BAIL(NC_ENOMEM); + } + } + + /* Get the fill value from the HDF5 property lust. */ + if (H5Pget_fill_value(propid, var->type_info->native_hdf_typeid, + var->fill_value) < 0) + BAIL(NC_EHDFERR); + } + else + var->no_fill = NC_TRUE; + + /* If it's a scale, mark it as such. */ + if (dim) + { + assert(ndims); + var->dimscale = NC_TRUE; + if (var->ndims > 1) + { + if ((retval = read_coord_dimids(grp, var))) + BAIL(retval); + } + else + { + /* sanity check */ + assert(0 == strcmp(var->hdr.name, dim->hdr.name)); + + var->dimids[0] = dim->hdr.id; + var->dim[0] = dim; + } + dim->coord_var = var; + } + /* If this is not a scale, but has scales, iterate + * through them. (i.e. this is a variable that is not a + * coordinate variable) */ + else + { + int num_scales = 0; + + /* Find out how many scales are attached to this + * dataset. H5DSget_num_scales returns an error if there are no + * scales, so convert a negative return value to zero. */ + num_scales = H5DSget_num_scales(datasetid, 0); + if (num_scales < 0) + num_scales = 0; + + if (num_scales && ndims) + { + /* Allocate space to remember whether the dimscale has been + * attached for each dimension. */ + if (!(var->dimscale_attached = calloc(ndims, sizeof(nc_bool_t)))) + BAIL(NC_ENOMEM); + + /* Store id information allowing us to match hdf5 + * dimscales to netcdf dimensions. */ + if (!(var->dimscale_hdf5_objids = malloc(ndims * + sizeof(struct hdf5_objid)))) + BAIL(NC_ENOMEM); + for (d = 0; d < var->ndims; d++) + { + if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor, + &(var->dimscale_hdf5_objids[d])) < 0) + BAIL(NC_EHDFERR); + var->dimscale_attached[d] = NC_TRUE; + } + } + } + + /* Read variable attributes. */ + var->atts_not_read = 1; + + /* Is this a deflated variable with a chunksize greater than the + * current cache size? */ + if ((retval = nc4_adjust_var_cache(grp, var))) + BAIL(retval); + +exit: + if(finalname) free(finalname); + if (retval) + { + if (incr_id_rc && H5Idec_ref(datasetid) < 0) + BAIL2(NC_EHDFERR); + if (var != NULL) { + nc4_var_list_del(grp,var); + } + } + if (access_pid && H5Pclose(access_pid) < 0) + BAIL2(NC_EHDFERR); + if (propid > 0 && H5Pclose(propid) < 0) + BAIL2(NC_EHDFERR); + return retval; +} + +/** + * @internal Given an HDF5 type, set a pointer to netcdf type. + * + * @param h5 Pointer to HDF5 file info struct. + * @param native_typeid HDF5 type ID. + * @param xtype Pointer that gets netCDF type. + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EHDFERR HDF5 returned error. + * @return ::NC_EBADTYPID Type not found. + * @author Ed Hartnett + */ +static int +get_netcdf_type(NC_FILE_INFO_T *h5, hid_t native_typeid, + nc_type *xtype) +{ + NC_TYPE_INFO_T *type; + H5T_class_t class; + htri_t is_str, equal = 0; + + assert(h5 && xtype); + + if ((class = H5Tget_class(native_typeid)) < 0) + return NC_EHDFERR; + + /* H5Tequal doesn't work with H5T_C_S1 for some reason. But + * H5Tget_class will return H5T_STRING if this is a string. */ + if (class == H5T_STRING) + { + if ((is_str = H5Tis_variable_str(native_typeid)) < 0) + return NC_EHDFERR; + if (is_str) + *xtype = NC_STRING; + else + *xtype = NC_CHAR; + return NC_NOERR; + } + else if (class == H5T_INTEGER || class == H5T_FLOAT) + { + /* For integers and floats, we don't have to worry about + * endianness if we compare native types. */ + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_BYTE; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_SHORT; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_INT; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_FLOAT; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_DOUBLE; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_UBYTE; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_USHORT; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_UINT; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_INT64; + return NC_NOERR; + } + if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0) + return NC_EHDFERR; + if (equal) + { + *xtype = NC_UINT64; + return NC_NOERR; + } + } + + /* Maybe we already know about this type. */ + if (!equal) + if((type = nc4_rec_find_hdf_type(h5, native_typeid))) + { + *xtype = type->hdr.id; + return NC_NOERR; + } + + *xtype = NC_NAT; + return NC_EBADTYPID; +} + +/** + * @internal Read an attribute. This is called by att_read_var_callbk(). + * + * @param grp Pointer to group info struct. + * @param attid Attribute ID. + * @param att Pointer that gets att info struct. + * + * @return ::NC_NOERR No error. + * @return ::NC_EHDFERR HDF5 returned error. + * @author Ed Hartnett + */ +static int +read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att) +{ + hid_t spaceid = 0, file_typeid = 0; + hsize_t dims[1] = {0}; /* netcdf attributes always 1-D. */ + int retval = NC_NOERR; + size_t type_size; + int att_ndims; + hssize_t att_npoints; + H5T_class_t att_class; + int fixed_len_string = 0; + size_t fixed_size = 0; + + assert(att->hdr.name); + LOG((5, "%s: att->hdr.id %d att->hdr.name %s att->nc_typeid %d att->len %d", + __func__, att->hdr.id, att->hdr.name, (int)att->nc_typeid, att->len)); + + /* Get type of attribute in file. */ + if ((file_typeid = H5Aget_type(attid)) < 0) + return NC_EATTMETA; + if ((att->native_hdf_typeid = H5Tget_native_type(file_typeid, + H5T_DIR_DEFAULT)) < 0) + BAIL(NC_EHDFERR); + if ((att_class = H5Tget_class(att->native_hdf_typeid)) < 0) + BAIL(NC_EATTMETA); + if (att_class == H5T_STRING && + !H5Tis_variable_str(att->native_hdf_typeid)) + { + fixed_len_string++; + if (!(fixed_size = H5Tget_size(att->native_hdf_typeid))) + BAIL(NC_EATTMETA); + } + if ((retval = get_netcdf_type(grp->nc4_info, att->native_hdf_typeid, + &(att->nc_typeid)))) + BAIL(retval); + + + /* Get len. */ + if ((spaceid = H5Aget_space(attid)) < 0) + BAIL(NC_EATTMETA); + if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0) + BAIL(NC_EATTMETA); + if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) + BAIL(NC_EATTMETA); + + /* If both att_ndims and att_npoints are zero, then this is a + * zero length att. */ + if (att_ndims == 0 && att_npoints == 0) + dims[0] = 0; + else if (att->nc_typeid == NC_STRING) + dims[0] = att_npoints; + else if (att->nc_typeid == NC_CHAR) + { + /* NC_CHAR attributes are written as a scalar in HDF5, of type + * H5T_C_S1, of variable length. */ + if (att_ndims == 0) + { + if (!(dims[0] = H5Tget_size(file_typeid))) + BAIL(NC_EATTMETA); + } + else + { + /* This is really a string type! */ + att->nc_typeid = NC_STRING; + dims[0] = att_npoints; + } + } + else + { + H5S_class_t space_class; + + /* All netcdf attributes are scalar or 1-D only. */ + if (att_ndims > 1) + BAIL(NC_EATTMETA); + + /* Check class of HDF5 dataspace */ + if ((space_class = H5Sget_simple_extent_type(spaceid)) < 0) + BAIL(NC_EATTMETA); + + /* Check for NULL HDF5 dataspace class (should be weeded out + * earlier) */ + if (H5S_NULL == space_class) + BAIL(NC_EATTMETA); + + /* check for SCALAR HDF5 dataspace class */ + if (H5S_SCALAR == space_class) + dims[0] = 1; + else /* Must be "simple" dataspace */ + { + /* Read the size of this attribute. */ + if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0) + BAIL(NC_EATTMETA); + } + } + + /* Tell the user what the length if this attribute is. */ + att->len = dims[0]; + + /* Allocate some memory if the len is not zero, and read the + attribute. */ + if (dims[0]) + { + if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->nc_typeid, + &type_size))) + return retval; + if (att_class == H5T_VLEN) + { + if (!(att->vldata = malloc((unsigned int)(att->len * sizeof(hvl_t))))) + BAIL(NC_ENOMEM); + if (H5Aread(attid, att->native_hdf_typeid, att->vldata) < 0) + BAIL(NC_EATTMETA); + } + else if (att->nc_typeid == NC_STRING) + { + if (!(att->stdata = calloc(att->len, sizeof(char *)))) + BAIL(NC_ENOMEM); + /* For a fixed length HDF5 string, the read requires + * contiguous memory. Meanwhile, the netCDF API requires that + * nc_free_string be called on string arrays, which would not + * work if one contiguous memory block were used. So here I + * convert the contiguous block of strings into an array of + * malloced strings (each string with its own malloc). Then I + * copy the data and free the contiguous memory. This + * involves copying the data, which is bad, but this only + * occurs for fixed length string attributes, and presumably + * these are small. (And netCDF-4 does not create them - it + * always uses variable length strings. */ + if (fixed_len_string) + { + int i; + char *contig_buf, *cur; + + /* Alloc space for the contiguous memory read. */ + if (!(contig_buf = malloc(att->len * fixed_size * sizeof(char)))) + BAIL(NC_ENOMEM); + + /* Read the fixed-len strings as one big block. */ + if (H5Aread(attid, att->native_hdf_typeid, contig_buf) < 0) { + free(contig_buf); + BAIL(NC_EATTMETA); + } + + /* Copy strings, one at a time, into their new home. Alloc + space for each string. The user will later free this + space with nc_free_string. */ + cur = contig_buf; + for (i = 0; i < att->len; i++) + { + if (!(att->stdata[i] = malloc(fixed_size))) { + free(contig_buf); + BAIL(NC_ENOMEM); + } + strncpy(att->stdata[i], cur, fixed_size); + cur += fixed_size; + } + + /* Free contiguous memory buffer. */ + free(contig_buf); + } + else + { + /* Read variable-length string atts. */ + if (H5Aread(attid, att->native_hdf_typeid, att->stdata) < 0) + BAIL(NC_EATTMETA); + } + } + else + { + if (!(att->data = malloc((unsigned int)(att->len * type_size)))) + BAIL(NC_ENOMEM); + if (H5Aread(attid, att->native_hdf_typeid, att->data) < 0) + BAIL(NC_EATTMETA); + } + } + + if (H5Tclose(file_typeid) < 0) + BAIL(NC_EHDFERR); + if (H5Sclose(spaceid) < 0) + return NC_EHDFERR; + + return NC_NOERR; + +exit: + if (H5Tclose(file_typeid) < 0) + BAIL2(NC_EHDFERR); + if (spaceid > 0 && H5Sclose(spaceid) < 0) + BAIL2(NC_EHDFERR); + return retval; +} + +/** + * @internal This function is called by nc4_rec_read_metadata to read + * all the group level attributes (the NC_GLOBAL atts for this + * group). + * + * @param grp Pointer to group info struct. + * + * @return ::NC_NOERR No error. + * @return ::NC_EHDFERR HDF5 returned error. + * @author Ed Hartnett + */ +int +nc4_read_grp_atts(NC_GRP_INFO_T *grp) +{ + hid_t attid = -1; + hsize_t num_obj, i; + NC_ATT_INFO_T *att; + NC_TYPE_INFO_T *type; + char obj_name[NC_MAX_HDF5_NAME + 1]; + int retval = NC_NOERR; + int hidden = 0; + + num_obj = H5Aget_num_attrs(grp->hdf_grpid); + for (i = 0; i < num_obj; i++) + { + if ((attid = H5Aopen_idx(grp->hdf_grpid, (unsigned int)i)) < 0) + BAIL(NC_EATTMETA); + if (H5Aget_name(attid, NC_MAX_NAME + 1, obj_name) < 0) + BAIL(NC_EATTMETA); + LOG((3, "reading attribute of _netCDF group, named %s", obj_name)); + + /* See if this a hidden, global attribute */ + hidden = 0; /* default */ + if(grp->nc4_info->root_grp == grp) { + const NC_reservedatt* ra = NC_findreserved(obj_name); + if(ra != NULL && (ra->flags & NAMEONLYFLAG)) + hidden = 1; + } + + /* This may be an attribute telling us that strict netcdf-3 + * rules are in effect. If so, we will make note of the fact, + * but not add this attribute to the metadata. It's not a user + * attribute, but an internal netcdf-4 one. */ + if(strcmp(obj_name, NC3_STRICT_ATT_NAME)==0) + grp->nc4_info->cmode |= NC_CLASSIC_MODEL; + else if(!hidden) { + /* Add an att struct at the end of the list, and then go to it. */ + if ((retval = nc4_att_list_add(grp->att, obj_name, &att))) + BAIL(retval); + retval = read_hdf5_att(grp, attid, att); + if(retval == NC_EBADTYPID) { + if((retval = nc4_att_list_del(grp->att, att))) + BAIL(retval); + } else if(retval) { + BAIL(retval); + } else { + att->created = NC_TRUE; + if ((retval = nc4_find_type(grp->nc4_info, att->nc_typeid, &type))) + BAIL(retval); + } + } + /* Unconditionally close the open attribute */ + H5Aclose(attid); + attid = -1; + } + + /* Remember that we have read the atts for this group. */ + grp->atts_not_read = 0; + +exit: + if (attid > 0) { + if(H5Aclose(attid) < 0) + BAIL2(NC_EHDFERR); + } + return retval; +} + +/** + * @internal Wrap HDF5 allocated memory free operations + * + * @param memory Pointer to memory to be freed. + * + * @return ::NC_NOERR No error. + * @author Dennis Heimbigner + */ +static void +hdf5free(void* memory) +{ +#ifndef JNA + /* On Windows using the microsoft runtime, it is an error + for one library to free memory allocated by a different library.*/ +#ifdef HDF5_HAS_H5FREE + if(memory != NULL) H5free_memory(memory); +#else +#ifndef _MSC_VER + if(memory != NULL) free(memory); +#endif +#endif +#endif +} + +/** + * @internal Read information about a user defined type from the HDF5 + * file, and stash it in the group's list of types. + * + * @param grp Pointer to group info struct. + * @param hdf_typeid HDF5 type ID. + * @param type_name Pointer that gets the type name. + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EHDFERR HDF5 returned error. + * @return ::NC_EBADTYPID Type not found. + * @author Ed Hartnett + */ +static int +read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name) +{ + NC_TYPE_INFO_T *type; + H5T_class_t class; + hid_t native_typeid; + size_t type_size; + int retval = NC_NOERR; + int nmembers; + + assert(grp && type_name); + + LOG((4, "%s: type_name %s grp->hdr.name %s", __func__, type_name, + grp->hdr.name)); + + /* What is the native type for this platform? */ + if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0) + return NC_EHDFERR; + + /* What is the size of this type on this platform. */ + if (!(type_size = H5Tget_size(native_typeid))) + return NC_EHDFERR; + LOG((5, "type_size %d", type_size)); + + /* Add to the list for this new type, and get a local pointer to it. */ + if ((retval = nc4_type_list_add(grp, type_size, type_name, &type))) + return retval; + + /* Remember common info about this type. */ + type->committed = NC_TRUE; + type->hdf_typeid = hdf_typeid; + H5Iinc_ref(type->hdf_typeid); /* Increment number of objects using ID */ + type->native_hdf_typeid = native_typeid; + + /* What is the class of this type, compound, vlen, etc. */ + if ((class = H5Tget_class(hdf_typeid)) < 0) + return NC_EHDFERR; + switch (class) + { + case H5T_STRING: + type->nc_type_class = NC_STRING; + break; + + case H5T_COMPOUND: + { + int nmembers; + unsigned int m; + char* member_name = NULL; +#ifdef JNA + char jna[1001]; +#endif + + type->nc_type_class = NC_COMPOUND; + + if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0) + return NC_EHDFERR; + LOG((5, "compound type has %d members", nmembers)); + type->u.c.field = nclistnew(); + nclistsetalloc(type->u.c.field,nmembers); + + for (m = 0; m < nmembers; m++) + { + hid_t member_hdf_typeid; + hid_t member_native_typeid; + size_t member_offset; + H5T_class_t mem_class; + nc_type member_xtype; + + /* Get the typeid and native typeid of this member of the + * compound type. */ + if ((member_hdf_typeid = H5Tget_member_type(type->native_hdf_typeid, m)) < 0) + return NC_EHDFERR; + + if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, + H5T_DIR_DEFAULT)) < 0) + return NC_EHDFERR; + + /* Get the name of the member.*/ + member_name = H5Tget_member_name(type->native_hdf_typeid, m); + if (!member_name || strlen(member_name) > NC_MAX_NAME) { + retval = NC_EBADNAME; + break; + } +#ifdef JNA + else { + strncpy(jna,member_name,1000); + member_name = jna; + } +#endif + + /* Offset in bytes on *this* platform. */ + member_offset = H5Tget_member_offset(type->native_hdf_typeid, m); + + /* Get dimensional data if this member is an array of something. */ + if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0) + return NC_EHDFERR; + if (mem_class == H5T_ARRAY) + { + int ndims, dim_size[NC_MAX_VAR_DIMS]; + hsize_t dims[NC_MAX_VAR_DIMS]; + int d; + + if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0) { + retval = NC_EHDFERR; + break; + } + if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims) { + retval = NC_EHDFERR; + break; + } + for (d = 0; d < ndims; d++) + dim_size[d] = dims[d]; + + /* What is the netCDF typeid of this member? */ + if ((retval = get_netcdf_type(grp->nc4_info, H5Tget_super(member_hdf_typeid), + &member_xtype))) + break; + + /* Add this member to our list of fields in this compound type. */ + if ((retval = nc4_field_list_add(type, member_name, + member_offset, H5Tget_super(member_hdf_typeid), + H5Tget_super(member_native_typeid), + member_xtype, ndims, dim_size))) + break; + } + else + { + /* What is the netCDF typeid of this member? */ + if ((retval = get_netcdf_type(grp->nc4_info, member_native_typeid, + &member_xtype))) + break; + + /* Add this member to our list of fields in this compound type. */ + if ((retval = nc4_field_list_add(type, member_name, + member_offset, member_hdf_typeid, member_native_typeid, + member_xtype, 0, NULL))) + break; + } + + hdf5free(member_name); + member_name = NULL; + } + hdf5free(member_name); + member_name = NULL; + if(retval) /* error exit from loop */ + return retval; + } + break; + + case H5T_VLEN: + { + htri_t ret; + + /* For conveninence we allow user to pass vlens of strings + * with null terminated strings. This means strings are + * treated slightly differently by the API, although they are + * really just VLENs of characters. */ + if ((ret = H5Tis_variable_str(hdf_typeid)) < 0) + return NC_EHDFERR; + if (ret) + type->nc_type_class = NC_STRING; + else + { + hid_t base_hdf_typeid; + nc_type base_nc_type = NC_NAT; + + type->nc_type_class = NC_VLEN; + + /* Find the base type of this vlen (i.e. what is this a + * vlen of?) */ + if (!(base_hdf_typeid = H5Tget_super(native_typeid))) + return NC_EHDFERR; + + /* What size is this type? */ + if (!(type_size = H5Tget_size(base_hdf_typeid))) + return NC_EHDFERR; + + /* What is the netcdf corresponding type. */ + if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid, + &base_nc_type))) + return retval; + LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d", + base_hdf_typeid, type_size, base_nc_type)); + + /* Remember the base types for this vlen */ + type->u.v.base_nc_typeid = base_nc_type; + type->u.v.base_hdf_typeid = base_hdf_typeid; + } + } + break; + + case H5T_OPAQUE: + type->nc_type_class = NC_OPAQUE; + break; + + case H5T_ENUM: + { + hid_t base_hdf_typeid; + nc_type base_nc_type = NC_NAT; + void *value; + int i; + char *member_name = NULL; +#ifdef JNA + char jna[1001]; +#endif + + type->nc_type_class = NC_ENUM; + + /* Find the base type of this enum (i.e. what is this a + * enum of?) */ + if (!(base_hdf_typeid = H5Tget_super(hdf_typeid))) + return NC_EHDFERR; + /* What size is this type? */ + if (!(type_size = H5Tget_size(base_hdf_typeid))) + return NC_EHDFERR; + /* What is the netcdf corresponding type. */ + if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid, + &base_nc_type))) + return retval; + LOG((5, "base_hdf_typeid 0x%x type_size %d base_nc_type %d", + base_hdf_typeid, type_size, base_nc_type)); + + /* Remember the base types for this enum */ + type->u.e.base_nc_typeid = base_nc_type; + type->u.e.base_hdf_typeid = base_hdf_typeid; + + /* Find out how many member are in the enum. */ + if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0) + return NC_EHDFERR; + type->u.e.enum_member = nclistnew(); + nclistsetalloc(type->u.e.enum_member,nmembers); + + /* Allocate space for one value. */ + if (!(value = calloc(1, type_size))) + return NC_ENOMEM; + + /* Read each name and value defined in the enum. */ + for (i = 0; i < nmembers; i++) + { + /* Get the name and value from HDF5. */ + if (!(member_name = H5Tget_member_name(hdf_typeid, i))) + { + retval = NC_EHDFERR; + break; + } +#ifdef JNA + strncpy(jna,member_name,1000); + member_name = jna; +#endif + + if (strlen(member_name) > NC_MAX_NAME) + { + retval = NC_EBADNAME; + break; + } + if (H5Tget_member_value(hdf_typeid, i, value) < 0) + { + retval = NC_EHDFERR; + break; + } + + /* Insert new field into this type's list of fields. */ + if ((retval = nc4_enum_member_add(type, type->size, + member_name, value))) + { + break; + } + + hdf5free(member_name); + member_name = NULL; + } + hdf5free(member_name); + member_name = NULL; + if(value) free(value); + if(retval) /* error exit from loop */ + return retval; + } + break; + + default: + LOG((0, "unknown class")); + return NC_EBADCLASS; + } + return retval; +} + +/** + * @internal Callback function for reading attributes. This is used by + * read_var(). + * + * @param loc_id HDF5 attribute ID. + * @param att_name Name of the attrigute. + * @param ainfo HDF5 info struct for attribute. + * @param att_data The attribute data. + * + * @return ::NC_NOERR No error. + * @return ::NC_EHDFERR HDF5 returned error. + * @return ::NC_ENOMEM Out of memory. + * @return ::NC_EATTMETA HDF5 can't open attribute. + * @return ::NC_EBADTYPID Can't read attribute type. + */ +static herr_t +att_read_var_callbk(hid_t loc_id, const char *att_name, + const H5A_info_t *ainfo, void *att_data) +{ + + hid_t attid = 0; + int retval = NC_NOERR; + NC_ATT_INFO_T *att; + att_iter_info *att_info = (att_iter_info *)att_data; + + /* Should we ignore this attribute? */ + const NC_reservedatt* ra = NC_findreserved(att_name); + if(ra != NULL) goto exit; /* ignore */ + + /* Add to the end of the list of atts for this var. */ + if ((retval = nc4_att_list_add(att_info->var->att, att_name, &att))) + BAIL(retval); + + /* Open the att by name. */ + if ((attid = H5Aopen(loc_id, att_name, H5P_DEFAULT)) < 0) + BAIL(NC_EATTMETA); + LOG((4, "%s:: att_name %s", __func__, att_name)); + + /* Read the rest of the info about the att, + * including its values. */ + if ((retval = read_hdf5_att(att_info->grp, attid, att))) + BAIL(retval); + + if (att) + att->created = NC_TRUE; + + if (attid > 0 && H5Aclose(attid) < 0) + BAIL2(NC_EHDFERR); + + return NC_NOERR; + +exit: + if(retval) { + if (retval == NC_EBADTYPID) { + /* NC_EBADTYPID will be normally converted to NC_NOERR so that + the parent iterator does not fail. */ + retval = nc4_att_list_del(att_info->var->att,att); + att = NULL; + } + } + if (attid > 0 && H5Aclose(attid) < 0) + retval = NC_EHDFERR; + return retval; +} + +/** + * @internal This function reads all the attributes of a variable. + * + * @param grp Pointer to the group info. + * @param var Pointer to the var info. + * + * @return NC_NOERR No error. + * @author Ed Hartnett + */ +int +nc4_read_var_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) +{ + att_iter_info att_info; /* Custom iteration information */ + + /* Check inputs. */ + assert(grp && var); + + /* Assign var and grp in struct. */ + att_info.var = var; + att_info.grp = grp; + + /* Now read all the attributes of this variable, ignoring the + ones that hold HDF5 dimension scale information. */ + if ((H5Aiterate2(var->hdf_datasetid, H5_INDEX_CRT_ORDER, H5_ITER_INC, NULL, + att_read_var_callbk, &att_info)) < 0) + return NC_EATTMETA; + + /* Remember that we have read the atts for this var. */ + var->atts_not_read = 0; + + return NC_NOERR; +} + +/** + * @internal This function is called by read_dataset when a dimension + * scale dataset is encountered. It reads in the dimension data + * (creating a new NC_DIM_INFO_T object), and also checks to see if + * this is a dimension without a variable - that is, a coordinate + * dimension which does not have any coordinate data. + * + * @param grp Pointer to group info struct. + * @param datasetid The HDF5 dataset ID. + * @param obj_name + * @param statbuf + * @param scale_size Size of dimension scale. + * @param max_scale_size Maximum size of dim scale. + * @param dim Pointer to pointer that gets new dim info struct. + * + * @returns ::NC_NOERR No error. + * @return ::NC_EHDFERR HDF5 returned error. + * @author Ed Hartnett + */ +static int +read_scale(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, + const H5G_stat_t *statbuf, hsize_t scale_size, hsize_t max_scale_size, + NC_DIM_INFO_T **dim) +{ + NC_DIM_INFO_T *new_dim; /* Dimension added to group */ + char dimscale_name_att[NC_MAX_NAME + 1] = ""; /* Dimscale name, for checking if dim without var */ + htri_t attr_exists = -1; /* Flag indicating hidden attribute exists */ + hid_t attid = -1; /* ID of hidden attribute (to store dim ID) */ + int dimscale_created = 0; /* Remember if a dimension was created (for error recovery) */ + short initial_next_dimid = grp->nc4_info->next_dimid;/* Retain for error recovery */ + int retval; + size_t len = 0; + int too_long = NC_FALSE; + int assigned_id = -1; + + /* Does this dataset have a hidden attribute that tells us its + * dimid? If so, read it. */ + if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0) + BAIL(NC_EHDFERR); + if (attr_exists) + { + if ((attid = H5Aopen_name(datasetid, NC_DIMID_ATT_NAME)) < 0) + BAIL(NC_EHDFERR); + + if (H5Aread(attid, H5T_NATIVE_INT, &assigned_id) < 0) + BAIL(NC_EHDFERR); + + /* Check if scale's dimid should impact the group's next dimid */ + if (assigned_id >= grp->nc4_info->next_dimid) + grp->nc4_info->next_dimid = assigned_id + 1; + } + + if (SIZEOF_SIZE_T < 8 && scale_size > NC_MAX_UINT) + { + len = NC_MAX_UINT; + too_long = NC_TRUE; + } + else + len = scale_size; + + /* Create the dimension for this scale. */ + if ((retval = nc4_dim_list_add(grp, obj_name, len, assigned_id, &new_dim))) + BAIL(retval); + + new_dim->too_long = too_long; + + dimscale_created++; + + new_dim->hdf5_objid.fileno[0] = statbuf->fileno[0]; + new_dim->hdf5_objid.fileno[1] = statbuf->fileno[1]; + new_dim->hdf5_objid.objno[0] = statbuf->objno[0]; + new_dim->hdf5_objid.objno[1] = statbuf->objno[1]; + + /* If the dimscale has an unlimited dimension, then this dimension + * is unlimited. */ + if (max_scale_size == H5S_UNLIMITED) + new_dim->unlimited = NC_TRUE; + + /* If the scale name is set to DIM_WITHOUT_VARIABLE, then this is a + * dimension, but not a variable. (If get_scale_name returns an + * error, just move on, there's no NAME.) */ + if (H5DSget_scale_name(datasetid, dimscale_name_att, NC_MAX_NAME) >= 0) + { + if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE, + strlen(DIM_WITHOUT_VARIABLE))) + { + if (new_dim->unlimited) + { + size_t len = 0, *lenp = &len; + + if ((retval = nc4_find_dim_len(grp, new_dim->hdr.id, &lenp))) + BAIL(retval); + new_dim->len = *lenp; + } + + /* Hold open the dataset, since the dimension doesn't have a + * coordinate variable */ + new_dim->hdf_dimscaleid = datasetid; + H5Iinc_ref(new_dim->hdf_dimscaleid); /* Increment number of objects using ID */ + } + } + + /* Set the dimension created */ + *dim = new_dim; + +exit: + /* Close the hidden attribute, if it was opened (error, or no + * error) */ + if (attid > 0 && H5Aclose(attid) < 0) + BAIL2(NC_EHDFERR); + + /* On error, undo any dimscale creation */ + if (retval < 0 && dimscale_created) + { + /* free the dimension */ + if ((retval = nc4_dim_list_del(grp, new_dim))) + BAIL2(retval); + + /* Reset the group's information */ + grp->nc4_info->next_dimid = initial_next_dimid; + } + + return retval; +} + +/** + * @internal This function is called when nc4_rec_read_metadata + * encounters an HDF5 dataset when reading a file. + * + * @param grp Pointer to group info struct. + * @param datasetid HDF5 dataset ID. + * @param obj_name Object name. + * @param statbuf HDF5 status buffer. + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID Bad ncid. + * @return ::NC_EHDFERR HDF5 returned error. + * @author Ed Hartnett + */ +static int +read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, + const H5G_stat_t *statbuf) +{ + NC_DIM_INFO_T *dim = NULL; /* Dimension created for scales */ + hid_t spaceid = 0; + int ndims; + htri_t is_scale; + int retval = NC_NOERR; + + /* Get the dimension information for this dataset. */ + if ((spaceid = H5Dget_space(datasetid)) < 0) + BAIL(NC_EHDFERR); + if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0) + BAIL(NC_EHDFERR); + + /* Is this a dimscale? */ + if ((is_scale = H5DSis_scale(datasetid)) < 0) + BAIL(NC_EHDFERR); + if (is_scale) + { + hsize_t dims[H5S_MAX_RANK]; + hsize_t max_dims[H5S_MAX_RANK]; + + /* Query the scale's size & max. size */ + if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0) + BAIL(NC_EHDFERR); + + /* Read the scale information. */ + if ((retval = read_scale(grp, datasetid, obj_name, statbuf, dims[0], + max_dims[0], &dim))) + BAIL(retval); + } + + /* Add a var to the linked list, and get its metadata, + * unless this is one of those funny dimscales that are a + * dimension in netCDF but not a variable. (Spooky!) */ + if (NULL == dim || (dim && !dim->hdf_dimscaleid)) + if ((retval = read_var(grp, datasetid, obj_name, ndims, dim))) + BAIL(retval); + +exit: + if (spaceid && H5Sclose(spaceid) <0) + BAIL2(retval); + + return retval; +} + +/** + * @internal Add callback function to list. + * + * @param udata - the callback state + * @param oinfo The object info. + * + * @return ::NC_NOERR No error. + * @return ::NC_ENOMEM Out of memory. + * @author Ed Hartnett + */ +static int +nc4_rec_read_metadata_cb_list_add(NC4_rec_read_metadata_ud_t* udata, + const NC4_rec_read_metadata_obj_info_t *oinfo) +{ + NC4_rec_read_metadata_obj_info_t *new_oinfo; /* Pointer to info for object */ + + /* Allocate memory for the object's info */ + if (!(new_oinfo = calloc(1, sizeof(*new_oinfo)))) + return NC_ENOMEM; + + /* Make a copy of the object's info */ + memcpy(new_oinfo, oinfo, sizeof(*oinfo)); + + nclistpush(udata->grps,new_oinfo); + return (NC_NOERR); +} + +/** + * @internal Callback function called from nc4_rec_read_metadata(). + * + * @param grpid HDF5 group ID. + * @param name Name of object. + * @param info Info struct for object. + * @param _op_data Pointer to data. + * + * @return ::NC_NOERR No error. + * @return H5_ITER_ERROR HDF5 error. + * @author Ed Hartnett + */ +static int +nc4_rec_read_metadata_cb(hid_t grpid, const char *name, const H5L_info_t *info, + void *_op_data) +{ + NC4_rec_read_metadata_ud_t *udata = (NC4_rec_read_metadata_ud_t *)_op_data; /* Pointer to user data for callback */ + NC4_rec_read_metadata_obj_info_t oinfo; /* Pointer to info for object */ + int retval = H5_ITER_CONT; + + /* Reset the memory for the object's info */ + memset(&oinfo, 0, sizeof(oinfo)); + + /* Open this critter. */ + if ((oinfo.oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0) + BAIL(H5_ITER_ERROR); + + /* Get info about the object.*/ + if (H5Gget_objinfo(oinfo.oid, ".", 1, &oinfo.statbuf) < 0) + BAIL(H5_ITER_ERROR); + + strncpy(oinfo.oname, name, NC_MAX_NAME); + + /* Add object to list, for later */ + switch(oinfo.statbuf.type) + { + case H5G_GROUP: + LOG((3, "found group %s", oinfo.oname)); + + /* Defer descending into child group immediately, so that the types + * in the current group can be processed and be ready for use by + * vars in the child group(s). + */ + if (nc4_rec_read_metadata_cb_list_add(udata, &oinfo)) + BAIL(H5_ITER_ERROR); + break; + + case H5G_DATASET: + LOG((3, "found dataset %s", oinfo.oname)); + + /* Learn all about this dataset, which may be a dimscale + * (i.e. dimension metadata), or real data. */ + if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, + &oinfo.statbuf))) + { + /* Allow NC_EBADTYPID to transparently skip over datasets + * which have a datatype that netCDF-4 doesn't undertand + * (currently), but break out of iteration for other + * errors. + */ + if(NC_EBADTYPID != retval) + BAIL(H5_ITER_ERROR); + else + retval = H5_ITER_CONT; + } + + /* Close the object */ + if (H5Oclose(oinfo.oid) < 0) + BAIL(H5_ITER_ERROR); + break; + + case H5G_TYPE: + LOG((3, "found datatype %s", oinfo.oname)); + + /* Process the named datatype */ + if (read_type(udata->grp, oinfo.oid, oinfo.oname)) + BAIL(H5_ITER_ERROR); + + /* Close the object */ + if (H5Oclose(oinfo.oid) < 0) + BAIL(H5_ITER_ERROR); + break; + + default: + LOG((0, "Unknown object class %d in %s!", oinfo.statbuf.type, __func__)); + BAIL(H5_ITER_ERROR); + } + +exit: + if (retval) + { + if (oinfo.oid > 0 && H5Oclose(oinfo.oid) < 0) + BAIL2(H5_ITER_ERROR); + } + + return (retval); +} + +/** + * @internal This is the main function to recursively read all the + * metadata for the file. The links in the 'grp' are iterated over + * and added to the file's metadata information. Note that child + * groups are not immediately processed, but are deferred until all + * the other links in the group are handled (so that vars in the child + * groups are guaranteed to have types that they use in a parent group + * in place). + * + * @param grp Pointer to a group. + * + * @return ::NC_NOERR No error. + * @author Ed Hartnett + */ +static int +nc4_rec_read_metadata(NC_GRP_INFO_T *grp) +{ + NC4_rec_read_metadata_ud_t udata; /* User data for iteration */ + NC4_rec_read_metadata_obj_info_t *oinfo; /* Pointer to info for object */ + hsize_t idx=0; + hid_t pid = 0; + unsigned crt_order_flags = 0; + H5_index_t iter_index; + int i, retval = NC_NOERR; /* everything worked! */ + + assert(grp && grp->hdr.name); + LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name)); + + /* Portably initialize user data for later */ + memset(&udata, 0, sizeof(udata)); + + /* Open this HDF5 group and retain its grpid. It will remain open + * with HDF5 until this file is nc_closed. */ + if (!grp->hdf_grpid) + { + if (grp->parent) + { + if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid, + grp->hdr.name, H5P_DEFAULT)) < 0) + BAIL(NC_EHDFERR); + } + else + { + if ((grp->hdf_grpid = H5Gopen2(grp->nc4_info->hdfid, + "/", H5P_DEFAULT)) < 0) + BAIL(NC_EHDFERR); + } + } + assert(grp->hdf_grpid > 0); + + /* Get the group creation flags, to check for creation ordering */ + pid = H5Gget_create_plist(grp->hdf_grpid); + H5Pget_link_creation_order(pid, &crt_order_flags); + if (H5Pclose(pid) < 0) + BAIL(NC_EHDFERR); + + /* Set the iteration index to use */ + if (crt_order_flags & H5P_CRT_ORDER_TRACKED) + iter_index = H5_INDEX_CRT_ORDER; + else + { + NC_FILE_INFO_T *h5 = grp->nc4_info; + + /* Without creation ordering, file must be read-only. */ + if (!h5->no_write) + BAIL(NC_ECANTWRITE); + + iter_index = H5_INDEX_NAME; + } + + /* Set user data for iteration */ + udata.grp = grp; + udata.grps = nclistnew(); + + /* Iterate over links in this group, building lists for the types, + * datasets and groups encountered. */ + if (H5Literate(grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, + nc4_rec_read_metadata_cb, (void *)&udata) < 0) + BAIL(NC_EHDFERR); + + /* Process the child groups found. (Deferred until now, so that the + * types in the current group get processed and are available for + * vars in the child group(s).) */ + for(i=0;ioname, &child_grp))) + BAIL(retval); + + /* Recursively read the child group's metadata */ + if ((retval = nc4_rec_read_metadata(child_grp))) + BAIL(retval); + + /* Close the object */ + if (H5Oclose(oinfo->oid) < 0) + BAIL(NC_EHDFERR); + } + + /* Defer the reading of global atts until someone asks for one. */ + grp->atts_not_read = 1; + + /* when exiting define mode, mark all variable written */ + for (i=0; ivars); i++) { + NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i); + if(var == NULL) continue; + var->written_to = NC_TRUE; + } + +exit: + /* Clean up local information, if anything remains */ + for(i=0;ioid) < 0) + BAIL2(NC_EHDFERR); + } + free(oinfo); + } + nclistfree(udata.grps); + udata.grps = NULL; + + return retval; +} diff --git a/libhdf5/hdf5type.c b/libhdf5/hdf5type.c index 9a113e4bd..c86a283a3 100644 --- a/libhdf5/hdf5type.c +++ b/libhdf5/hdf5type.c @@ -12,6 +12,7 @@ * @author Ed Hartnett */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c new file mode 100644 index 000000000..a5e8ee2f1 --- /dev/null +++ b/libhdf5/hdf5var.c @@ -0,0 +1,1250 @@ +/* Copyright 2003-2006, University Corporation for Atmospheric + * Research. See COPYRIGHT file for copying and redistribution + * conditions.*/ +/** + * @file + * @internal This file handles the HDF5 variable functions. + * + * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher + */ + +#include "config.h" +#include +#include + +/** @internal Default size for unlimited dim chunksize */ +#define DEFAULT_1D_UNLIM_SIZE (4096) + +/** + * @internal If the HDF5 dataset for this variable is open, then close + * it and reopen it, with the perhaps new settings for chunk caching. + * + * @param grp Pointer to the group info. + * @param var Pointer to the var info. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EHDFERR HDF5 error. + * @author Ed Hartnett + */ +int +nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) +{ + hid_t access_pid; + + if (var->hdf_datasetid) + { + if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + return NC_EHDFERR; + if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems, + var->chunk_cache_size, + var->chunk_cache_preemption) < 0) + return NC_EHDFERR; + if (H5Dclose(var->hdf_datasetid) < 0) + return NC_EHDFERR; + if ((var->hdf_datasetid = H5Dopen2(grp->hdf_grpid, var->hdr.name, + access_pid)) < 0) + return NC_EHDFERR; + if (H5Pclose(access_pid) < 0) + return NC_EHDFERR; + } + + return NC_NOERR; +} + +/** + * @internal Check a set of chunksizes to see if they specify a chunk + * that is too big. + * + * @param grp Pointer to the group info. + * @param var Pointer to the var info. + * @param chunksizes Array of chunksizes to check. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_EBADCHUNK Bad chunksize. + */ +static int +check_chunksizes(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, const size_t *chunksizes) +{ + double dprod; + size_t type_len; + int d; + int retval; + + if ((retval = nc4_get_typelen_mem(grp->nc4_info, var->type_info->hdr.id, &type_len))) + return retval; + if (var->type_info->nc_type_class == NC_VLEN) + dprod = (double)sizeof(hvl_t); + else + dprod = (double)type_len; + for (d = 0; d < var->ndims; d++) + dprod *= (double)chunksizes[d]; + + if (dprod > (double) NC_MAX_UINT) + return NC_EBADCHUNK; + + return NC_NOERR; +} + +/** + * @internal Determine some default chunksizes for a variable. + * + * @param grp Pointer to the group info. + * @param var Pointer to the var info. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @author Ed Hartnett + */ +static int +nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) +{ + int d; + size_t type_size; + float num_values = 1, num_unlim = 0; + int retval; + size_t suggested_size; +#ifdef LOGGING + double total_chunk_size; +#endif + + if (var->type_info->nc_type_class == NC_STRING) + type_size = sizeof(char *); + else + type_size = var->type_info->size; + +#ifdef LOGGING + /* Later this will become the total number of bytes in the default + * chunk. */ + total_chunk_size = (double) type_size; +#endif + + /* How many values in the variable (or one record, if there are + * unlimited dimensions). */ + for (d = 0; d < var->ndims; d++) + { + assert(var->dim[d]); + if (! var->dim[d]->unlimited) + num_values *= (float)var->dim[d]->len; + else { + num_unlim++; + var->chunksizes[d] = 1; /* overwritten below, if all dims are unlimited */ + } + } + /* Special case to avoid 1D vars with unlim dim taking huge amount + of space (DEFAULT_CHUNK_SIZE bytes). Instead we limit to about + 4KB */ + if (var->ndims == 1 && num_unlim == 1) { + if (DEFAULT_CHUNK_SIZE / type_size <= 0) + suggested_size = 1; + else if (DEFAULT_CHUNK_SIZE / type_size > DEFAULT_1D_UNLIM_SIZE) + suggested_size = DEFAULT_1D_UNLIM_SIZE; + else + suggested_size = DEFAULT_CHUNK_SIZE / type_size; + var->chunksizes[0] = suggested_size / type_size; + LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " + "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[0])); + } + if (var->ndims > 1 && var->ndims == num_unlim) { /* all dims unlimited */ + suggested_size = pow((double)DEFAULT_CHUNK_SIZE/type_size, 1.0/(double)(var->ndims)); + for (d = 0; d < var->ndims; d++) + { + var->chunksizes[d] = suggested_size ? suggested_size : 1; + LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " + "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d])); + } + } + + /* Pick a chunk length for each dimension, if one has not already + * been picked above. */ + for (d = 0; d < var->ndims; d++) + if (!var->chunksizes[d]) + { + suggested_size = (pow((double)DEFAULT_CHUNK_SIZE/(num_values * type_size), + 1.0/(double)(var->ndims - num_unlim)) * var->dim[d]->len - .5); + if (suggested_size > var->dim[d]->len) + suggested_size = var->dim[d]->len; + var->chunksizes[d] = suggested_size ? suggested_size : 1; + LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " + "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d])); + } + +#ifdef LOGGING + /* Find total chunk size. */ + for (d = 0; d < var->ndims; d++) + total_chunk_size *= (double) var->chunksizes[d]; + LOG((4, "total_chunk_size %f", total_chunk_size)); +#endif + + /* But did this result in a chunk that is too big? */ + retval = check_chunksizes(grp, var, var->chunksizes); + if (retval) + { + /* Other error? */ + if (retval != NC_EBADCHUNK) + return retval; + + /* Chunk is too big! Reduce each dimension by half and try again. */ + for ( ; retval == NC_EBADCHUNK; retval = check_chunksizes(grp, var, var->chunksizes)) + for (d = 0; d < var->ndims; d++) + var->chunksizes[d] = var->chunksizes[d]/2 ? var->chunksizes[d]/2 : 1; + } + + /* Do we have any big data overhangs? They can be dangerous to + * babies, the elderly, or confused campers who have had too much + * beer. */ + for (d = 0; d < var->ndims; d++) + { + size_t num_chunks; + size_t overhang; + assert(var->chunksizes[d] > 0); + num_chunks = (var->dim[d]->len + var->chunksizes[d] - 1) / var->chunksizes[d]; + if(num_chunks > 0) { + overhang = (num_chunks * var->chunksizes[d]) - var->dim[d]->len; + var->chunksizes[d] -= overhang / num_chunks; + } + } + + return NC_NOERR; +} + +/** + * @internal This is called when a new netCDF-4 variable is defined + * with nc_def_var(). + * + * @param ncid File ID. + * @param name Name. + * @param xtype Type. + * @param ndims Number of dims. + * @param dimidsp Array of dim IDs. + * @param varidp Gets the var ID. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 + * netcdf-4 file. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EPERM File is read only. + * @returns ::NC_EMAXDIMS Classic model file exceeds ::NC_MAX_VAR_DIMS. + * @returns ::NC_ESTRICTNC3 Attempting to create netCDF-4 type var in + * classic model file + * @returns ::NC_EBADNAME Bad name. + * @returns ::NC_EBADTYPE Bad type. + * @returns ::NC_ENOMEM Out of memory. + * @returns ::NC_EHDFERR Error returned by HDF5 layer. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_def_var(int ncid, const char *name, nc_type xtype, + int ndims, const int *dimidsp, int *varidp) +{ + NC_GRP_INFO_T *grp; + NC_VAR_INFO_T *var; + NC_DIM_INFO_T *dim; + NC_FILE_INFO_T *h5; + NC_TYPE_INFO_T *type_info = NULL; + char norm_name[NC_MAX_NAME + 1]; + int d; + int retval; + + /* Find info for this file and group, and set pointer to each. */ + if ((retval = nc4_find_grp_h5(ncid, &grp, &h5))) + BAIL(retval); + assert(grp && h5); + + /* If it's not in define mode, strict nc3 files error out, + * otherwise switch to define mode. This will also check that the + * file is writable. */ + if (!(h5->flags & NC_INDEF)) + { + if (h5->cmode & NC_CLASSIC_MODEL) + BAIL(NC_ENOTINDEFINE); + if ((retval = NC4_redef(ncid))) + BAIL(retval); + } + assert(!h5->no_write); + + /* Check and normalize the name. */ + if ((retval = nc4_check_name(name, norm_name))) + BAIL(retval); + + /* Not a Type is, well, not a type.*/ + if (xtype == NC_NAT) + BAIL(NC_EBADTYPE); + + /* For classic files, only classic types are allowed. */ + if (h5->cmode & NC_CLASSIC_MODEL && xtype > NC_DOUBLE) + BAIL(NC_ESTRICTNC3); + + /* For classic files */ + if (h5->cmode & NC_CLASSIC_MODEL && ndims > NC_MAX_VAR_DIMS) + BAIL(NC_EMAXDIMS); + + /* cast needed for braindead systems with signed size_t */ + if((unsigned long) ndims > X_INT_MAX) /* Backward compat */ + BAIL(NC_EINVAL); + + /* Check that this name is not in use as a var, grp, or type. */ + if ((retval = nc4_check_dup_name(grp, norm_name))) + BAIL(retval); + + /* For non-scalar vars, dim IDs must be provided. */ + if (ndims && !dimidsp) + BAIL(NC_EINVAL); + + /* Check all the dimids to make sure they exist. */ + for (d = 0; d < ndims; d++) + if ((retval = nc4_find_dim(grp, dimidsp[d], &dim, NULL))) + BAIL(retval); + + /* These degrubbing messages sure are handy! */ + LOG((2, "%s: name %s type %d ndims %d", __func__, norm_name, xtype, ndims)); +#ifdef LOGGING + { + int dd; + for (dd = 0; dd < ndims; dd++) + LOG((4, "dimid[%d] %d", dd, dimidsp[dd])); + } +#endif + + /* If this is a user-defined type, there is a type_info struct with + * all the type information. For atomic types, fake up a type_info + * struct. */ + if (xtype <= NC_STRING) + { + size_t len; + char name[NC_MAX_NAME+1]; + char* atomname = nc4_atomic_name[xtype]; + size_t namelen = strlen(atomname); + memcpy(name,atomname,namelen); + name[namelen] = '\0'; + nc4_get_typelen_mem(h5,xtype,&len); + if((retval = nc4_type_new(grp,len,name,xtype,&type_info))) + BAIL(retval); + type_info->endianness = NC_ENDIAN_NATIVE; + if ((retval = nc4_get_hdf_typeid(h5, xtype, &type_info->hdf_typeid, + type_info->endianness))) + BAIL(retval); + if ((type_info->native_hdf_typeid = H5Tget_native_type(type_info->hdf_typeid, + H5T_DIR_DEFAULT)) < 0) + BAIL(NC_EHDFERR); + if ((retval = nc4_get_typelen_mem(h5, type_info->hdr.id, &type_info->size))) + BAIL(retval); + + /* Set the "class" of the type */ + if (xtype == NC_CHAR) + type_info->nc_type_class = NC_CHAR; + else + { + H5T_class_t class; + + if ((class = H5Tget_class(type_info->hdf_typeid)) < 0) + BAIL(NC_EHDFERR); + switch(class) + { + case H5T_STRING: + type_info->nc_type_class = NC_STRING; + break; + + case H5T_INTEGER: + type_info->nc_type_class = NC_INT; + break; + + case H5T_FLOAT: + type_info->nc_type_class = NC_FLOAT; + break; + + default: + BAIL(NC_EBADTYPID); + } + } + } + /* If this is a user defined type, find it. */ + else + { + if (nc4_find_type(grp->nc4_info, xtype, &type_info)) + BAIL(NC_EBADTYPE); + } + + /* Create a new var and fill in some HDF5 cache setting values. */ + if ((retval = nc4_var_list_add(grp, norm_name, ndims, &var))) + BAIL(retval); + var->is_new_var = NC_TRUE; + + /* Point to the type, and increment its ref. count */ + var->type_info = type_info; + var->type_info->rc++; + type_info = NULL; + + /* Set variables no_fill to match the database default + * unless the variable type is variable length (NC_STRING or NC_VLEN) + * or is user-defined type. + */ + if (var->type_info->nc_type_class < NC_STRING) + var->no_fill = h5->fill_mode; + + /* Assign dimensions to the variable */ + /* At the same time, check to see if this is a coordinate + * variable. If so, it will have the same name as one of its + * dimensions. If it is a coordinate var, is it a coordinate var in + * the same group as the dim? */ + /* Also, check whether we should use contiguous or chunked storage */ + var->contiguous = NC_TRUE; + for (d = 0; d < ndims; d++) + { + NC_GRP_INFO_T *dim_grp; + + /* Look up each dimension */ + if ((retval = nc4_find_dim(grp, dimidsp[d], &dim, &dim_grp))) + BAIL(retval); + + /* Check for dim index 0 having the same name, in the same group */ + if (d == 0 && dim_grp == grp && strcmp(dim->hdr.name, norm_name) == 0) + { + var->dimscale = NC_TRUE; + dim->coord_var = var; + + /* Use variable's dataset ID for the dimscale ID. So delete + * the HDF5 DIM_WITHOUT_VARIABLE dataset that was created for + * this dim. */ + if (dim->hdf_dimscaleid) + { + /* Detach dimscale from any variables using it */ + if ((retval = rec_detach_scales(grp, dimidsp[d], dim->hdf_dimscaleid)) < 0) + BAIL(retval); + + /* Close the HDF5 DIM_WITHOUT_VARIABLE dataset. */ + if (H5Dclose(dim->hdf_dimscaleid) < 0) + BAIL(NC_EHDFERR); + dim->hdf_dimscaleid = 0; + + /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be + * recreated later, if necessary). */ + if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) + BAIL(NC_EDIMMETA); + } + } + + /* Check for unlimited dimension and turn off contiguous storage. */ + if (dim->unlimited) + var->contiguous = NC_FALSE; + + /* Track dimensions for variable */ + var->dimids[d] = dimidsp[d]; + var->dim[d] = dim; + } + + /* Determine default chunksizes for this variable. (Even for + * variables which may be contiguous.) */ + LOG((4, "allocating array of %d size_t to hold chunksizes for var %s", + var->ndims, var->hdr.name)); + if (var->ndims) + if (!(var->chunksizes = calloc(var->ndims, sizeof(size_t)))) + BAIL(NC_ENOMEM); + + if ((retval = nc4_find_default_chunksizes2(grp, var))) + BAIL(retval); + + /* Is this a variable with a chunksize greater than the current + * cache size? */ + if ((retval = nc4_adjust_var_cache(grp, var))) + BAIL(retval); + + /* If the user names this variable the same as a dimension, but + * doesn't use that dimension first in its list of dimension ids, + * is not a coordinate variable. I need to change its HDF5 name, + * because the dimension will cause a HDF5 dataset to be created, + * and this var has the same name. */ + dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); + if (dim && (!var->ndims || dimidsp[0] != dim->hdr.id)) + { + /* Set a different hdf5 name for this variable to avoid name + * clash. */ + if (strlen(norm_name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) + BAIL(NC_EMAXNAME); + if (!(var->hdf5_name = malloc((strlen(NON_COORD_PREPEND) + + strlen(norm_name) + 1) * sizeof(char)))) + BAIL(NC_ENOMEM); + + sprintf(var->hdf5_name, "%s%s", NON_COORD_PREPEND, norm_name); + } + + /* If this is a coordinate var, it is marked as a HDF5 dimension + * scale. (We found dim above.) Otherwise, allocate space to + * remember whether dimension scales have been attached to each + * dimension. */ + if (!var->dimscale && ndims) + if (!(var->dimscale_attached = calloc(ndims, sizeof(nc_bool_t)))) + BAIL(NC_ENOMEM); + + /* Return the varid. */ + if (varidp) + *varidp = var->hdr.id; + LOG((4, "new varid %d", var->hdr.id)); + +exit: + if (type_info) + if ((retval = nc4_type_free(type_info))) + BAIL2(retval); + + return retval; +} + +/** + * @internal This functions sets extra stuff about a netCDF-4 variable which + * must be set before the enddef but after the def_var. + * + * @note All pointer parameters may be NULL, in which case they are ignored. + * @param ncid File ID. + * @param varid Variable ID. + * @param shuffle Pointer to shuffle setting. + * @param deflate Pointer to deflate setting. + * @param deflate_level Pointer to deflate level. + * @param fletcher32 Pointer to fletcher32 setting. + * @param contiguous Pointer to contiguous setting. + * @param chunksizes Array of chunksizes. + * @param no_fill Pointer to no_fill setting. + * @param fill_value Pointer to fill value. + * @param endianness Pointer to endianness setting. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 + * netcdf-4 file. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EPERM File is read only. + * @returns ::NC_EINVAL Invalid input + * @returns ::NC_EBADCHUNK Bad chunksize. + * @author Ed Hartnett + */ +static int +nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, + int *deflate_level, int *fletcher32, int *contiguous, + const size_t *chunksizes, int *no_fill, + const void *fill_value, int *endianness) +{ + NC *nc; + NC_GRP_INFO_T *grp; + NC_FILE_INFO_T *h5; + NC_VAR_INFO_T *var; + int d; + int retval; + + /* All or none of these will be provided. */ + assert((deflate && deflate_level && shuffle) || + (!deflate && !deflate_level && !shuffle)); + + LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); + + /* Find info for this file and group, and set pointer to each. */ + if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return retval; + assert(nc && grp && h5); + + /* Trying to write to a read-only file? No way, Jose! */ + if (h5->no_write) + return NC_EPERM; + + /* Find the var. */ + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + if(!var) + return NC_ENOTVAR; + assert(var && var->hdr.id == varid); + + /* Can't turn on parallel and deflate/fletcher32/szip/shuffle. */ + if (nc->mode & (NC_MPIIO | NC_MPIPOSIX)) { + if (deflate || fletcher32 || shuffle) + return NC_EINVAL; + } + + /* If the HDF5 dataset has already been created, then it is too + * late to set all the extra stuff. */ + if (var->created) + return NC_ELATEDEF; + + /* Check compression options. */ + if (deflate && !deflate_level) + return NC_EINVAL; + + /* Valid deflate level? */ + if (deflate) + { + if (*deflate) + if (*deflate_level < NC_MIN_DEFLATE_LEVEL || + *deflate_level > NC_MAX_DEFLATE_LEVEL) + return NC_EINVAL; + + /* For scalars, just ignore attempt to deflate. */ + if (!var->ndims) + return NC_NOERR; + + /* Well, if we couldn't find any errors, I guess we have to take + * the users settings. Darn! */ + var->contiguous = NC_FALSE; + var->deflate = *deflate; + if (*deflate) + var->deflate_level = *deflate_level; + LOG((3, "%s: *deflate_level %d", __func__, *deflate_level)); + } + + /* Shuffle filter? */ + if (shuffle) + { + var->shuffle = *shuffle; + var->contiguous = NC_FALSE; + } + + /* Fletcher32 checksum error protection? */ + if (fletcher32) + { + var->fletcher32 = *fletcher32; + var->contiguous = NC_FALSE; + } + + /* Does the user want a contiguous dataset? Not so fast! Make sure + * that there are no unlimited dimensions, and no filters in use + * for this data. */ + if (contiguous && *contiguous) + { + if (var->deflate || var->fletcher32 || var->shuffle) + return NC_EINVAL; + + for (d = 0; d < var->ndims; d++) + if (var->dim[d]->unlimited) + return NC_EINVAL; + var->contiguous = NC_TRUE; + } + + /* Chunksizes anyone? */ + if (contiguous && *contiguous == NC_CHUNKED) + { + var->contiguous = NC_FALSE; + + /* If the user provided chunksizes, check that they are not too + * big, and that their total size of chunk is less than 4 GB. */ + if (chunksizes) + { + + if ((retval = check_chunksizes(grp, var, chunksizes))) + return retval; + + /* Ensure chunksize is smaller than dimension size */ + for (d = 0; d < var->ndims; d++) + if(!var->dim[d]->unlimited && var->dim[d]->len > 0 && chunksizes[d] > var->dim[d]->len) + return NC_EBADCHUNK; + + /* Set the chunksizes for this variable. */ + for (d = 0; d < var->ndims; d++) + var->chunksizes[d] = chunksizes[d]; + } + } + + /* Is this a variable with a chunksize greater than the current + * cache size? */ + if (!var->contiguous && (deflate || contiguous)) + { + /* Determine default chunksizes for this variable (do nothing + * for scalar vars). */ + if (var->chunksizes && !var->chunksizes[0]) + if ((retval = nc4_find_default_chunksizes2(grp, var))) + return retval; + + /* Adjust the cache. */ + if ((retval = nc4_adjust_var_cache(grp, var))) + return retval; + } + + /* Are we setting a fill modes? */ + if (no_fill) + { + if (*no_fill) + { + /* NC_STRING types may not turn off fill mode. It's disallowed + * by HDF5 and will cause a HDF5 error later. */ + if (*no_fill) + if (var->type_info->hdr.id == NC_STRING) + return NC_EINVAL; + + /* Set the no-fill mode. */ + var->no_fill = NC_TRUE; + } + else + var->no_fill = NC_FALSE; + } + + /* Are we setting a fill value? */ + if (fill_value && !var->no_fill) + { + /* Copy the fill_value. */ + LOG((4, "Copying fill value into metadata for variable %s", + var->hdr.name)); + + /* If there's a _FillValue attribute, delete it. */ + retval = NC4_del_att(ncid, varid, _FillValue); + if (retval && retval != NC_ENOTATT) + return retval; + + /* Create a _FillValue attribute. */ + if ((retval = nc_put_att(ncid, varid, _FillValue, var->type_info->hdr.id, 1, fill_value))) + return retval; + } + + /* Is the user setting the endianness? */ + if (endianness) + var->type_info->endianness = *endianness; + + return NC_NOERR; +} + +/** + * @internal Set compression settings on a variable. This is called by + * nc_def_var_deflate(). + * + * @param ncid File ID. + * @param varid Variable ID. + * @param shuffle True to turn on the shuffle filter. + * @param deflate True to turn on deflation. + * @param deflate_level A number between 0 (no compression) and 9 + * (maximum compression). + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate, + int deflate_level) +{ + return nc_def_var_extra(ncid, varid, &shuffle, &deflate, + &deflate_level, NULL, NULL, NULL, NULL, NULL, NULL); +} + +/** + * @internal Set checksum on a variable. This is called by + * nc_def_var_fletcher32(). + * + * @param ncid File ID. + * @param varid Variable ID. + * @param fletcher32 Pointer to fletcher32 setting. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_def_var_fletcher32(int ncid, int varid, int fletcher32) +{ + return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, &fletcher32, + NULL, NULL, NULL, NULL, NULL); +} + +/** + * @internal Define chunking stuff for a var. This is called by + * nc_def_var_chunking(). Chunking is required in any dataset with one + * or more unlimited dimensions in HDF5, or any dataset using a + * filter. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param contiguous Pointer to contiguous setting. + * @param chunksizesp Array of chunksizes. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EINVAL Invalid input + * @returns ::NC_EBADCHUNK Bad chunksize. + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_def_var_chunking(int ncid, int varid, int contiguous, const size_t *chunksizesp) +{ + return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, + &contiguous, chunksizesp, NULL, NULL, NULL); +} + +/** + * @internal Define chunking stuff for a var. This is called by + * the fortran API. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param contiguous Pointer to contiguous setting. + * @param chunksizesp Array of chunksizes. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EINVAL Invalid input + * @returns ::NC_EBADCHUNK Bad chunksize. + * @author Ed Hartnett + */ +int +nc_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp) +{ + NC *nc; + NC_GRP_INFO_T *grp; + NC_VAR_INFO_T *var; + NC_FILE_INFO_T *h5; + size_t *cs = NULL; + int i, retval; + + /* Find this ncid's file info. */ + if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return retval; + assert(nc); + + /* Find var cause I need the number of dims. */ + if ((retval = nc4_find_g_var_nc(nc, ncid, varid, &grp, &var))) + return retval; + + /* Allocate space for the size_t copy of the chunksizes array. */ + if (var->ndims) + if (!(cs = malloc(var->ndims * sizeof(size_t)))) + return NC_ENOMEM; + + /* Copy to size_t array. */ + for (i = 0; i < var->ndims; i++) + cs[i] = chunksizesp[i]; + + retval = nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, + &contiguous, cs, NULL, NULL, NULL); + + if (var->ndims) + free(cs); + return retval; +} + +/** + * @internal This functions sets fill value and no_fill mode for a + * netCDF-4 variable. It is called by nc_def_var_fill(). + * + * @note All pointer parameters may be NULL, in which case they are ignored. + * @param ncid File ID. + * @param varid Variable ID. + * @param no_fill No_fill setting. + * @param fill_value Pointer to fill value. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 + * netcdf-4 file. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EPERM File is read only. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett + */ +int +NC4_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) +{ + return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, + NULL, &no_fill, fill_value, NULL); +} + +/** + * @internal This functions sets endianness for a netCDF-4 + * variable. Called by nc_def_var_endian(). + * + * @note All pointer parameters may be NULL, in which case they are ignored. + * @param ncid File ID. + * @param varid Variable ID. + * @param endianness Endianness setting. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 + * netcdf-4 file. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_ENOTINDEFINE Not in define mode. + * @returns ::NC_EPERM File is read only. + * @returns ::NC_EINVAL Invalid input + * @author Ed Hartnett + */ +int +NC4_def_var_endian(int ncid, int varid, int endianness) +{ + return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, &endianness); +} + +/** + * @internal Define filter settings. Called by nc_def_var_filter(). + * + * @param ncid File ID. + * @param varid Variable ID. + * @param id Filter ID + * @param nparams Number of parameters for filter. + * @param parms Filter parameters. + * + * @returns ::NC_NOERR for success + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is + * not netCDF-4/HDF5. + * @returns ::NC_ELATEDEF Too late to change settings for this variable. + * @returns ::NC_EFILTER Filter error. + * @returns ::NC_EINVAL Invalid input + * @author Dennis Heimbigner + */ +int +NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, + const unsigned int* parms) +{ + int retval = NC_NOERR; + NC *nc; + NC_GRP_INFO_T *grp; + NC_FILE_INFO_T *h5; + NC_VAR_INFO_T *var; + + LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); + + /* Find info for this file and group, and set pointer to each. */ + if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return retval; + + assert(nc && grp && h5); + + /* Find the var. */ + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + if(!var) + return NC_ENOTVAR; + assert(var->hdr.id == varid); + + /* Can't turn on parallel and filters */ + if (nc->mode & (NC_MPIIO | NC_MPIPOSIX)) { + return NC_EINVAL; + } + + /* If the HDF5 dataset has already been created, then it is too + * late to set all the extra stuff. */ + if (var->created) + return NC_ELATEDEF; + +#ifdef HAVE_H5Z_SZIP + if(id == H5Z_FILTER_SZIP) { + if(nparams != 2) + return NC_EFILTER; /* incorrect no. of parameters */ + } +#else /*!HAVE_H5Z_SZIP*/ + if(id == H5Z_FILTER_SZIP) + return NC_EFILTER; /* Not allowed */ +#endif + +#if 0 + { + unsigned int fcfg = 0; + herr_t herr = H5Zget_filter_info(id,&fcfg); + if(herr < 0) + return NC_EFILTER; + if((H5Z_FILTER_CONFIG_ENCODE_ENABLED & fcfg) == 0 + || (H5Z_FILTER_CONFIG_DECODE_ENABLED & fcfg) == 0) + return NC_EFILTER; + } +#endif /*0*/ + + var->filterid = id; + var->nparams = nparams; + var->params = NULL; + if(parms != NULL) { + var->params = (unsigned int*)calloc(nparams,sizeof(unsigned int)); + if(var->params == NULL) return NC_ENOMEM; + memcpy(var->params,parms,sizeof(unsigned int)*var->nparams); + } + return NC_NOERR; +} + +/** + * @internal Rename a var to "bubba," for example. This is called by + * nc_rename_var() for netCDF-4 files. + * + * @param ncid File ID. + * @param varid Variable ID + * @param name New name of the variable. + * + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADID Bad ncid. + * @returns ::NC_ENOTVAR Invalid variable ID. + * @returns ::NC_EBADNAME Bad name. + * @returns ::NC_EMAXNAME Name is too long. + * @returns ::NC_ENAMEINUSE Name in use. + * @returns ::NC_ENOMEM Out of memory. + * @author Ed Hartnett +*/ +int +NC4_rename_var(int ncid, int varid, const char *name) +{ + NC *nc; + NC_GRP_INFO_T *grp; + NC_FILE_INFO_T *h5; + NC_VAR_INFO_T *var, *tmpvar; + int retval = NC_NOERR; + + if (!name) + return NC_EINVAL; + + LOG((2, "%s: ncid 0x%x varid %d name %s", __func__, ncid, varid, + name)); + + /* Find info for this file and group, and set pointer to each. */ + if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) + return retval; + assert(h5 && grp && h5); + + /* Is the new name too long? */ + if (strlen(name) > NC_MAX_NAME) + return NC_EMAXNAME; + + /* Trying to write to a read-only file? No way, Jose! */ + if (h5->no_write) + return NC_EPERM; + + /* Check name validity, if strict nc3 rules are in effect for this + * file. */ + if ((retval = NC_check_name(name))) + return retval; + + /* Get the variable wrt varid */ + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + if (!var) + return NC_ENOTVAR; + + /* Check if new name is in use; note that renaming to same name is still an error + according to the nc_test/test_write.c code. Why?*/ + tmpvar = (NC_VAR_INFO_T*)ncindexlookup(grp->vars,name); + if(tmpvar != NULL) + return NC_ENAMEINUSE; + + /* If we're not in define mode, new name must be of equal or + less size, if strict nc3 rules are in effect for this . */ + if (!(h5->flags & NC_INDEF) && strlen(name) > strlen(var->hdr.name) && + (h5->cmode & NC_CLASSIC_MODEL)) + return NC_ENOTINDEFINE; + + /* Change the HDF5 file, if this var has already been created + there. Should we check here to ensure there is not already a + dimscale dataset of name name??? */ + if (var->created) + { + /* Is there an existing dimscale-only dataset of this name? If + * so, it must be deleted. */ + if (var->ndims && var->dim[0]->hdf_dimscaleid) + { + if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, var->dim[0]))) + return retval; + } + + LOG((3, "Moving dataset %s to %s", var->hdr.name, name)); + if (H5Gmove(grp->hdf_grpid, var->hdr.name, name) < 0) + BAIL(NC_EHDFERR); + } + + /* Now change the name in our metadata. */ + free(var->hdr.name); + if (!(var->hdr.name = strdup(name))) + return NC_ENOMEM; + LOG((3, "var is now %s", var->hdr.name)); + var->hdr.hashkey = NC_hashmapkey(var->hdr.name,strlen(var->hdr.name)); /* Fix hash key */ + + if(!ncindexrebuild(grp->vars)) + return NC_EINTERNAL; + + /* Check if this was a coordinate variable previously, but names are different now */ + if (var->dimscale && strcmp(var->hdr.name, var->dim[0]->hdr.name)) + { + /* Break up the coordinate variable */ + if ((retval = nc4_break_coord_var(grp, var, var->dim[0]))) + return retval; + } + + /* Check if this should become a coordinate variable */ + if (!var->dimscale) + { + /* Only variables with >0 dimensions can become coordinate variables */ + if (var->ndims) + { + NC_GRP_INFO_T *dim_grp; + NC_DIM_INFO_T *dim; + + /* Check to see if this is became a coordinate variable. If so, it + * will have the same name as dimension index 0. If it is a + * coordinate var, is it a coordinate var in the same group as the dim? + */ + if ((retval = nc4_find_dim(grp, var->dimids[0], &dim, &dim_grp))) + return retval; + if (!strcmp(dim->hdr.name, name) && dim_grp == grp) + { + /* Reform the coordinate variable */ + if ((retval = nc4_reform_coord_var(grp, var, dim))) + return retval; + var->became_coord_var = NC_TRUE; + } + } + } + +exit: + return retval; +} + +/** + * @internal Write an array of data to a variable. This is called by + * nc_put_vara() and other nc_put_vara_* functions, for netCDF-4 + * files. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param startp Array of start indices. + * @param countp Array of counts. + * @param op pointer that gets the data. + * @param memtype The type of these data in memory. + * + * @returns ::NC_NOERR for success + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_put_vara(int ncid, int varid, const size_t *startp, + const size_t *countp, const void *op, int memtype) +{ + NC *nc; + + if (!(nc = nc4_find_nc_file(ncid, NULL))) + return NC_EBADID; + + return nc4_put_vars(nc, ncid, varid, startp, countp, NULL, memtype, + (void *)op); +} + +/** + * Read an array of values. This is called by nc_get_vara() for + * netCDF-4 files, as well as all the other nc_get_vara_* + * functions. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param startp Array of start indices. + * @param countp Array of counts. + * @param ip pointer that gets the data. + * @param memtype The type of these data after it is read into memory. + + * @returns ::NC_NOERR for success + * @author Ed Hartnett, Dennis Heimbigner + */ +int +NC4_get_vara(int ncid, int varid, const size_t *startp, + const size_t *countp, void *ip, int memtype) +{ + NC *nc; + NC_FILE_INFO_T* h5; + + LOG((2, "%s: ncid 0x%x varid %d memtype %d", __func__, ncid, varid, + memtype)); + + if (!(nc = nc4_find_nc_file(ncid, &h5))) + return NC_EBADID; + + /* Get the data. */ + return nc4_get_vars(nc, ncid, varid, startp, countp, NULL, memtype, + (void *)ip); +} + +/** + * @internal Write an array of data to a variable. This is called by + * nc_put_vars() and other nc_put_vars_* functions, for netCDF-4 + * files. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param startp Array of start indices. + * @param countp Array of counts. + * @param stridep Array of strides. + * @param op pointer that gets the data. + * @param memtype The type of these data in memory. + * + * @returns ::NC_NOERR for success + * @author Dennis Heimbigner + */ +int +NC4_put_vars(int ncid, int varid, const size_t *startp, + const size_t *countp, const ptrdiff_t* stridep, + const void *op, int memtype) +{ + NC *nc; + + if (!(nc = nc4_find_nc_file(ncid, NULL))) + return NC_EBADID; + + return nc4_put_vars(nc, ncid, varid, startp, countp, stridep, memtype, + (void *)op); +} + +/** + * Read an array of values. This is called by nc_get_vars() for + * netCDF-4 files, as well as all the other nc_get_vars_* + * functions. + * + * @param ncid File ID. + * @param varid Variable ID. + * @param startp Array of start indices. + * @param countp Array of counts. + * @param stridep Array of strides. + * @param ip pointer that gets the data. + * @param memtype The type of these data after it is read into memory. + + * @returns ::NC_NOERR for success + * @author Dennis Heimbigner + */ +int +NC4_get_vars(int ncid, int varid, const size_t *startp, + const size_t *countp, const ptrdiff_t *stridep, + void *ip, int memtype) +{ + NC *nc; + NC_FILE_INFO_T *h5; + + LOG((2, "%s: ncid 0x%x varid %d memtype %d", __func__, ncid, varid, + memtype)); + + if (!(nc = nc4_find_nc_file(ncid, &h5))) + return NC_EBADID; + + /* Get the data. */ + return nc4_get_vars(nc, ncid, varid, startp, countp, stridep, memtype, + (void *)ip); +} diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 3c220a60a..dc3d8f984 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -15,6 +15,7 @@ #include "config.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include #include diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index f754aa2d5..b2286bc0c 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -12,6 +12,7 @@ #include #include "netcdf.h" #include "nc4internal.h" +#include "hdf5internal.h" #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */ diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index 6a2090141..fe4fd03e8 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -18,6 +18,7 @@ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "ncdispatch.h" diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index 30d1d08cb..25617394d 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -10,23 +10,12 @@ * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher */ +#include "config.h" #include #include "nc4dispatch.h" +#include "hdf5internal.h" #include -/* Szip options. */ -#define NC_SZIP_EC_OPTION_MASK 4 /**< @internal SZIP EC option mask. */ -#define NC_SZIP_NN_OPTION_MASK 32 /**< @internal SZIP NN option mask. */ -#define NC_SZIP_MAX_PIXELS_PER_BLOCK 32 /**< @internal SZIP max pixels per block. */ - -/** @internal Default size for unlimited dim chunksize */ -#define DEFAULT_1D_UNLIM_SIZE (4096) - -#define NC_ARRAY_GROWBY 4 /**< @internal Amount to grow array. */ - -extern int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, - void *fill_value); - /** * @internal Set chunk cache size for a variable. This is the internal * function called by nc_set_var_chunk_cache(). @@ -201,165 +190,6 @@ nc_get_var_chunk_cache_ints(int ncid, int varid, int *sizep, return NC_NOERR; } -/** - * @internal Check a set of chunksizes to see if they specify a chunk - * that is too big. - * - * @param grp Pointer to the group info. - * @param var Pointer to the var info. - * @param chunksizes Array of chunksizes to check. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_EBADCHUNK Bad chunksize. - */ -static int -check_chunksizes(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, const size_t *chunksizes) -{ - double dprod; - size_t type_len; - int d; - int retval; - - if ((retval = nc4_get_typelen_mem(grp->nc4_info, var->type_info->hdr.id, &type_len))) - return retval; - if (var->type_info->nc_type_class == NC_VLEN) - dprod = (double)sizeof(hvl_t); - else - dprod = (double)type_len; - for (d = 0; d < var->ndims; d++) - dprod *= (double)chunksizes[d]; - - if (dprod > (double) NC_MAX_UINT) - return NC_EBADCHUNK; - - return NC_NOERR; -} - -/** - * @internal Determine some default chunksizes for a variable. - * - * @param grp Pointer to the group info. - * @param var Pointer to the var info. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @author Ed Hartnett - */ -static int -nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) -{ - int d; - size_t type_size; - float num_values = 1, num_unlim = 0; - int retval; - size_t suggested_size; -#ifdef LOGGING - double total_chunk_size; -#endif - - if (var->type_info->nc_type_class == NC_STRING) - type_size = sizeof(char *); - else - type_size = var->type_info->size; - -#ifdef LOGGING - /* Later this will become the total number of bytes in the default - * chunk. */ - total_chunk_size = (double) type_size; -#endif - - /* How many values in the variable (or one record, if there are - * unlimited dimensions). */ - for (d = 0; d < var->ndims; d++) - { - assert(var->dim[d]); - if (! var->dim[d]->unlimited) - num_values *= (float)var->dim[d]->len; - else { - num_unlim++; - var->chunksizes[d] = 1; /* overwritten below, if all dims are unlimited */ - } - } - /* Special case to avoid 1D vars with unlim dim taking huge amount - of space (DEFAULT_CHUNK_SIZE bytes). Instead we limit to about - 4KB */ - if (var->ndims == 1 && num_unlim == 1) { - if (DEFAULT_CHUNK_SIZE / type_size <= 0) - suggested_size = 1; - else if (DEFAULT_CHUNK_SIZE / type_size > DEFAULT_1D_UNLIM_SIZE) - suggested_size = DEFAULT_1D_UNLIM_SIZE; - else - suggested_size = DEFAULT_CHUNK_SIZE / type_size; - var->chunksizes[0] = suggested_size / type_size; - LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " - "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[0])); - } - if (var->ndims > 1 && var->ndims == num_unlim) { /* all dims unlimited */ - suggested_size = pow((double)DEFAULT_CHUNK_SIZE/type_size, 1.0/(double)(var->ndims)); - for (d = 0; d < var->ndims; d++) - { - var->chunksizes[d] = suggested_size ? suggested_size : 1; - LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " - "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d])); - } - } - - /* Pick a chunk length for each dimension, if one has not already - * been picked above. */ - for (d = 0; d < var->ndims; d++) - if (!var->chunksizes[d]) - { - suggested_size = (pow((double)DEFAULT_CHUNK_SIZE/(num_values * type_size), - 1.0/(double)(var->ndims - num_unlim)) * var->dim[d]->len - .5); - if (suggested_size > var->dim[d]->len) - suggested_size = var->dim[d]->len; - var->chunksizes[d] = suggested_size ? suggested_size : 1; - LOG((4, "%s: name %s dim %d DEFAULT_CHUNK_SIZE %d num_values %f type_size %d " - "chunksize %ld", __func__, var->hdr.name, d, DEFAULT_CHUNK_SIZE, num_values, type_size, var->chunksizes[d])); - } - -#ifdef LOGGING - /* Find total chunk size. */ - for (d = 0; d < var->ndims; d++) - total_chunk_size *= (double) var->chunksizes[d]; - LOG((4, "total_chunk_size %f", total_chunk_size)); -#endif - - /* But did this result in a chunk that is too big? */ - retval = check_chunksizes(grp, var, var->chunksizes); - if (retval) - { - /* Other error? */ - if (retval != NC_EBADCHUNK) - return retval; - - /* Chunk is too big! Reduce each dimension by half and try again. */ - for ( ; retval == NC_EBADCHUNK; retval = check_chunksizes(grp, var, var->chunksizes)) - for (d = 0; d < var->ndims; d++) - var->chunksizes[d] = var->chunksizes[d]/2 ? var->chunksizes[d]/2 : 1; - } - - /* Do we have any big data overhangs? They can be dangerous to - * babies, the elderly, or confused campers who have had too much - * beer. */ - for (d = 0; d < var->ndims; d++) - { - size_t num_chunks; - size_t overhang; - assert(var->chunksizes[d] > 0); - num_chunks = (var->dim[d]->len + var->chunksizes[d] - 1) / var->chunksizes[d]; - if(num_chunks > 0) { - overhang = (num_chunks * var->chunksizes[d]) - var->dim[d]->len; - var->chunksizes[d] -= overhang / num_chunks; - } - } - - return NC_NOERR; -} - /** * @internal Get all the information about a variable. Pass NULL for * whatever you don't care about. This is the internal function called @@ -605,62 +435,6 @@ nc_inq_var_chunking_ints(int ncid, int varid, int *contiguousp, int *chunksizesp return retval; } -/** - * @internal Define chunking stuff for a var. This is called by - * the fortran API. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param contiguous Pointer to contiguous setting. - * @param chunksizesp Array of chunksizes. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EINVAL Invalid input - * @returns ::NC_EBADCHUNK Bad chunksize. - * @author Ed Hartnett - */ -int -nc_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp) -{ - NC *nc; - NC_GRP_INFO_T *grp; - NC_VAR_INFO_T *var; - NC_FILE_INFO_T *h5; - size_t *cs = NULL; - int i, retval; - - /* Find this ncid's file info. */ - if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; - assert(nc); - - /* Find var cause I need the number of dims. */ - if ((retval = nc4_find_g_var_nc(nc, ncid, varid, &grp, &var))) - return retval; - - /* Allocate space for the size_t copy of the chunksizes array. */ - if (var->ndims) - if (!(cs = malloc(var->ndims * sizeof(size_t)))) - return NC_ENOMEM; - - /* Copy to size_t array. */ - for (i = 0; i < var->ndims; i++) - cs[i] = chunksizesp[i]; - - retval = nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, - &contiguous, cs, NULL, NULL, NULL); - - if (var->ndims) - free(cs); - return retval; -} - /** * @internal Find the ID of a variable, from the name. This function * is called by nc_inq_varid(). diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index 9f979be8e..c91fd4458 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -605,9 +605,8 @@ main(int argc, char **argv) NC_memio filedata3; NC_memio filedata4; - nc_set_log_level(0); - #ifdef USE_NETCDF4 + nc_set_log_level(0); H5Eprint1(stderr); #endif diff --git a/nc_test4/tst_atts.c b/nc_test4/tst_atts.c index 4090dc2e8..44e94808a 100644 --- a/nc_test4/tst_atts.c +++ b/nc_test4/tst_atts.c @@ -10,7 +10,7 @@ #include #include #include "err_macros.h" -#include "nc4internal.h" +#include "hdf5internal.h" /* The data file we will create. */ #define FILE_NAME "tst_atts.nc" diff --git a/ncdump/tst_dimsizes.c b/ncdump/tst_dimsizes.c index edb0eb80d..eee6c340e 100644 --- a/ncdump/tst_dimsizes.c +++ b/ncdump/tst_dimsizes.c @@ -34,7 +34,6 @@ main(int argc, char **argv) size_t dimsize; int dimid; int stat = NC_NOERR; - nc_set_log_level(5); printf("\n*** Testing Max Dimension Sizes\n"); printf("\n|size_t|=%lu\n",(unsigned long)sizeof(size_t));