mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-02-11 16:40:36 +08:00
Merge pull request #2193 from WardF/Dave-Allured-format-compat-on-reopen
Dave allured format compat on reopen
This commit is contained in:
Ward Fisher
GitHub
commit
850c4e816c
4
.github/workflows/run_tests_osx.yml
vendored
4
.github/workflows/run_tests_osx.yml
vendored
@ -1,11 +1,11 @@
|
||||
name: Run netCDF Tests on OSX
|
||||
|
||||
###
|
||||
# Build hdf4, hdf5 dependencies and cache them in a combined directory.
|
||||
# See https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
|
||||
# for information related to github runners.
|
||||
###
|
||||
|
||||
name: Run macOS-based netCDF Tests
|
||||
|
||||
|
||||
on: [ pull_request ]
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* Copyright 2018-2018 University Corporation for Atmospheric
|
||||
/* Copyright 2018-2022 University Corporation for Atmospheric
|
||||
Research/Unidata. */
|
||||
/**
|
||||
* @file This header file contains macros, types, and prototypes for
|
||||
@ -216,4 +216,6 @@ extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
|
||||
EXTERNL hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
|
||||
EXTERNL hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);
|
||||
|
||||
int hdf5set_format_compatibility(hid_t fapl_id);
|
||||
|
||||
#endif /* _HDF5INTERNAL_ */
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
## This is a CMake file, part of Unidata's netCDF package.
|
||||
# Copyright 2018, see the COPYRIGHT file for more information.
|
||||
# Copyright 2018-2022, see the COPYRIGHT file for more information.
|
||||
#
|
||||
# This builds the HDF5 dispatch layer.
|
||||
#
|
||||
@ -9,13 +9,13 @@
|
||||
SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c
|
||||
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c
|
||||
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c
|
||||
hdf5debug.c)
|
||||
hdf5set_format_compatibility.c hdf5debug.c)
|
||||
|
||||
IF(ENABLE_BYTERANGE)
|
||||
SET(libnchdf5_SOURCES ${libnchdf5_SOURCES} H5FDhttp.c)
|
||||
ENDIF()
|
||||
|
||||
# Build the HDF4 dispatch layer as a library that will be included in
|
||||
# Build the HDF5 dispatch layer as a library that will be included in
|
||||
# the netCDF library.
|
||||
add_library(netcdfhdf5 OBJECT ${libnchdf5_SOURCES})
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# This is part of Unidata's netCDF package. Copyright 2018, see the
|
||||
# COPYRIGHT file for more information.
|
||||
# This is part of Unidata's netCDF package. Copyright 2018-2022,
|
||||
# see the COPYRIGHT file for more information.
|
||||
|
||||
# Build the HDF5 dispatch layer.
|
||||
|
||||
@ -16,7 +16,7 @@ noinst_LTLIBRARIES = libnchdf5.la
|
||||
libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
|
||||
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \
|
||||
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c \
|
||||
hdf5debug.c hdf5debug.h hdf5err.h
|
||||
hdf5set_format_compatibility.c hdf5debug.c hdf5debug.h hdf5err.h
|
||||
|
||||
if ENABLE_BYTERANGE
|
||||
libnchdf5_la_SOURCES += H5FDhttp.c H5FDhttp.h
|
||||
@ -24,8 +24,3 @@ endif
|
||||
|
||||
# Package this for cmake build.
|
||||
EXTRA_DIST = CMakeLists.txt
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* Copyright 2003-2018, University Corporation for Atmospheric
|
||||
/* Copyright 2003-2022, University Corporation for Atmospheric
|
||||
* Research. See COPYRIGHT file for copying and redistribution
|
||||
* conditions. */
|
||||
/**
|
||||
@ -117,9 +117,10 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
|
||||
}
|
||||
}
|
||||
|
||||
/* Need this access plist to control how HDF5 handles open objects
|
||||
* on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to
|
||||
* fail if there are any open objects in the file. This may happen when virtual
|
||||
/* Need this FILE ACCESS plist to control how HDF5 handles open
|
||||
* objects on file close; as well as for other controls below.
|
||||
* (Setting H5F_CLOSE_WEAK will cause H5Fclose not to fail if there
|
||||
* are any open objects in the file. This may happen when virtual
|
||||
* datasets are opened). */
|
||||
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
@ -127,8 +128,8 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
#ifdef USE_PARALLEL4
|
||||
/* If this is a parallel file create, set up the file creation
|
||||
property list. */
|
||||
/* If this is a parallel file create, set up the file access
|
||||
property list for MPI/IO. */
|
||||
if (mpiinfo != NULL) {
|
||||
nc4_info->parallel = NC_TRUE;
|
||||
LOG((4, "creating parallel file with MPI/IO"));
|
||||
@ -164,21 +165,14 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
|
||||
nc4_chunk_cache_preemption));
|
||||
}
|
||||
|
||||
#if H5_VERSION_GE(1,10,2)
|
||||
/* lib versions 1.10.2 and higher */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0)
|
||||
#else
|
||||
#if H5_VERSION_GE(1,10,0)
|
||||
/* lib versions 1.10.0, 1.10.1 */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0)
|
||||
#else
|
||||
/* all HDF5 1.8 lib versions */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
|
||||
#endif
|
||||
#endif
|
||||
BAIL(NC_EHDFERR);
|
||||
/* Set HDF5 format compatibility in the FILE ACCESS property list.
|
||||
* Compatibility is transient and must be reselected every time
|
||||
* a file is opened for writing. */
|
||||
retval = hdf5set_format_compatibility(fapl_id);
|
||||
if (retval != NC_NOERR)
|
||||
BAIL(retval);
|
||||
|
||||
/* Create the property list. */
|
||||
/* Begin setup for the FILE CREATION property list. */
|
||||
if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
@ -186,9 +180,8 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
|
||||
if (H5Pset_obj_track_times(fcpl_id,0)<0)
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
/* Set latest_format in access propertly list and
|
||||
* H5P_CRT_ORDER_TRACKED in the creation property list. This turns
|
||||
* on HDF5 creation ordering. */
|
||||
/* Set H5P_CRT_ORDER_TRACKED in the creation property list.
|
||||
* This turns on HDF5 creation ordering. */
|
||||
if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
|
||||
H5P_CRT_ORDER_INDEXED)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
/* Copyright 2003-2018, University Corporation for Atmospheric
|
||||
/* Copyright 2003-2022, University Corporation for Atmospheric
|
||||
* Research. See COPYRIGHT file for copying and redistribution
|
||||
* conditions. */
|
||||
/**
|
||||
@ -765,9 +765,10 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
|
||||
mpiinfo = (NC_MPI_INFO *)parameters; /* assume, may be changed if inmemory is true */
|
||||
#endif /* !USE_PARALLEL4 */
|
||||
|
||||
/* Need this access plist to control how HDF5 handles open objects
|
||||
* on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to
|
||||
* fail if there are any open objects in the file. This may happen when virtual
|
||||
/* Need this FILE ACCESS plist to control how HDF5 handles open
|
||||
* objects on file close; as well as for other controls below.
|
||||
* (Setting H5F_CLOSE_WEAK will cause H5Fclose not to fail if there
|
||||
* are any open objects in the file. This may happen when virtual
|
||||
* datasets are opened). */
|
||||
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
|
||||
BAIL(NC_EHDFERR);
|
||||
@ -820,6 +821,13 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
|
||||
nc4_chunk_cache_preemption));
|
||||
}
|
||||
|
||||
/* Set HDF5 format compatibility in the FILE ACCESS property list.
|
||||
* Compatibility is transient and must be reselected every time
|
||||
* a file is opened for writing. */
|
||||
retval = hdf5set_format_compatibility(fapl_id);
|
||||
if (retval != NC_NOERR)
|
||||
BAIL(retval);
|
||||
|
||||
/* Process NC_INMEMORY */
|
||||
if(nc4_info->mem.inmemory) {
|
||||
NC_memio* memio;
|
||||
|
||||
92
libhdf5/hdf5set_format_compatibility.c
Normal file
92
libhdf5/hdf5set_format_compatibility.c
Normal file
@ -0,0 +1,92 @@
|
||||
/* Copyright 2022, University Corporation for Atmospheric Research.
|
||||
* See COPYRIGHT file for copying and redistribution conditions. */
|
||||
/**
|
||||
* @file
|
||||
* @internal This function selects the best HDF5 file format options
|
||||
* to create netCDF-4 files that can be read and written by older
|
||||
* library versions.
|
||||
*
|
||||
* Format compatibility is transient, not baked in to an HDF5 file
|
||||
* at creation time. Therefore the desired compatibilty options
|
||||
* must be selected every time a file is opened for writing.
|
||||
*
|
||||
* This function should be called before every call to create a new
|
||||
* netCDF-4 file, or to open an existing netCDF-4 file for writing.
|
||||
* This function has no effect when opening a file for read only.
|
||||
*
|
||||
* This function should work correctly with all HDF5 library versions
|
||||
* from 1.8.0 through 1.13.0 and beyond, with no further changes.
|
||||
* This assumes that HDF5 versioning controls remain consistent
|
||||
* into the future.
|
||||
*
|
||||
* The basic functionality is to select the traditional HDF5 v1.8
|
||||
* format compatibility, whenever possible. The less desirable
|
||||
* v1.6 compatibily is selected in a few strange cases when it is
|
||||
* not possible to select v1.8.
|
||||
*
|
||||
* Files created or updated with v1.10 and higher compatibility are
|
||||
* not legal netCDF-4 format, as of 2022 January. They are not
|
||||
* readable by any netCDF library version linked with any HDF5 v1.8
|
||||
* or older library version. However, it is possible for advanced
|
||||
* or experimental software to deliberately override these default
|
||||
* format settings, to create advanced format files for special
|
||||
* purposes.
|
||||
*
|
||||
* Files created with v1.6 compatibility have superblock version 0.
|
||||
* Files created with v1.8 compatibility have superblock version 2.
|
||||
* Files created with v1.10 compatibility have superblock version 3,
|
||||
* and are avoided by default. Et cetera.
|
||||
*
|
||||
* The superblock version is locked in when a file is first created.
|
||||
* It is then possible to get a mix of v1.6 and v1.8 internal
|
||||
* object versions, when an existing netCDF-4 file is modified by
|
||||
* a different software version than the one that originally
|
||||
* created the file. Mixed-object files of this nature are common
|
||||
* and do not suffer any serious problems.
|
||||
*
|
||||
* See netcdf-c github issues #250 and #951 for more details about
|
||||
* the rationale and evolution of netCDF-4 format compatibility.
|
||||
*
|
||||
* See HDF5 documentation for H5Pset_libver_bounds and related RFC's,
|
||||
* for more details about HDF5 file object versioning.
|
||||
*
|
||||
* @author Dave Allured, NOAA/PSL/CIRES @date 2022 January 11
|
||||
*/
|
||||
|
||||
#include "config.h"
|
||||
#include "hdf5internal.h"
|
||||
|
||||
/**
|
||||
* @internal Function to set HDF5 file access options for backward
|
||||
* format compatibility. Call this before every call to H5Fcreate
|
||||
* or H5Fopen.
|
||||
*
|
||||
* @param fapl_id Identifier for valid file access property list to
|
||||
* be used in the next call to H5Fcreate or H5Fopen.
|
||||
*
|
||||
* @return ::NC_NOERR No error.
|
||||
* @return ::NC_EHDFERR HDF5 returned error.
|
||||
*
|
||||
* @author Dave Allured, NOAA/PSL/CIRES @date 2022 January 11
|
||||
*/
|
||||
int
|
||||
hdf5set_format_compatibility(hid_t fapl_id)
|
||||
{
|
||||
#if H5_VERSION_GE(1,10,2)
|
||||
/* lib versions 1.10.2 and higher */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0)
|
||||
|
||||
#else
|
||||
#if H5_VERSION_GE(1,10,0)
|
||||
/* lib versions 1.10.0, 1.10.1 */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0)
|
||||
|
||||
#else
|
||||
/* all HDF5 1.8 lib versions */
|
||||
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
|
||||
#endif
|
||||
#endif
|
||||
return NC_EHDFERR; /* failure exit */
|
||||
|
||||
return NC_NOERR; /* normal exit */
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user