mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-13 17:18:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1361 from Unidata/simpleprov.dmh

Browse Source 
Simplify libhdf5/nc5info.c to move to avoid parsing
This commit is contained in:
Ward Fisher 2019-04-19 15:35:01 -06:00 committed by GitHub
commit 038bebfead
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 609 additions and 581 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
include/hdf5internal.h
View File

@ -150,36 +150,6 @@ extern int nc4_give_var_secret_name(NC_VAR_INFO_T *var);
/* Get the fill value for a var. */
int nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp);
/* Provenance Management (moved from nc4internal.h) */
/* Initialize the fileinfo global state */
extern int NC4_provenance_init();
/* Finalize the fileinfo global state */
extern int NC4_provenance_finalize();
/* Extract the provenance from a file, using dfalt as default */
extern int NC4_get_provenance(NC_FILE_INFO_T* file, const char* propstring, const struct NCPROPINFO* dfalt);
/* Set the provenance for a created file using dfalt as default */
extern int NC4_set_provenance(NC_FILE_INFO_T* file, const struct NCPROPINFO* dfalt);
/* Recover memory of an NCPROVENANCE object */
extern int NC4_free_provenance(struct NCPROVENANCE* prov);
extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/
extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
/* Convert a NCPROPINFO instance to a single string. */
extern int NC4_buildpropinfo(struct NCPROPINFO* info, char** propdatap);
/* Use HDF5 API to read the _NCProperties attribute */
extern int NC4_read_ncproperties(NC_FILE_INFO_T*);
/* Use HDF5 API to write the _NCProperties attribute */
extern int NC4_write_ncproperties(NC_FILE_INFO_T*);
/* Find file, group, var, and att info, doing lazy reads if needed. */
int nc4_hdf5_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
int use_name, char *norm_name, NC_FILE_INFO_T **h5,
@ -193,4 +163,9 @@ int nc4_hdf5_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
/* Perform lazy read of the rest of the metadata for a var. */
int nc4_get_var_meta(NC_VAR_INFO_T *var);
/* Support functions for provenance info (defined in nc4hdf.c) */
extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/
extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/
extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/
#endif /* _HDF5INTERNAL_ */

54
include/nc4internal.h
View File

@ -86,7 +86,6 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT;
typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
/*Forward*/
struct NCPROVENANCE;
struct NC_GRP_INFO;
struct NC_TYPE_INFO;
@ -275,7 +274,7 @@ typedef struct NC_FILE_INFO
NClist* alltypes;
NClist* allgroups; /* including root group */
void *format_file_info;
struct NCPROVENANCE* provenance;
NC4_Provenance provenance;
struct NC4_Memio {
NC_memio memio; /* What we sent to image_init and what comes back*/
int locked; /* do not copy and do not free */
@ -423,55 +422,4 @@ extern const NC_reservedatt* NC_findreserved(const char* name);
#define NC_ATT_COORDINATES COORDINATES /*defined above*/
#define NC_ATT_FORMAT "_Format"
/**************************************************/
/**
For netcdf4 files, capture state information about the following:
1. Global: netcdf library version
2. Global: hdf5 library version
3. Per file: superblock version
4. Per File: was it created by netcdf-4?
5. Per file: _NCProperties attribute
*/
/* Most of this needs to be moved to hdf5internal.h */
#define NCPROPS "_NCProperties"
#define NCPVERSION "version" /* Of the properties format */
#define NCPHDF5LIB1 "hdf5libversion"
#define NCPNCLIB1 "netcdflibversion"
#define NCPHDF5LIB2 "hdf5"
#define NCPNCLIB2 "netcdf"
#define NCPROPS_VERSION (2)
/* Version 2 changes this because '|' was causing bash problems */
#define NCPROPSSEP1 '|'
#define NCPROPSSEP2 ','
/* Other hidden attributes */
#define ISNETCDF4ATT "_IsNetcdf4"
#define SUPERBLOCKATT "_SuperblockVersion"
struct NCPROVENANCE {
int superblockversion;
struct NCPROPINFO {
int version; /* 0 => not defined */
/* Following is filled from NCPROPS attribute or from global version */
/* Version 1 format is:
"netcdflibversion=<version|hdf5libversion=<version>"
Version 2 format is:
"<mainbuildlib>=<version|<supportlib1>=<version>...|<other>=..."
*/
/* The _NCProperties values are stored as an arbitrary
set of (key,value) pairs */
/* It is assumed that the first entry is the primary library
used to build the file, and it is followed by other libraries
used in the build, and finally an arbitrary list of other
(key,value) pairs. */
NClist* properties;
} propattr;
};
/* Provenance Initialization */
extern struct NCPROPINFO globalpropinfo;
#endif /* _NC4INTERNAL_ */

50
include/nc_provenance.h
View File

@ -12,11 +12,12 @@
*
*
* For netcdf4 files, capture state information about the following:
* 1. Global: netcdf library version
* 2. Global: hdf5 library version
* 3. Per file: superblock version
* 4. Per File: was it created by netcdf-4?
* 5. Per file: _NCProperties attribute
* - Global: netcdf library version
* - Global: hdf5 library version
* - Global: any parameters specified by the --with-ncproperties option for ./configure
* - Per file: superblock version
* - Per File: was it created by netcdf-4?
* - Per file: _NCProperties attribute
*
* @author Dennis Heimbigner, Ward Fisher
*/
@ -39,4 +40,43 @@
#define ISNETCDF4ATT "_IsNetcdf4"
#define SUPERBLOCKATT "_SuperblockVersion"
/* Forward */
struct NC_FILE_INFO;
/**************************************************/
/**
For netcdf4 files, capture state information about the following:
1. Global: netcdf library version
2. Global: hdf5 library version
3. Per file: superblock version
4. Per File: was it created by netcdf-4?
5. Per file: _NCProperties attribute
*/
typedef struct NC4_Provenance {
char* ncproperties; /* raw value of _NCProperties; NULL => does not exist */
int version;
int superblockversion;
} NC4_Provenance;
/* Provenance Management (moved from nc4internal.h) */
/* Initialize the provenance global state */
extern int NC4_provenance_init(void);
/* Finalize the provenance global state */
extern int NC4_provenance_finalize(void);
/* Read and store the provenance from an existing file */
extern int NC4_read_provenance(struct NC_FILE_INFO* file);
/* Write the provenance into a newly created file */
extern int NC4_write_provenance(struct NC_FILE_INFO* file);
/* Create the provenance for a newly created file */
extern int NC4_new_provenance(struct NC_FILE_INFO* file);
/* Clean up the provenance info in a file */
extern int NC4_clear_provenance(NC4_Provenance* prov);
#endif /* _NCPROVENANCE_ */

2
libhdf5/Makefile.am
View File

@ -7,7 +7,7 @@
# Get AM_CPPFLAGS.
include $(top_srcdir)/lib_flags.am
libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS}
libnchdf5_la_CPPFLAGS = ${AM_CPPFLAGS}
# This is our output. The HDF5 convenience library.
noinst_LTLIBRARIES = libnchdf5.la

13
libhdf5/hdf5attr.c
View File

@ -91,28 +91,23 @@ nc4_get_att_special(NC_FILE_INFO_T* h5, const char* name,
return NC_EATTMETA;
if(strcmp(name,NCPROPS)==0) {
char* propdata = NULL;
int stat = NC_NOERR;
int len;
if(h5->provenance->propattr.version == 0)
if(h5->provenance.ncproperties == NULL)
return NC_ENOTATT;
if(mem_type == NC_NAT) mem_type = NC_CHAR;
if(mem_type != NC_CHAR)
return NC_ECHAR;
if(filetypep) *filetypep = NC_CHAR;
stat = NC4_buildpropinfo(&h5->provenance->propattr, &propdata);
if(stat != NC_NOERR) return stat;
len = strlen(propdata);
len = strlen(h5->provenance.ncproperties);
if(lenp) *lenp = len;
if(data) strncpy((char*)data,propdata,len+1);
free(propdata);
if(data) strncpy((char*)data,h5->provenance.ncproperties,len+1);
} else if(strcmp(name,ISNETCDF4ATT)==0
|| strcmp(name,SUPERBLOCKATT)==0) {
unsigned long long iv = 0;
if(filetypep) *filetypep = NC_INT;
if(lenp) *lenp = 1;
if(strcmp(name,SUPERBLOCKATT)==0)
iv = (unsigned long long)h5->provenance->superblockversion;
iv = (unsigned long long)h5->provenance.superblockversion;
else /* strcmp(name,ISNETCDF4ATT)==0 */
iv = NC4_isnetcdf4(h5);
if(mem_type == NC_NAT) mem_type = NC_INT;

2
libhdf5/hdf5create.c
View File

@ -234,7 +234,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
nc4_info->flags |= NC_INDEF;
/* Save the HDF5 superblock number and set the _NCProperties attribute. */
if ((retval = NC4_set_provenance(nc4_info, &globalpropinfo)))
if ((retval = NC4_new_provenance(nc4_info)))
BAIL(retval);
return NC_NOERR;

8
libhdf5/hdf5file.c
View File

@ -212,8 +212,8 @@ sync_netcdf4_file(NC_FILE_INFO_T *h5)
if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order)))
return retval;
/* Write out _NCProperties */
if((retval = NC4_write_ncproperties(h5)))
/* Write out provenance*/
if((retval = NC4_write_provenance(h5)))
return retval;
}
@ -276,9 +276,7 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio)
/* Free the fileinfo struct, which holds info from the fileinfo
* hidden attribute. */
if (h5->provenance)
NC4_free_provenance(h5->provenance);
h5->provenance = NULL; /* Avoid double dealloc */
NC4_clear_provenance(&h5->provenance);
/* Close hdf file. It may not be open, since this function is also
* called by NC_create() when a file opening is aborted. */

15
libhdf5/hdf5open.c
View File

@ -676,7 +676,7 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
h5->http.iosp = 1;
/* Kill off any conflicting modes flags */
mode &= ~(NC_WRITE|NC_DISKLESS|NC_PERSIST|NC_INMEMORY);
parameters = NULL; /* kill off parallel */
parameters = NULL; /* kill off parallel */
} else
h5->http.iosp = 0;
#endif /*ENABLE_BYTERANGE*/
@ -796,25 +796,24 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
if ((h5->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
BAIL(NC_EHDFERR);
}
/* Now read in all the metadata. Some types and dimscale
* information may be difficult to resolve here, if, for example, a
* dataset of user-defined type is encountered before the
* definition of that type. */
if ((retval = rec_read_metadata(nc4_info->root_grp)))
BAIL(retval);
/* Check for classic model attribute. */
if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic)))
BAIL(retval);
if (is_classic)
nc4_info->cmode |= NC_CLASSIC_MODEL;
/* See if this file contained _NCPROPERTIES, and if yes, process
* it, if no, then fake it. */
if ((retval = NC4_read_ncproperties(nc4_info)))
/* Set the provenance info for this file */
if ((retval = NC4_read_provenance(nc4_info)))
BAIL(retval);
/* Now figure out which netCDF dims are indicated by the dimscale
* information. */
if ((retval = rec_match_dimscales(nc4_info->root_grp)))

988
libhdf5/nc4info.c
View File

File diff suppressed because it is too large Load Diff

1
nc_perf/perftest.sh
View File

@ -9,7 +9,6 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -x
set -e
echo "Testing performance of nc_create and nc_open on file with large metadata"

2
ncdump/Makefile.am
View File

@ -136,7 +136,7 @@ tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \
tst_ncgen4_cycle.sh tst_null_byte_padding.sh \
ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc \
ref_tst_radix.cdl tst_radix.cdl test_radix.sh \
ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh
ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh ref_no_ncproperty.nc
# The L512.bin file is file containing exactly 512 bytes each of value 0.
# It is used for creating hdf5 files with varying offsets for testing.

1
ncdump/ncdump.c
View File

@ -48,6 +48,7 @@ int optind;
#include "nclog.h"
#include "ncwinpath.h"
#include "netcdf_aux.h"
#include "nclist.h"
#include "nc_provenance.h"
#ifdef USE_NETCDF4

BIN
ncdump/ref_no_ncproperty.nc Normal file
View File

Binary file not shown.

19
ncdump/tst_fileinfo.sh
View File

@ -14,6 +14,7 @@ HDF="./hdf5_fileinfo.hdf"
NF="${top_srcdir}/ncdump/ref_tst_compounds4.nc"
NPV1="${top_srcdir}/ncdump/ref_provenance_v1.nc"
NPNCP="${top_srcdir}/ncdump/ref_no_ncproperty.nc"
# Create various files
${execdir}/tst_fileinfo
@ -26,8 +27,18 @@ else
echo "FAIL: False negative for file: $NF"
EXIT=1
fi
rm -f ./tst_fileinfo.tmp
# Verify handling of a file with no _NCProperties attribute
rm -f ./tst_fileinfo.tmp
if $NCDUMP -s $NPNCP | fgrep '_NCProperties=' > ./tst_fileinfo.tmp ; then
echo "Fail: $NPNCP has _NCProperties attribute"
EXIT=1
else
echo "Pass: $NPNCP has no _NCProperties attribute"
fi
rm -f ./tst_fileinfo.tmp
if test -e $NCF ; then
# look at the _IsNetcdf4 flag
N_IS=`${NCDUMP} -s $NCF | fgrep '_IsNetcdf4' | tr -d ' ;'`
@ -46,15 +57,17 @@ else
echo "FAIL: tst_fileinfo: $NCF does not exist"
EXIT=1
fi
echo "PASS: $NCF is marked as netcdf-4"
# Test what happens when we read a file that used provenance version 1
rm -f ./tst_fileinfo.tmp ./tst_fileinfo2.tmp
$NCDUMP -hs $NPV1 >tst_fileinfo2.tmp
fgrep '_NCProperties' <tst_fileinfo2.tmp > ./tst_fileinfo.tmp
if ! fgrep 'version=1' tst_fileinfo.tmp ; then
if ! XXX=`fgrep 'version=1' tst_fileinfo.tmp` ; then
echo "FAIL: $NPV1 is not marked as version=1"
EXIT=1
fi
echo "PASS: $NPV1 is marked as version=1"
rm -f $NCF
rm -f $HDF
@ -62,5 +75,7 @@ rm -f tst_fileinfo.tmp tst_fileinfo2.tmp
if test "x$EXIT" = x0 ; then
echo "*** Pass all tests"
else
echo "*** FAIL one or more tests"
fi
exit $EXIT