mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

performance test for fast global att reads

Browse Source
This commit is contained in:
Ed Hartnett 2018-07-21 07:29:12 -06:00
parent 43f094ca50
commit 7aed50a902
6 changed files with 138 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/nc4internal.h
View File

@ -334,7 +334,7 @@ int nc4_rec_write_groups_types(NC_GRP_INFO_T *grp);
int nc4_enddef_netcdf4_file(NC_FILE_INFO_T *h5);
int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
int nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T * var);
int nc4_read_grp_atts(NC_GRP_INFO_T *grp);
/* int nc4_read_grp_atts(NC_GRP_INFO_T *grp); */
int nc4_read_grp_atts2(NC_GRP_INFO_T *grp);
int nc4_read_var_atts(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);

142
libhdf5/hdf5open.c
View File

@ -1185,82 +1185,82 @@ exit:
return retval;
}
/**
* @internal This function is called by nc4_rec_read_metadata to read
* all the group level attributes (the NC_GLOBAL atts for this
* group).
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @return ::NC_EHDFERR HDF5 returned error.
* @author Ed Hartnett
*/
int
nc4_read_grp_atts(NC_GRP_INFO_T *grp)
{
hid_t attid = -1;
hsize_t num_obj, i;
NC_ATT_INFO_T *att;
NC_TYPE_INFO_T *type;
char obj_name[NC_MAX_HDF5_NAME + 1];
int retval = NC_NOERR;
int hidden = 0;
/* /\** */
/* * @internal This function is called by nc4_rec_read_metadata to read */
/* * all the group level attributes (the NC_GLOBAL atts for this */
/* * group). */
/* * */
/* * @param grp Pointer to group info struct. */
/* * */
/* * @return ::NC_NOERR No error. */
/* * @return ::NC_EHDFERR HDF5 returned error. */
/* * @author Ed Hartnett */
/* *\/ */
/* int */
/* nc4_read_grp_atts(NC_GRP_INFO_T *grp) */
/* { */
/* hid_t attid = -1; */
/* hsize_t num_obj, i; */
/* NC_ATT_INFO_T *att; */
/* NC_TYPE_INFO_T *type; */
/* char obj_name[NC_MAX_HDF5_NAME + 1]; */
/* int retval = NC_NOERR; */
/* int hidden = 0; */
num_obj = H5Aget_num_attrs(grp->hdf_grpid);
for (i = 0; i < num_obj; i++)
{
if ((attid = H5Aopen_idx(grp->hdf_grpid, (unsigned int)i)) < 0)
BAIL(NC_EATTMETA);
if (H5Aget_name(attid, NC_MAX_NAME + 1, obj_name) < 0)
BAIL(NC_EATTMETA);
LOG((3, "reading attribute of _netCDF group, named %s", obj_name));
/* num_obj = H5Aget_num_attrs(grp->hdf_grpid); */
/* for (i = 0; i < num_obj; i++) */
/* { */
/* if ((attid = H5Aopen_idx(grp->hdf_grpid, (unsigned int)i)) < 0) */
/* BAIL(NC_EATTMETA); */
/* if (H5Aget_name(attid, NC_MAX_NAME + 1, obj_name) < 0) */
/* BAIL(NC_EATTMETA); */
/* LOG((3, "reading attribute of _netCDF group, named %s", obj_name)); */
/* See if this a hidden, global attribute */
hidden = 0; /* default */
if(grp->nc4_info->root_grp == grp) {
const NC_reservedatt* ra = NC_findreserved(obj_name);
if(ra != NULL && (ra->flags & NAMEONLYFLAG))
hidden = 1;
}
/* /\* See if this a hidden, global attribute *\/ */
/* hidden = 0; /\* default *\/ */
/* if(grp->nc4_info->root_grp == grp) { */
/* const NC_reservedatt* ra = NC_findreserved(obj_name); */
/* if(ra != NULL && (ra->flags & NAMEONLYFLAG)) */
/* hidden = 1; */
/* } */
/* This may be an attribute telling us that strict netcdf-3
* rules are in effect. If so, we will make note of the fact,
* but not add this attribute to the metadata. It's not a user
* attribute, but an internal netcdf-4 one. */
if(strcmp(obj_name, NC3_STRICT_ATT_NAME)==0)
grp->nc4_info->cmode |= NC_CLASSIC_MODEL;
else if(!hidden) {
/* Add an att struct at the end of the list, and then go to it. */
if ((retval = nc4_att_list_add(grp->att, obj_name, &att)))
BAIL(retval);
retval = read_hdf5_att(grp, attid, att);
if(retval == NC_EBADTYPID) {
if((retval = nc4_att_list_del(grp->att, att)))
BAIL(retval);
} else if(retval) {
BAIL(retval);
} else {
att->created = NC_TRUE;
if ((retval = nc4_find_type(grp->nc4_info, att->nc_typeid, &type)))
BAIL(retval);
}
}
/* Unconditionally close the open attribute */
H5Aclose(attid);
attid = -1;
}
/* /\* This may be an attribute telling us that strict netcdf-3 */
/* * rules are in effect. If so, we will make note of the fact, */
/* * but not add this attribute to the metadata. It's not a user */
/* * attribute, but an internal netcdf-4 one. *\/ */
/* if(strcmp(obj_name, NC3_STRICT_ATT_NAME)==0) */
/* grp->nc4_info->cmode |= NC_CLASSIC_MODEL; */
/* else if(!hidden) { */
/* /\* Add an att struct at the end of the list, and then go to it. *\/ */
/* if ((retval = nc4_att_list_add(grp->att, obj_name, &att))) */
/* BAIL(retval); */
/* retval = read_hdf5_att(grp, attid, att); */
/* if(retval == NC_EBADTYPID) { */
/* if((retval = nc4_att_list_del(grp->att, att))) */
/* BAIL(retval); */
/* } else if(retval) { */
/* BAIL(retval); */
/* } else { */
/* att->created = NC_TRUE; */
/* if ((retval = nc4_find_type(grp->nc4_info, att->nc_typeid, &type))) */
/* BAIL(retval); */
/* } */
/* } */
/* /\* Unconditionally close the open attribute *\/ */
/* H5Aclose(attid); */
/* attid = -1; */
/* } */
/* Remember that we have read the atts for this group. */
grp->atts_not_read = 0;
/* /\* Remember that we have read the atts for this group. *\/ */
/* grp->atts_not_read = 0; */
exit:
if (attid > 0) {
if(H5Aclose(attid) < 0)
BAIL2(NC_EHDFERR);
}
return retval;
}
/* exit: */
/* if (attid > 0) { */
/* if(H5Aclose(attid) < 0) */
/* BAIL2(NC_EHDFERR); */
/* } */
/* return retval; */
/* } */
/**
* @internal Wrap HDF5 allocated memory free operations

2
libsrc4/nc4attr.c
View File

@ -156,7 +156,7 @@ nc4_get_att(int ncid, int varid, const char *name, nc_type *xtype,
if (varid == NC_GLOBAL)
{
if (grp->atts_not_read)
if ((retval = nc4_read_grp_atts(grp)))
if ((retval = nc4_read_grp_atts2(grp)))
return retval;
}
else

2
libsrc4/nc4internal.c
View File

@ -473,7 +473,7 @@ nc4_find_grp_att(NC_GRP_INFO_T *grp, int varid, const char *name, int attnum,
/* Do we need to read the atts? */
if (grp->atts_not_read)
if ((retval = nc4_read_grp_atts(grp)))
if ((retval = nc4_read_grp_atts2(grp)))
return retval;
}
else

2
libsrc4/nc4var.c
View File

@ -261,7 +261,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
{
/* Do we need to read the atts? */
if (grp->atts_not_read)
if ((retval = nc4_read_grp_atts(grp)))
if ((retval = nc4_read_grp_atts2(grp)))
return retval;
*nattsp = ncindexcount(grp->att);

96
nc_test4/tst_attsperf.c
View File

@ -87,17 +87,27 @@ buildfile(size_t num_vars, size_t num_atts, size_t att_len,
/* Open/close the file with netCDF. */
int
readfile(char *file_name, long long *delta)
readfile(char *file_name, long long *delta, int do_inq, int num_vars)
{
int ncid;
struct timeval starttime, endtime;
long long startt, endt;
int natts;
int v;
/* Start the clock. */
gettimeofday(&starttime, NULL);
/* Open and close the file. */
/* Open the file. */
if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR;
/* Do an inq if desired, triggering read of atts. */
for (v = 0; v < num_vars; v++)
if (nc_inq_varnatts(ncid, v, &natts)) ERR;
if (nc_inq_natts(ncid, &natts)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
gettimeofday(&endtime, NULL);
@ -112,7 +122,7 @@ readfile(char *file_name, long long *delta)
/* Open/close the file with HDF5. */
int
readfile_hdf5(char *file_name, long long *delta)
readfile_hdf5(char *file_name, long long *delta, int do_inq, int num_vars)
{
hid_t hdfid, hdf_grpid;
hid_t fapl_id;
@ -127,6 +137,22 @@ readfile_hdf5(char *file_name, long long *delta)
if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI)) ERR;
if ((hdfid = H5Fopen(file_name, H5F_ACC_RDONLY, fapl_id)) < 0) ERR;
if ((hdf_grpid = H5Gopen2(hdfid, "/", H5P_DEFAULT)) < 0) ERR;
/* Do we want to do an inq? */
if (do_inq)
{
if (num_vars)
{
}
else /* global atts */
{
hsize_t num_obj;
/* Find out how many attributes. */
if ((num_obj = H5Aget_num_attrs(hdf_grpid)) < 0) ERR;
}
}
if (H5Gclose(hdf_grpid) < 0) ERR;
if (H5Fclose(hdfid) < 0) ERR;
@ -143,6 +169,7 @@ readfile_hdf5(char *file_name, long long *delta)
#define NUM_RUNS 5
#define NUM_STEPS 20
#define FACTOR 5
#define NUM_INQ_TESTS 2
int
main(int argc, char **argv)
{
@ -150,45 +177,48 @@ main(int argc, char **argv)
char file_name[NC_MAX_NAME + 1];
float tot_nc4, tot_hdf5;
int factor;
int r, s, num_vars;
int r, s, num_vars, do_inq;
for (num_vars = 0; num_vars <= NUM_VARS; num_vars++)
for (do_inq = 0; do_inq < NUM_INQ_TESTS; do_inq++)
{
/* Reset. */
num_atts = 1;
/* Set higher factor for var atts, since they are much faster. */
factor = num_vars ? FACTOR * 10 : FACTOR;
printf("*** %s\n", num_vars ? "variable attributes" : "global attributes");
printf("Number of Attributes\tHDF5 Open Time (s)\tNetcdf4 Open Time (s)\n");
for (s = 0; s < NUM_STEPS; s++)
for (num_vars = 0; num_vars <= NUM_VARS; num_vars++)
{
tot_nc4 = 0;
tot_hdf5 = 0;
num_atts += factor * s;
/* Reset. */
num_atts = 1;
for (r = 0; r < NUM_RUNS; r++)
/* Set higher factor for var atts, since they are much faster. */
factor = num_vars ? FACTOR * 10 : FACTOR;
printf("*** %s %s\n", num_vars ? "variable attributes" : "global attributes",
do_inq ? "with inq" : "");
printf("Number of Attributes\tHDF5 Open Time (s)\tNetcdf4 Open Time (s)\n");
for (s = 0; s < NUM_STEPS; s++)
{
long long nc4_open_time;
long long hdf5_open_time;
tot_nc4 = 0;
tot_hdf5 = 0;
num_atts += factor * s;
/* Determine file name. */
sprintf(file_name, "%s_%d_%d_%d.nc", TEST, num_vars, s, r);
for (r = 0; r < NUM_RUNS; r++)
{
long long nc4_open_time;
long long hdf5_open_time;
if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR;
if (readfile(file_name, &nc4_open_time)) ERR;
if (readfile_hdf5(file_name, &hdf5_open_time)) ERR;
tot_nc4 += nc4_open_time;
tot_hdf5 += hdf5_open_time;
/* Determine file name. */
sprintf(file_name, "%s_%d_%d_%d.nc", TEST, num_vars, s, r);
if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR;
if (readfile(file_name, &nc4_open_time, do_inq, num_vars)) ERR;
if (readfile_hdf5(file_name, &hdf5_open_time, do_inq, num_vars)) ERR;
tot_nc4 += nc4_open_time;
tot_hdf5 += hdf5_open_time;
}
/* Print average results to the millisec */
printf("%ld\t%g\t%g\n", num_atts, tot_hdf5/((float)NUM_RUNS * 1000000),
tot_nc4/((float)NUM_RUNS * 1000000));
}
/* Print average results to the millisec */
printf("%ld\t%g\t%g\n", num_atts, tot_hdf5/((float)NUM_RUNS * 1000000),
tot_nc4/((float)NUM_RUNS * 1000000));
}
}
} /* next do_inq */
SUMMARIZE_ERR;
FINAL_RESULTS;
}