mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

merged ejh_fast_var

Browse Source
This commit is contained in:
Ed Hartnett 2019-01-03 06:26:45 -07:00
commit 4f2d505724
7 changed files with 163 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/hdf5internal.h
View File

@ -174,5 +174,8 @@ int nc4_hdf5_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
NC_GRP_INFO_T **grp, NC_VAR_INFO_T **var,
NC_ATT_INFO_T **att);
/* Find var, doing lazy var metadata read if needed. */
int nc4_hdf5_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
NC_GRP_INFO_T **grp, NC_VAR_INFO_T **var);
#endif /* _HDF5INTERNAL_ */

1
include/nc4internal.h
View File

@ -154,6 +154,7 @@ typedef struct NC_VAR_INFO
nc_bool_t written_to; /* True if variable has data written to it */
struct NC_TYPE_INFO *type_info;
int atts_not_read; /* If true, the atts have not yet been read. */
nc_bool_t meta_read; /* True if this vars metadata has been completely read. */
NCindex *att; /* NCindex<NC_ATT_INFO_T*> */
nc_bool_t no_fill; /* True if no fill value is defined for var */
void *fill_value;

62
libhdf5/hdf5internal.c
View File

@ -728,8 +728,61 @@ nc4_rec_grp_HDF5_del(NC_GRP_INFO_T *grp)
}
/**
* @internal Given an ncid and varid, find an att. Lazy reads are done
* as needed.
* @internal Given an ncid and varid, get pointers to the group and var
* metadata. Lazy var metadata reads are done as needed.
*
* @param ncid File ID.
* @param varid Variable ID.
* @param h5 Pointer that gets pointer to the NC_FILE_INFO_T struct
* for this file. Ignored if NULL.
* @param grp Pointer that gets pointer to group info. Ignored if
* NULL.
* @param var Pointer that gets pointer to var info. Ignored if NULL.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENOTVAR Variable not found.
* @author Ed Hartnett
*/
int
nc4_hdf5_find_grp_h5_var(int ncid, int varid, NC_FILE_INFO_T **h5,
NC_GRP_INFO_T **grp, NC_VAR_INFO_T **var)
{
NC_FILE_INFO_T *my_h5;
NC_GRP_INFO_T *my_grp;
NC_VAR_INFO_T *my_var;
int retval;
/* Look up file and group metadata. */
if ((retval = nc4_find_grp_h5(ncid, &my_grp, &my_h5)))
return retval;
assert(my_grp && my_h5);
/* Find the var. */
if (!(my_var = (NC_VAR_INFO_T *)ncindexith(my_grp->vars, varid)))
return NC_ENOTVAR;
assert(my_var && my_var->hdr.id == varid);
/* Do we need to read var metadata? */
if (!my_var->meta_read && my_var->created)
if ((retval = nc4_get_var_meta(my_var)))
return retval;
/* Return pointers that caller wants. */
if (h5)
*h5 = my_h5;
if (grp)
*grp = my_grp;
if (var)
*var = my_var;
return NC_NOERR;
}
/**
* @internal Given an ncid, varid, and attribute name, return
* normalized name and pointers to the file, group, var, and att info
* structs. Lazy reads of attributes and variable metadata are done as
* needed.
*
* @param ncid File/group ID.
* @param varid Variable ID.
@ -801,6 +854,11 @@ nc4_hdf5_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
if ((retval = nc4_read_atts(my_grp, my_var)))
return retval;
/* Do we need to read var metadata? */
if (!my_var->meta_read && my_var->created)
if ((retval = nc4_get_var_meta(my_var)))
return retval;
attlist = my_var->att;
}
assert(attlist);

41
libhdf5/hdf5open.c
View File

@ -99,8 +99,7 @@ typedef struct {
* @author Ed Hartnett
*/
static int
get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid,
NC_TYPE_INFO_T **type_info)
get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid, NC_TYPE_INFO_T **type_info)
{
NC_HDF5_TYPE_INFO_T *hdf5_type;
htri_t is_str, equal = 0;
@ -191,17 +190,17 @@ get_type_info2(NC_FILE_INFO_T *h5, hid_t datasetid,
/* Find out about endianness. As of HDF 1.8.6, this works
* with all data types Not just H5T_INTEGER. See
* https://www.hdfgroup.org/HDF5/doc/RM/RM_H5T.html#Datatype-GetOrder */
if((order = H5Tget_order(hdf_typeid)) < 0)
if ((order = H5Tget_order(hdf_typeid)) < 0)
return NC_EHDFERR;
if(order == H5T_ORDER_LE)
if (order == H5T_ORDER_LE)
(*type_info)->endianness = NC_ENDIAN_LITTLE;
else if(order == H5T_ORDER_BE)
else if (order == H5T_ORDER_BE)
(*type_info)->endianness = NC_ENDIAN_BIG;
else
return NC_EBADTYPE;
if(class == H5T_INTEGER)
if (class == H5T_INTEGER)
(*type_info)->nc_type_class = NC_INT;
else
(*type_info)->nc_type_class = NC_FLOAT;
@ -543,8 +542,6 @@ rec_match_dimscales(NC_GRP_INFO_T *grp)
}
} /* next dim */
} /* next grp */
LOG((5, "%s: dimid for this dimscale is %d", __func__,
var->type_info->hdr.id));
} /* next var->dim */
}
else
@ -1124,8 +1121,8 @@ get_scale_info(NC_GRP_INFO_T *grp, NC_DIM_INFO_T *dim, NC_VAR_INFO_T *var,
* @return ::NC_EVARMETA Error with var metadata.
* @author Ed Hartnett
*/
static int
get_var_meta(NC_VAR_INFO_T *var)
int
nc4_get_var_meta(NC_VAR_INFO_T *var)
{
NC_HDF5_VAR_INFO_T *hdf5_var;
hid_t access_pid = 0;
@ -1135,6 +1132,10 @@ get_var_meta(NC_VAR_INFO_T *var)
assert(var && var->format_var_info);
/* Have we already read the var metadata? */
if (var->meta_read)
return NC_NOERR;
/* Get pointer to the HDF5-specific var info struct. */
hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
@ -1160,14 +1161,6 @@ get_var_meta(NC_VAR_INFO_T *var)
if ((retval = get_filter_info(propid, var)))
BAIL(retval);
/* Learn all about the type of this variable. */
if ((retval = get_type_info2(var->container->nc4_info, hdf5_var->hdf_datasetid,
&var->type_info)))
BAIL(retval);
/* Indicate that the variable has a pointer to the type */
var->type_info->rc++;
/* Get fill value, if defined. */
if ((retval = get_fill_info(propid, var)))
BAIL(retval);
@ -1177,6 +1170,9 @@ get_var_meta(NC_VAR_INFO_T *var)
if ((retval = nc4_adjust_var_cache(var->container, var)))
BAIL(retval);
/* Remember that we have read the metadata for this var. */
var->meta_read = NC_TRUE;
exit:
if (access_pid && H5Pclose(access_pid) < 0)
BAIL2(NC_EHDFERR);
@ -1255,15 +1251,20 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
retval = read_coord_dimids(grp, var);
if (retval && retval != NC_ENOTATT)
BAIL(retval);
retval = NC_NOERR;
/* Handle scale info. */
if ((retval = get_scale_info(grp, dim, var, hdf5_var, ndims, datasetid)))
BAIL(retval);
/* Get the rest of the metadata for this variable. */
if ((retval = get_var_meta(var)))
/* Learn all about the type of this variable. */
if ((retval = get_type_info2(var->container->nc4_info, hdf5_var->hdf_datasetid,
&var->type_info)))
BAIL(retval);
/* Indicate that the variable has a pointer to the type */
var->type_info->rc++;
exit:
if (finalname)
free(finalname);

12
libhdf5/hdf5var.c
View File

@ -409,6 +409,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype,
BAIL(NC_ENOMEM);
var->is_new_var = NC_TRUE;
var->meta_read = NC_TRUE;
/* Point to the type, and increment its ref. count */
var->type_info = type;
@ -852,7 +853,7 @@ nc_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp)
int i, retval;
/* Get pointer to the var. */
if ((retval = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
if ((retval = nc4_hdf5_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
return retval;
assert(var);
@ -1088,6 +1089,11 @@ NC4_rename_var(int ncid, int varid, const char *name)
there. */
if (var->created)
{
/* Do we need to read var metadata? */
if (!var->meta_read)
if ((retval = nc4_get_var_meta(var)))
return retval;
if (var->ndims)
{
NC_HDF5_DIM_INFO_T *hdf5_d0;
@ -1353,7 +1359,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
size_t len = 1;
/* Find info for this file, group, and var. */
if ((retval = nc4_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
if ((retval = nc4_hdf5_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
return retval;
assert(h5 && grp && var && var->hdr.id == varid && var->format_var_info);
@ -1679,7 +1685,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
size_t len = 1;
/* Find info for this file, group, and var. */
if ((retval = nc4_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
if ((retval = nc4_hdf5_find_grp_h5_var(ncid, varid, &h5, &grp, &var)))
return retval;
assert(h5 && grp && var && var->hdr.id == varid && var->format_var_info &&
var->type_info && var->type_info->size && var->type_info->format_type_info);

90
nc_test4/tst_attsperf.c
View File

@ -28,6 +28,7 @@
#define NUM_ATTS 100
#define ATT_LEN 100
#define NUM_VARS 1
#define NUM_VARS_MANY 10000
int
add_attributes(int ncid, int varid, size_t num_atts, size_t att_len)
@ -59,8 +60,7 @@ add_attributes(int ncid, int varid, size_t num_atts, size_t att_len)
/* Build the test file. */
int
buildfile(size_t num_vars, size_t num_atts, size_t att_len,
char *file_name)
buildfile(size_t num_vars, size_t num_atts, size_t att_len, char *file_name)
{
int ncid, varid;
int dimids[NDIMS];
@ -102,10 +102,13 @@ readfile(char *file_name, long long *delta, int do_inq, int num_vars)
if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR;
/* Do an inq if desired, triggering read of atts. */
for (v = 0; v < num_vars; v++)
if (nc_inq_varnatts(ncid, v, &natts)) ERR;
if (do_inq)
{
for (v = 0; v < num_vars; v++)
if (nc_inq_varnatts(ncid, v, &natts)) ERR;
if (nc_inq_natts(ncid, &natts)) ERR;
if (nc_inq_natts(ncid, &natts)) ERR;
}
/* Close the file. */
if (nc_close(ncid)) ERR;
@ -166,9 +169,10 @@ readfile_hdf5(char *file_name, long long *delta, int do_inq, int num_vars)
return 0;
}
#define NUM_RUNS 5
#define NUM_RUNS 1
#define NUM_STEPS 20
#define FACTOR 100
#define VAR_FACTOR 100
#define NUM_INQ_TESTS 2
int
main(int argc, char **argv)
@ -179,45 +183,81 @@ main(int argc, char **argv)
int factor;
int r, s, num_vars, do_inq;
for (do_inq = 0; do_inq < NUM_INQ_TESTS; do_inq++)
printf("Testing with many vars...\n");
{
for (num_vars = 0; num_vars <= NUM_VARS; num_vars++)
for (do_inq = 0; do_inq < 1; do_inq++)
{
/* Reset. */
num_atts = 1;
factor = FACTOR;
printf("*** %s %s\n", num_vars ? "variable attributes" : "global attributes",
do_inq ? "with inq" : "");
printf("Number of Attributes\tHDF5 Open Time (s)\tNetcdf4 Open Time (s)\n");
for (s = 0; s < NUM_STEPS; s++)
printf("Number of Variables\tHDF5 Open Time (s)\tNetcdf4 Open Time (s)\n");
for (num_vars = 1; num_vars <= NUM_VARS_MANY; num_vars += VAR_FACTOR)
{
num_atts = 10;
tot_nc4 = 0;
tot_hdf5 = 0;
num_atts += factor * s;
for (r = 0; r < NUM_RUNS; r++)
{
long long nc4_open_time;
long long hdf5_open_time;
long long nc4_open_time = 0;
long long hdf5_open_time = 0;
/* Determine file name. */
sprintf(file_name, "%s_%d_%d_%d.nc", TEST, num_vars, s, r);
sprintf(file_name, "%s_%d_%d.nc", TEST, num_vars, r);
if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR;
if (readfile(file_name, &nc4_open_time, do_inq, num_vars)) ERR;
if (readfile_hdf5(file_name, &hdf5_open_time, do_inq, num_vars)) ERR;
/* if (readfile_hdf5(file_name, &hdf5_open_time, do_inq, num_vars)) ERR; */
tot_nc4 += nc4_open_time;
tot_hdf5 += hdf5_open_time;
}
/* Print average results to the millisec */
printf("%ld\t%g\t%g\n", num_atts, tot_hdf5/((float)NUM_RUNS * 1000000),
printf("%d\t%g\t%g\n", num_vars, tot_hdf5/((float)NUM_RUNS * 1000000),
tot_nc4/((float)NUM_RUNS * 1000000));
}
}
} /* next do_inq */
} /* next do_inq */
}
SUMMARIZE_ERR;
printf("Testing with many atts...\n");
{
for (do_inq = 0; do_inq < NUM_INQ_TESTS; do_inq++)
{
for (num_vars = 0; num_vars <= NUM_VARS; num_vars++)
{
/* Reset. */
num_atts = 1;
factor = FACTOR;
printf("*** %s %s\n", num_vars ? "variable attributes" : "global attributes",
do_inq ? "with inq" : "");
printf("Number of Attributes\tHDF5 Open Time (s)\tNetcdf4 Open Time (s)\n");
for (s = 0; s < NUM_STEPS; s++)
{
tot_nc4 = 0;
tot_hdf5 = 0;
num_atts += factor * s;
for (r = 0; r < NUM_RUNS; r++)
{
long long nc4_open_time;
long long hdf5_open_time;
/* Determine file name. */
sprintf(file_name, "%s_%d_%d_%d.nc", TEST, num_vars, s, r);
if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR;
if (readfile(file_name, &nc4_open_time, do_inq, num_vars)) ERR;
if (readfile_hdf5(file_name, &hdf5_open_time, do_inq, num_vars)) ERR;
tot_nc4 += nc4_open_time;
tot_hdf5 += hdf5_open_time;
}
/* Print average results to the millisec */
printf("%ld\t%g\t%g\n", num_atts, tot_hdf5/((float)NUM_RUNS * 1000000),
tot_nc4/((float)NUM_RUNS * 1000000));
}
}
} /* next do_inq */
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}

8
nc_test4/tst_grps.c
View File

@ -120,11 +120,11 @@ main(int argc, char **argv)
/* Create a file with one group, a group to contain data about
* Henry VII. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_var(ncid, HENRY_IV, NC_INT, 0, NULL, NULL)) ERR;
if (nc_def_var(ncid, HENRY_IV, NC_INT, 0, NULL, NULL)) ERR;
/* Turn off define mode. It will automatically be turned back on
* when nc_def_grp is called. */
if (nc_enddef(ncid)) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_def_grp(ncid, HENRY_VII, &henry_vii_id)) ERR;
/* Check it out. */
@ -377,7 +377,7 @@ main(int argc, char **argv)
if (nc_inq_grps(ncid, NULL, grpid_in)) ERR;
if (nc_inq_grpname(grpid_in[0], name_in)) ERR;
if (strcmp(name_in, HENRY_VII)) ERR;
if (nc_inq_grpname(grpid_in[0], NULL)) ERR;
if (nc_inq_grpname(grpid_in[0], NULL)) ERR;
if (grpid_in[0] != grp_ncid) ERR;
strcat(full_name, HENRY_VII);
if (nc_inq_grpname_full(grpid_in[0], &len, full_name_in)) ERR;
@ -443,7 +443,7 @@ main(int argc, char **argv)
if (nc_inq_grpname_full(grpid_in[0], &len1, NULL)) ERR;
if (len1 != strlen(full_name)) ERR;
if (nc_inq_grpname_full(grpid_in[0], &len, full_name_in1)) ERR;
if (strcmp(full_name_in1, full_name)) ERR;
if (strcmp(full_name_in1, full_name)) ERR;
if (nc_close(ncid)) ERR;