mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1663 from NOAA-GSD/ejh_refactor_storage

Browse Source 
refactor storage in NC_VAR_INFO_T
This commit is contained in:
Ward Fisher 2020-03-09 15:44:30 -06:00 committed by GitHub
commit acce5eec88
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 510 additions and 519 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

929
h5_test/tst_h_vars2.c
View File

File diff suppressed because it is too large Load Diff

3
include/nc4internal.h
View File

@ -208,8 +208,7 @@ typedef struct NC_VAR_INFO
nc_bool_t no_fill; /**< True if no fill value is defined for var. */
void *fill_value; /**< Pointer to fill value, or NULL. */
size_t *chunksizes; /**< For chunked storage, an array (size ndims) of chunksizes. */
nc_bool_t contiguous; /**< True if variable is stored contiguously in HDF5 file. */
nc_bool_t compact; /**< True if variable is in comact storage in HDF5 file. */
int storage; /**< Storage of this var, compact, contiguous, or chunked. */
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
nc_bool_t dimscale; /**< True if var is a dimscale. */
nc_bool_t *dimscale_attached; /**< Array of flags that are true if dimscale is attached for that dim index. */

5
libhdf4/hdf4file.c
View File

@ -435,7 +435,10 @@ nc4_var_list_add_full(NC_GRP_INFO_T* grp, const char* name, int ndims, nc_type x
}
/* Var contiguous or chunked? */
(*var)->contiguous = contiguous;
if (contiguous)
(*var)->storage = NC_CONTIGUOUS;
else
(*var)->storage = NC_CHUNKED;
/* Were chunksizes provided? */
if (chunksizes)

2
libhdf5/hdf5filter.c
View File

@ -277,7 +277,7 @@ NC4_filter_actions(int ncid, int varid, int op, NC_Filterobject* args)
return THROW(NC_EFILTER); /* Not allowed */
#endif
/* Filter => chunking */
var->contiguous = NC_FALSE;
var->storage = NC_CHUNKED;
/* Determine default chunksizes for this variable unless already specified */
if(var->chunksizes && !var->chunksizes[0]) {
/* Should this throw error? */

9
libhdf5/hdf5open.c
View File

@ -1096,6 +1096,7 @@ get_chunking_info(hid_t propid, NC_VAR_INFO_T *var)
/* Remember the layout and, if chunked, the chunksizes. */
if (layout == H5D_CHUNKED)
{
var->storage = NC_CHUNKED;
if (H5Pget_chunk(propid, H5S_MAX_RANK, chunksize) < 0)
return NC_EHDFERR;
if (!(var->chunksizes = malloc(var->ndims * sizeof(size_t))))
@ -1104,9 +1105,13 @@ get_chunking_info(hid_t propid, NC_VAR_INFO_T *var)
var->chunksizes[d] = chunksize[d];
}
else if (layout == H5D_CONTIGUOUS)
var->contiguous = NC_TRUE;
{
var->storage = NC_CONTIGUOUS;
}
else if (layout == H5D_COMPACT)
var->compact = NC_TRUE;
{
var->storage = NC_COMPACT;
}
return NC_NOERR;
}

45
libhdf5/hdf5var.c
View File

@ -174,8 +174,8 @@ nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
#endif
if(var->chunksizes == NULL) {
if((var->chunksizes = calloc(1,sizeof(size_t)*var->ndims)) == NULL)
return NC_ENOMEM;
if((var->chunksizes = calloc(1,sizeof(size_t)*var->ndims)) == NULL)
return NC_ENOMEM;
}
/* How many values in the variable (or one record, if there are
@ -509,7 +509,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims,
* same name as one of its dimensions. If it is a coordinate var,
* is it a coordinate var in the same group as the dim? Also, check
* whether we should use contiguous or chunked storage. */
var->contiguous = NC_TRUE;
var->storage = NC_CONTIGUOUS;
for (d = 0; d < ndims; d++)
{
NC_GRP_INFO_T *dim_grp;
@ -549,9 +549,12 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims,
}
}
/* Check for unlimited dimension and turn off contiguous storage. */
/* Check for unlimited dimension. If present, we must use
* chunked storage. */
if (dim->unlimited)
var->contiguous = NC_FALSE;
{
var->storage = NC_CHUNKED;
}
/* Track dimensions for variable */
var->dimids[d] = dimidsp[d];
@ -570,7 +573,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, int ndims,
/* Is this a variable with a chunksize greater than the current
* cache size? */
if ((retval = nc4_adjust_var_cache(grp, var)))
BAIL(retval);
BAIL(retval);
}
/* If the user names this variable the same as a dimension, but
@ -678,25 +681,24 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
/* Cannot set filters of any sort on scalars */
if(var->ndims == 0) {
if(shuffle && *shuffle)
return NC_EINVAL;
if(fletcher32 && *fletcher32)
return NC_EINVAL;
if(shuffle && *shuffle)
return NC_EINVAL;
if(fletcher32 && *fletcher32)
return NC_EINVAL;
}
/* Shuffle filter? */
if (shuffle)
{
var->shuffle = *shuffle;
var->contiguous = NC_FALSE;
var->compact = NC_FALSE; }
var->storage = NC_CHUNKED;
}
/* Fletcher32 checksum error protection? */
if (fletcher32)
{
var->fletcher32 = *fletcher32;
var->contiguous = NC_FALSE;
var->compact = NC_FALSE;
var->storage = NC_CHUNKED;
}
#ifdef USE_PARALLEL
@ -730,11 +732,12 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
/* Handle chunked storage settings. */
if (*storage == NC_CHUNKED && var->ndims == 0)
{
/* Chunked not allowed for scalar vars. */
return NC_EINVAL;
} else if (*storage == NC_CHUNKED)
}
else if (*storage == NC_CHUNKED)
{
var->contiguous = NC_FALSE;
var->compact = NC_FALSE;
var->storage = NC_CHUNKED;
/* If the user provided chunksizes, check that they are not too
* big, and that their total size of chunk is less than 4 GB. */
@ -757,8 +760,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
}
else if (*storage == NC_CONTIGUOUS)
{
var->contiguous = NC_TRUE;
var->compact = NC_FALSE;
var->storage = NC_CONTIGUOUS;
}
else if (*storage == NC_COMPACT)
{
@ -773,14 +775,13 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
if (ndata * var->type_info->size > SIXTY_FOUR_KB)
return NC_EVARSIZE;
var->contiguous = NC_FALSE;
var->compact = NC_TRUE;
var->storage = NC_COMPACT;
}
}
/* Is this a variable with a chunksize greater than the current
* cache size? */
if (!var->contiguous && !var->compact)
if (var->storage == NC_CHUNKED)
{
/* Determine default chunksizes for this variable (do nothing
* for scalar vars). */

17
libhdf5/nc4hdf.c
View File

@ -970,7 +970,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
* better performance. */
if (!var->shuffle && !var->fletcher32 && nclistlength(var->filters) == 0 &&
(var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
var->contiguous = NC_TRUE;
var->storage = NC_CONTIGUOUS;
/* Gather current & maximum dimension sizes, along with chunk
* sizes. */
@ -980,7 +980,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
assert(dim && dim->hdr.id == var->dimids[d]);
dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
if (!var->contiguous && !var->compact)
if (var->storage == NC_CHUNKED)
{
if (var->chunksizes[d])
chunksize[d] = var->chunksizes[d];
@ -1023,12 +1023,12 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
/* Set the var storage to contiguous, compact, or chunked. Don't
* try to set chunking for scalar vars, they will default to
* contiguous if not set to compact. */
if (var->contiguous)
if (var->storage == NC_CONTIGUOUS)
{
if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
BAIL(NC_EHDFERR);
}
else if (var->compact)
else if (var->storage == NC_COMPACT)
{
if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
BAIL(NC_EHDFERR);
@ -1045,7 +1045,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
BAIL(NC_EHDFERR);
/* Set per-var chunk cache, for chunked datasets. */
if (!var->contiguous && !var->compact && var->chunk_cache_size)
if (var->storage == NC_CHUNKED && var->chunk_cache_size)
if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
var->chunk_cache_size, var->chunk_cache_preemption) < 0)
BAIL(NC_EHDFERR);
@ -1117,10 +1117,13 @@ exit:
* @internal Adjust the chunk cache of a var for better
* performance.
*
* @note For contiguous and compact storage vars, or when parallel I/O
* is in use, this function will do nothing and return ::NC_NOERR;
*
* @param grp Pointer to group info struct.
* @param var Pointer to var info struct.
*
* @return NC_NOERR No error.
* @return ::NC_NOERR No error.
* @author Ed Hartnett
*/
int
@ -1131,7 +1134,7 @@ nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
int retval;
/* Nothing to be done for contiguous or compact data. */
if (var->contiguous || var->compact)
if (var->storage != NC_CHUNKED)
return NC_NOERR;
#ifdef USE_PARALLEL4

4
libsrc4/nc4internal.c
View File

@ -1696,9 +1696,9 @@ rec_print_metadata(NC_GRP_INFO_T *grp, int tab_count)
}
if (!var->meta_read)
strcat(storage_str, "unknown");
else if (var->contiguous)
else if (var->storage == NC_CONTIGUOUS)
strcat(storage_str, "contiguous");
else if (var->compact)
else if (var->storage == NC_COMPACT)
strcat(storage_str, "compact");
else
strcat(storage_str, "chunked");

15
libsrc4/nc4var.c
View File

@ -123,7 +123,7 @@ nc_get_var_chunk_cache_ints(int ncid, int varid, int *sizep,
* @param deflatep Gets deflate setting.
* @param deflate_levelp Gets deflate level.
* @param fletcher32p Gets fletcher32 setting.
* @param storagep Gets contiguous setting.
* @param storagep Gets storage setting.
* @param chunksizesp Gets chunksizes.
* @param no_fill Gets fill mode.
* @param fill_valuep Gets fill value.
@ -188,7 +188,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
*nattsp = ncindexcount(var->att);
/* Did the user want the chunksizes? */
if (!var->contiguous && !var->compact && chunksizesp)
if (var->storage == NC_CHUNKED && chunksizesp)
{
for (d = 0; d < var->ndims; d++)
{
@ -199,14 +199,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
/* Did the user inquire about the storage? */
if (storagep)
{
if (var->contiguous)
*storagep = NC_CONTIGUOUS;
else if (var->compact)
*storagep = NC_COMPACT;
else
*storagep = NC_CHUNKED;
}
*storagep = var->storage;
/* Filter stuff. */
if (shufflep)
@ -330,7 +323,7 @@ nc_inq_var_chunking_ints(int ncid, int varid, int *storagep, int *chunksizesp)
NULL, NULL, NULL, NULL, NULL);
/* Copy from size_t array. */
if (!retval && chunksizesp && var->contiguous == NC_CHUNKED)
if (!retval && chunksizesp && var->storage == NC_CHUNKED)
{
for (i = 0; i < var->ndims; i++)
{