mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-17 16:50:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

starting to support compact storage

Browse Source
This commit is contained in:
Edward Hartnett 2019-12-04 07:53:37 -07:00
parent 7ba4e4d48d
commit 82df2876b6
4 changed files with 58 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/nc4internal.h
View File

@ -205,6 +205,7 @@ typedef struct NC_VAR_INFO
void *fill_value;
size_t *chunksizes;
nc_bool_t contiguous; /**< True if variable is stored contiguously in HDF5 file */
nc_bool_t compact; /**< True if variable is in comact storage in HDF5 file */
int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent) */
nc_bool_t dimscale; /**< True if var is a dimscale */
nc_bool_t *dimscale_attached; /**< Array of flags that are true if dimscale is attached for that dim index */

1
include/netcdf.h
View File

@ -293,6 +293,7 @@ NOTE: The NC_MAX_DIMS, NC_MAX_ATTRS, and NC_MAX_VARS limits
/**@{*/
#define NC_CHUNKED 0
#define NC_CONTIGUOUS 1
#define NC_COMPACT 2
/**@}*/
/** In HDF5 files you can set check-summing for each variable.

79
libhdf5/hdf5var.c
View File

@ -22,6 +22,9 @@
* order. */
#define NC_TEMP_NAME "_netcdf4_temporary_variable_name_for_rename"
/** Number of bytes in 64 MB. */
#define SIXTY_FOUR_MB (67108864)
#ifdef LOGGING
/**
* Report the chunksizes selected for a variable.
@ -707,41 +710,59 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
var->contiguous = NC_FALSE;
}
/* Does the user want a contiguous dataset? Not so fast! Make sure
* that there are no unlimited dimensions, and no filters in use
* for this data. */
if (contiguous && *contiguous)
/* Handle storage settings. */
if (contiguous)
{
if (var->deflate || var->fletcher32 || var->shuffle)
return NC_EINVAL;
for (d = 0; d < var->ndims; d++)
if (var->dim[d]->unlimited)
return NC_EINVAL;
var->contiguous = NC_TRUE;
}
/* Chunksizes anyone? */
if (contiguous && *contiguous == NC_CHUNKED)
{
var->contiguous = NC_FALSE;
/* If the user provided chunksizes, check that they are not too
* big, and that their total size of chunk is less than 4 GB. */
if (chunksizes)
/* Does the user want a contiguous or compact dataset? Not so
* fast! Make sure that there are no unlimited dimensions, and
* no filters in use for this data. */
if (*contiguous)
{
if (var->deflate || var->fletcher32 || var->shuffle)
return NC_EINVAL;
if ((retval = check_chunksizes(grp, var, chunksizes)))
return retval;
/* Ensure chunksize is smaller than dimension size */
for (d = 0; d < var->ndims; d++)
if(!var->dim[d]->unlimited && var->dim[d]->len > 0 && chunksizes[d] > var->dim[d]->len)
return NC_EBADCHUNK;
if (var->dim[d]->unlimited)
return NC_EINVAL;
var->contiguous = NC_TRUE;
}
/* Set the chunksizes for this variable. */
/* Handle chunked storage settings. */
if (*contiguous == NC_CHUNKED)
{
var->contiguous = NC_FALSE;
/* If the user provided chunksizes, check that they are not too
* big, and that their total size of chunk is less than 4 GB. */
if (chunksizes)
{
/* Check the chunksizes for validity. */
if ((retval = check_chunksizes(grp, var, chunksizes)))
return retval;
/* Ensure chunksize is smaller than dimension size */
for (d = 0; d < var->ndims; d++)
if (!var->dim[d]->unlimited && var->dim[d]->len > 0 &&
chunksizes[d] > var->dim[d]->len)
return NC_EBADCHUNK;
/* Set the chunksizes for this variable. */
for (d = 0; d < var->ndims; d++)
var->chunksizes[d] = chunksizes[d];
}
}
else if (*contiguous == NC_COMPACT)
{
size_t ndata = 1;
/* Ensure that total var is < 64 MB. */
for (d = 0; d < var->ndims; d++)
var->chunksizes[d] = chunksizes[d];
ndata *= var->dim[d]->len;
/* Ensure var is small enough to fit in compact storage. */
if (ndata * var->type_info->size > SIXTY_FOUR_MB)
return NC_EINVAL;
var->compact++;
}
}

6
libhdf5/nc4hdf.c
View File

@ -994,11 +994,17 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
}
/* Set the var storage to contiguous, compact, or chunked. */
if (var->contiguous)
{
if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
BAIL(NC_EHDFERR);
}
else if (var->compact)
{
if (H5Pset_layout(plistid, H5D_COMPACT) < 0)
BAIL(NC_EHDFERR);
}
else
{
if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)