From 58e5d53e9630001801d167e01de396a6e91581a4 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Wed, 8 Apr 2020 09:09:46 -0600 Subject: [PATCH] documentation improvements --- libdispatch/dvar.c | 35 +++++++++++++++++++++++++++++++---- libdispatch/dvarinq.c | 17 +++++++++++------ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 91d93230d..143cbff8f 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -317,12 +317,16 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) } /** - Set the zlib compression settings for a netCDF-4/HDF5 variable. + Set the zlib compression and shuffle settings for a variable in an + netCDF/HDF5 file. This function must be called after nc_def_var and before nc_enddef or any functions which writes data to the file. - Deflation and shuffline require chunked data. If this function is + Deflation and shuffle are only available for HDF5 files. Attempting + to set them on non-HDF5 files will return ::NC_ENOTNC4. + + Deflation and shuffle require chunked data. If this function is called on a variable with contiguous data, then the data is changed to chunked data, with default chunksizes. Use nc_def_var_chunking() to tune performance with user-defined chunksizes. @@ -330,8 +334,9 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) If this function is called on a scalar variable, ::NC_EINVAL is returned. Only chunked variables may use filters. - If this function is called on a variable which already has szip - compression turned on, ::NC_EINVAL is returned. + Zlib compression cannot be used with szip compression. If this + function is called on a variable which already has szip compression + turned on, ::NC_EINVAL is returned. @note Parallel I/O reads work with compressed data. Parallel I/O writes work with compressed data in netcdf-c-4.7.4 and later @@ -341,6 +346,28 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) variable in a file opened for parallel I/O will automatically switch the access for that variable to collective access. + @note The HDF5 manual has this to say about shuffle: + + The shuffle filter de-interlaces a block of data by reordering + the bytes. All the bytes from one consistent byte position of + each data element are placed together in one block; all bytes + from a second consistent byte position of each data element are + placed together a second block; etc. For example, given three + data elements of a 4-byte datatype stored as 012301230123, + shuffling will re-order data as 000111222333. This can be a + valuable step in an effective compression algorithm because the + bytes in each byte position are often closely related to each + other and putting them together can increase the compression + ratio. + + As implied above, the primary value of the shuffle filter lies + in its coordinated use with a compression filter; it does not + provide data compression when used alone. When the shuffle + filter is applied to a dataset immediately prior to the use of a + compression filter, the compression ratio achieved is often + superior to that achieved by the use of a compression filter + without the shuffle filter. + @param ncid NetCDF or group ID, from a previous call to nc_open(), nc_create(), nc_def_grp(), or associated inquiry functions such as nc_inq_ncid(). diff --git a/libdispatch/dvarinq.c b/libdispatch/dvarinq.c index 7a5e67b04..0e3958f65 100644 --- a/libdispatch/dvarinq.c +++ b/libdispatch/dvarinq.c @@ -256,13 +256,18 @@ nc_inq_varnatts(int ncid, int varid, int *nattsp) } /** -\ingroup variables Learn the storage and deflate settings for a -variable. +\ingroup variables -Deflation is compression with the zlib library. Deflation is only -available for netCDF/HDF5 files. For classic and other files, this -function will return shuffle off, deflate off, and a deflate level of -0. +Learn the shuffle and deflate settings for a variable. + +Deflation is compression with the zlib library. Shuffle re-orders the +data bytes to provide better compression (see nc_def_var_deflate()). + +Deflation is only +available for HDF5 files. For classic and other files, this function +will return setting that indicate that deflation is not in use, and +that the shuffle filter is not in use. That is: shuffle off, deflate +off, and a deflate level of 0. \param ncid NetCDF or group ID, from a previous call to nc_open(), nc_create(), nc_def_grp(), or associated inquiry functions such as