mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-19 17:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'csz_bitround' of https://github.com/nco/netcdf-c into gh2232.wif

Browse Source
This commit is contained in:
Ward Fisher 2022-04-01 10:43:34 -06:00
commit 2ccdf14697
17 changed files with 1411 additions and 1191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/netcdf.h
View File

@ -331,19 +331,23 @@ there. */
#define NC_NOQUANTIZE 0 /**< No quantization in use. */
#define NC_QUANTIZE_BITGROOM 1 /**< Use BitGroom quantization. */
#define NC_QUANTIZE_GRANULARBR 2 /**< Use Granular BitRound quantization. */
#define NC_QUANTIZE_BITROUND 3 /**< Use BitRound quantization. */
/** When quantization is used for a variable, an attribute of the
* appropriate name is added. */
#define NC_QUANTIZE_BITGROOM_ATT_NAME "_QuantizeBitgroomNumberOfSignificantDigits"
#define NC_QUANTIZE_BITGROOM_ATT_NAME "_QuantizeBitGroomNumberOfSignificantDigits"
#define NC_QUANTIZE_GRANULARBR_ATT_NAME "_QuantizeGranularBitRoundNumberOfSignificantDigits"
#define NC_QUANTIZE_BITROUND_ATT_NAME "_QuantizeBitRoundNumberOfSignificantBits"
/** For quantization, the allowed value of number of significant
* digits for float. */
* decimal and binary digits, respectively, for float. */
#define NC_QUANTIZE_MAX_FLOAT_NSD (7)
#define NC_QUANTIZE_MAX_FLOAT_NSB (23)
/** For quantization, the allowed value of number of significant
* digits for double. */
* decimal and binary digits, respectively, for double. */
#define NC_QUANTIZE_MAX_DOUBLE_NSD (15)
#define NC_QUANTIZE_MAX_DOUBLE_NSB (52)
/** The netcdf version 3 functions all return integer error status.
* These are the possible values, in addition to certain values from

26
libdispatch/dvar.c
View File

@ -469,7 +469,7 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
The data are quantized by setting unneeded bits to zeros or ones
so that they may compress well. BitGroom sets bits alternately to 1/0,
while Granular BitRound (GBR) sets (more) bits to zeros.
while BitRound and Granular BitRound (GBR) round (more) bits to zeros
Quantization is lossy (data are irretrievably altered), and it
improves the compression ratio provided by a subsequent lossless
compression filter. Quantization alone will not reduce the data size.
@ -479,9 +479,8 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
compression will result in significant improvent in the final data
size.
This data quantization used the BitGroom algorithm. A notable
feature of BitGroom is that the data it processes remain in IEEE754
format after quantization. Therefore the BitGroom algorithm does
A notable feature of all the quantization algorithms is data remain
in IEEE754 format afterwards. Therefore quantization algorithms do
nothing when data are read.
Quantization is only available for variables of type NC_FLOAT or
@ -489,8 +488,8 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
types return an error (NC_EINVAL).
Variables that use quantize will have added an attribute with name
::NC_QUANTIZE_ATT_NAME, which will contain the number of
significant digits. Users should not delete or change this
::NC_QUANTIZE_[ALGORITHM_NAME]_ATT_NAME, which will contain the
number of significant digits. Users should not delete or change this
attribute. This is the only record that quantize has been applied
to the data.
@ -529,11 +528,16 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
@param ncid File ID.
@param varid Variable ID. ::NC_GLOBAL may not be used.
@param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
@param nsd Number of significant digits. May be any integer from 1
to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT)
or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type
::NC_DOUBLE). Ignored if quantize_mode = NC_NOQUANTIZE.
::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR or
::NC_QUANTIZE_BITROUND.
@param nsd Number of significant digits (either decimal or binary).
May be any integer from 1 to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables
of type ::NC_FLOAT) or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables
of type ::NC_DOUBLE) for mode ::NC_QUANTIZE_BITGROOM and mode
::NC_QUANTIZE_GRANULARBR. May be any integer from 1 to
::NC_QUANTIZE_MAX_FLOAT_NSB (for variables of type ::NC_FLOAT) or
::NC_QUANTIZE_MAX_DOUBLE_NSB (for variables of type ::NC_DOUBLE)
for mode ::NC_QUANTIZE_BITROUND. Ignored if quantize_mode = NC_NOQUANTIZE.
@return ::NC_NOERR No error.
@return ::NC_EGLOBAL Can't use ::NC_GLOBAL with this function.

11
libhdf5/hdf5open.c
View File

@ -1218,7 +1218,16 @@ static int get_quantize_info(NC_VAR_INFO_T *var)
attid = H5Aopen_by_name(datasetid, ".", NC_QUANTIZE_GRANULARBR_ATT_NAME,
H5P_DEFAULT, H5P_DEFAULT);
if (attid > 0)
{
var->quantize_mode = NC_QUANTIZE_GRANULARBR;
}
else
{
attid = H5Aopen_by_name(datasetid, ".", NC_QUANTIZE_BITROUND_ATT_NAME,
H5P_DEFAULT, H5P_DEFAULT);
if (attid > 0)
var->quantize_mode = NC_QUANTIZE_BITROUND;
}
}
/* If there is an attribute, read it for the nsd. */
@ -2307,7 +2316,7 @@ read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid, char *type_name)
* for both global and variable attributes.
*
* @param loc_id HDF5 attribute ID.
* @param att_name Name of the attrigute.
* @param att_name Name of the attribute.
* @param ainfo HDF5 info struct for attribute.
* @param att_data Pointer to an att_iter_info struct, which contains
* pointers to the NC_GRP_INFO_T and (for variable attributes) the

86
libhdf5/hdf5var.c
View File

@ -715,36 +715,54 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
}
/* Remember quantization settings. They will be used when data are
* written. */
* written.
* Code block is identical to one in zvar.c---consider functionalizing */
if (quantize_mode)
{
/* Only two valid mode settings. */
/* Only four valid mode settings. */
if (*quantize_mode != NC_NOQUANTIZE &&
*quantize_mode != NC_QUANTIZE_BITGROOM &&
*quantize_mode != NC_QUANTIZE_GRANULARBR)
*quantize_mode != NC_QUANTIZE_GRANULARBR &&
*quantize_mode != NC_QUANTIZE_BITROUND)
return NC_EINVAL;
if (*quantize_mode == NC_QUANTIZE_BITGROOM || *quantize_mode == NC_QUANTIZE_GRANULARBR)
if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
*quantize_mode == NC_QUANTIZE_GRANULARBR ||
*quantize_mode == NC_QUANTIZE_BITROUND)
{
/* Only float and double types can have quantization. */
if (var->type_info->hdr.id != NC_FLOAT &&
var->type_info->hdr.id != NC_DOUBLE)
return NC_EINVAL;
/* For bitgroom, number of significant digits is required. */
/* All quantization codecs require number of significant digits */
if (!nsd)
return NC_EINVAL;
/* NSD must be in range. */
if (*nsd <= 0)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
return NC_EINVAL;
if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
*quantize_mode == NC_QUANTIZE_GRANULARBR)
{
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
return NC_EINVAL;
}
else if (*quantize_mode == NC_QUANTIZE_BITROUND)
{
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSB)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSB)
return NC_EINVAL;
}
var->nsd = *nsd;
}
@ -812,12 +830,25 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
* error.)
*
* When quantize is turned on, and the number of significant digits
* has been specified, then the netCDF library will quantize according
* to the selected algorithm. BitGroom will apply all zeros or
* all ones (alternating) to bits which are not needed to specify the
* value to the number of significant digits. GranularBR will zero
* more bits than BG, and thus be more compressible and less accurate.
* Both will change the value of the data, and will make it more compressible.
* (NSD) has been specified, then the netCDF library will quantize according
* to the selected algorithm. BitGroom interprets NSD as decimal digits
* will apply all zeros or all ones (alternating) to bits which are not
* needed to specify the value to the number of significant decimal digits.
* BitGroom retain the same number of bits for all values of a variable.
* BitRound (BR) interprets NSD as binary digits (i.e., bits) and keeps the
* the user-specified number of significant bits then rounds the result
* to the nearest representable number according to IEEE rounding rules.
* BG and BR both retain a uniform number of significant bits for all
* values of a variable. Granular BitRound interprest NSD as decimal
* digits. GranularBR determines the number of bits to necessary to
* retain the user-specified number of significant digits individually
* for every value of the variable. GranularBR then applies the BR
* quantization algorithm on a granular, value-by-value, rather than
* uniformly for the entire variable. GranularBR quantizes more bits
* than BG, and is thus more compressive and less accurate than BG.
* BR knows bits and makes no guarantees about decimal precision.
* All quantization algorithms change the values of the data, and make
* it more compressible.
*
* Quantizing the data does not reduce the size of the data on disk,
* but combining quantize with compression will allow for better
@ -829,10 +860,10 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
* size.
*
* Variables which use quantize will have added an attribute with name
* ::NC_QUANTIZE_[ALG_NAME]_ATT_NAME, which will contain the number of
* significant digits. Users should not delete or change this
* attribute. This is the only record that quantize has been applied
* to the data.
* ::NC_QUANTIZE_BITGROOM_ATT_NAME, ::NC_QUANTIZE_GRANULARBR_ATT_NAME,
* or ::NC_QUANTIZE_BITROUND_ATT_NAME that contains the number of
* significant digits. Users should not delete or change this attribute.
* This is the only record that quantize has been applied to the data.
*
* As with the deflate settings, quantize settings may only be
* modified before the first call to nc_enddef(). Once nc_enddef() is
@ -847,10 +878,15 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
* @param ncid File ID.
* @param varid Variable ID. NC_GLOBAL may not be used.
* @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
* @param nsd Number of significant digits. May be any integer from 1
* to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT) or
* ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type ::NC_DOUBLE).
* ::NC_QUANTIZE_BITGROOM, ::NC_QUANTIZE_BITROUND or ::NC_QUANTIZE_GRANULARBR.
* @param nsd Number of significant digits (either decimal or binary).
* May be any integer from 1 to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables
* of type ::NC_FLOAT) or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables
* of type ::NC_DOUBLE) for mode ::NC_QUANTIZE_BITGROOM and mode
* ::NC_QUANTIZE_GRANULARBR. May be any integer from 1 to
* ::NC_QUANTIZE_MAX_FLOAT_NSB (for variables of type ::NC_FLOAT) or
* ::NC_QUANTIZE_MAX_DOUBLE_NSB (for variables of type ::NC_DOUBLE)
* for mode ::NC_QUANTIZE_BITROUND.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.

9
libhdf5/nc4hdf.c
View File

@ -1012,7 +1012,9 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
/* If quantization is in use, write an attribute indicating it, a
* single integer which is the number of significant digits. */
* single integer which is the number of significant digits
* (NSD, for BitGroom and Granular BitRound) or number of significant bits
* (NSB, for BitRound). */
if (var->quantize_mode == NC_QUANTIZE_BITGROOM)
if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITGROOM_ATT_NAME, NC_INT, 1,
&var->nsd, NC_INT, 0)))
@ -1023,6 +1025,11 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
&var->nsd, NC_INT, 0)))
BAIL(retval);
if (var->quantize_mode == NC_QUANTIZE_BITROUND)
if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_BITROUND_ATT_NAME, NC_INT, 1,
&var->nsd, NC_INT, 0)))
BAIL(retval);
/* Write attributes for this var. */
if ((retval = write_attlist(var->att, var->hdr.id, grp)))
BAIL(retval);

4
libnczarr/zsync.c
View File

@ -727,6 +727,10 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist, int isc
if((stat = NCJinsert(jatts,NC_QUANTIZE_GRANULARBR_ATT_NAME,jint))) goto done;
jint = NULL;
break;
case NC_QUANTIZE_BITROUND:
if((stat = NCJinsert(jatts,NC_QUANTIZE_BITROUND_ATT_NAME,jint))) goto done;
jint = NULL;
break;
default: break;
}
}

94
libnczarr/zvar.c
View File

@ -767,34 +767,54 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
}
/* Remember quantization settings. They will be used when data are
* written. */
* written.
* Code block is identical to one in hdf5var.c---consider functionalizing */
if (quantize_mode)
{
/* Only two valid mode settings. */
/* Only four valid mode settings. */
if (*quantize_mode != NC_NOQUANTIZE &&
*quantize_mode != NC_QUANTIZE_BITGROOM)
*quantize_mode != NC_QUANTIZE_BITGROOM &&
*quantize_mode != NC_QUANTIZE_GRANULARBR &&
*quantize_mode != NC_QUANTIZE_BITROUND)
return NC_EINVAL;
if (*quantize_mode == NC_QUANTIZE_BITGROOM)
{
if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
*quantize_mode == NC_QUANTIZE_GRANULARBR ||
*quantize_mode == NC_QUANTIZE_BITROUND)
{
/* Only float and double types can have quantization. */
if (var->type_info->hdr.id != NC_FLOAT &&
var->type_info->hdr.id != NC_DOUBLE)
return NC_EINVAL;
/* For bitgroom, number of significant digits is required. */
/* All quantization codecs require number of significant digits */
if (!nsd)
return NC_EINVAL;
/* NSD must be in range. */
if (*nsd <= 0)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
return NC_EINVAL;
if (*quantize_mode == NC_QUANTIZE_BITGROOM ||
*quantize_mode == NC_QUANTIZE_GRANULARBR)
{
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
return NC_EINVAL;
}
else if (*quantize_mode == NC_QUANTIZE_BITROUND)
{
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSB)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSB)
return NC_EINVAL;
}
var->nsd = *nsd;
}
@ -1018,10 +1038,25 @@ NCZ_def_var_endian(int ncid, int varid, int endianness)
* error.)
*
* When quantize is turned on, and the number of significant digits
* has been specified, then the netCDF library will apply all zeros or
* all ones (alternating) to bits which are not needed to specify the
* value to the number of significant digits. This will change the
* value of the data, but will make it more compressable.
* (NSD) has been specified, then the netCDF library will quantize according
* to the selected algorithm. BitGroom interprets NSD as decimal digits
* will apply all zeros or all ones (alternating) to bits which are not
* needed to specify the value to the number of significant decimal digits.
* BitGroom retain the same number of bits for all values of a variable.
* BitRound (BR) interprets NSD as binary digits (i.e., bits) and keeps the
* the user-specified number of significant bits then rounds the result
* to the nearest representable number according to IEEE rounding rules.
* BG and BR both retain a uniform number of significant bits for all
* values of a variable. Granular BitRound interprest NSD as decimal
* digits. GranularBR determines the number of bits to necessary to
* retain the user-specified number of significant digits individually
* for every value of the variable. GranularBR then applies the BR
* quantization algorithm on a granular, value-by-value, rather than
* uniformly for the entire variable. GranularBR quantizes more bits
* than BG, and is thus more compressive and less accurate than BG.
* BR knows bits and makes no guarantees about decimal precision.
* All quantization algorithms change the values of the data, and make
* it more compressible.
*
* Quantizing the data does not reduce the size of the data on disk,
* but combining quantize with compression will allow for better
@ -1032,12 +1067,11 @@ NCZ_def_var_endian(int ncid, int varid, int endianness)
* compression will result in significant improvent in the final data
* size.
*
* Variables which use quantize will have added an attribute with either the name
* ::NC_QUANTIZE_BITGROOM_ATT_NAME or ::NC_QUANTIZE_GRANULARBR, but in either case
* will contain the number of significant digits.
* Users should not delete or change this
* attribute. This is the only record that quantize has been applied
* to the data.
* Variables which use quantize will have added an attribute with name
* ::NC_QUANTIZE_BITGROOM_ATT_NAME, ::NC_QUANTIZE_GRANULARBR_ATT_NAME,
* or ::NC_QUANTIZE_BITROUND_ATT_NAME that contains the number of
* significant digits. Users should not delete or change this attribute.
* This is the only record that quantize has been applied to the data.
*
* As with the deflate settings, quantize settings may only be
* modified before the first call to nc_enddef(). Once nc_enddef() is
@ -1052,10 +1086,15 @@ NCZ_def_var_endian(int ncid, int varid, int endianness)
* @param ncid File ID.
* @param varid Variable ID. NC_GLOBAL may not be used.
* @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
* @param nsd Number of significant digits. May be any integer from 1
* to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT) or
* ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type ::NC_DOUBLE).
* ::NC_QUANTIZE_BITGROOM, ::NC_QUANTIZE_BITROUND or ::NC_QUANTIZE_GRANULARBR.
* @param nsd Number of significant digits (either decimal or binary).
* May be any integer from 1 to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables
* of type ::NC_FLOAT) or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables
* of type ::NC_DOUBLE) for mode ::NC_QUANTIZE_BITGROOM and mode
* ::NC_QUANTIZE_GRANULARBR. May be any integer from 1 to
* ::NC_QUANTIZE_MAX_FLOAT_NSB (for variables of type ::NC_FLOAT) or
* ::NC_QUANTIZE_MAX_DOUBLE_NSB (for variables of type ::NC_DOUBLE)
* for mode ::NC_QUANTIZE_BITROUND.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
@ -1092,6 +1131,9 @@ NCZ_ensure_quantizer(int ncid, NC_VAR_INFO_T* var)
} else if(NCZ_get_att(ncid,var->hdr.id,NC_QUANTIZE_GRANULARBR_ATT_NAME,&nsd,NC_INT)==NC_NOERR) {
var->quantize_mode = NC_QUANTIZE_GRANULARBR;
var->nsd = nsd;
} else if(NCZ_get_att(ncid,var->hdr.id,NC_QUANTIZE_BITROUND_ATT_NAME,&nsd,NC_INT)==NC_NOERR) {
var->quantize_mode = NC_QUANTIZE_BITROUND;
var->nsd = nsd;
} else {
var->quantize_mode = NC_NOQUANTIZE;
var->nsd = 0;

134
libsrc4/nc4var.c
View File

@ -21,14 +21,6 @@
/** @internal Default size for unlimited dim chunksize. */
#define DEFAULT_1D_UNLIM_SIZE (4096)
/** @internal Minimum number of explicit significand bits to preserve
* when zeroing/bit-masking floating point values. Codes will preserve
* at least two explicit bits, IEEE significand representation
* contains one implicit bit Thus preserve a least three bits which is
* approximately one sigificant decimal digit Used in
* nco_ppc_bitmask() and nco_ppc_bitmask_scl() */
#define NCO_PPC_BIT_XPL_NBR_MIN 2
/* Define log_e for 10 and 2. Prefer constants defined in math.h,
* however, GCC environments can have hard time defining M_LN10/M_LN2
* despite finding math.h */
@ -41,13 +33,15 @@
/** Used in quantize code. Number of explicit bits in significand for
* floats. Bits 0-22 of SP significands are explicit. Bit 23 is
* implicitly 1. */
* implicitly 1. Currently redundant with NC_QUANTIZE_MAX_FLOAT_NSB
* and with limits.h/climit (FLT_MANT_DIG-1) */
#define BIT_XPL_NBR_SGN_FLT (23)
/** Used in quantize code. Number of explicit bits in significand for
* doubles. Bits 0-52 of DP significands are explicit. Bit 53 is
* implicitly 1. */
#define BIT_XPL_NBR_SGN_DBL (53)
* doubles. Bits 0-51 of DP significands are explicit. Bit 52 is
* implicitly 1. Currently redundant with NC_QUANTIZE_MAX_DOUBLE_NSB
* and with limits.h/climit (DBL_MANT_DIG-1) */
#define BIT_XPL_NBR_SGN_DBL (52)
/** Pointer union for floating point and bitmask types. */
typedef union { /* ptr_unn */
@ -503,8 +497,8 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* values that overflow the type.
*
* This function applies quantization to float and double data, if
* desired. The code to do this is derived from the bitgroom filter in
* the CCR project (see
* desired. The code to do this is derived from the corresponding
* filter in the CCR project (e.g.,
* https://github.com/ccr/ccr/blob/master/hdf5_plugins/BITGROOM/src/H5Zbitgroom.c).
*
* @param src Pointer to source of data.
@ -515,11 +509,11 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* @param range_error Pointer that gets 1 if there was a range error.
* @param fill_value The fill value.
* @param strict_nc3 Non-zero if strict model in effect.
* @param quantize_mode May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM or ::NC_QUANTIZE_GRANULARBR.
* @param nsd Number of significant diggits for quantizize. Ignored
* unless quantize_mode is ::NC_QUANTIZE_BITGROOM or
* ::NC_QUANTIZE_GRANULARBR.
* @param quantize_mode May be ::NC_NOQUANTIZE, ::NC_QUANTIZE_BITGROOM,
* ::NC_QUANTIZE_GRANULARBR, or ::NC_QUANTIZE_BITROUND.
* @param nsd Number of significant digits for quantize. Ignored
* unless quantize_mode is ::NC_QUANTIZE_BITGROOM,
* ::NC_QUANTIZE_GRANULARBR, or ::NC_QUANTIZE_BITROUND
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADTYPE Type not found.
@ -576,42 +570,53 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
/* If quantize is in use, set up some values. Quantize can only be
* used when the destination type is NC_FLOAT or NC_DOUBLE. */
if (quantize_mode != NC_NOQUANTIZE)
{
{
assert(dest_type == NC_FLOAT || dest_type == NC_DOUBLE);
/* Parameters shared by both BitGroom and GranularBR */
/* Parameters shared by all quantization codecs */
if (dest_type == NC_FLOAT)
{
{
/* Determine the fill value. */
if (fill_value)
mss_val_cmp_flt = *(float *)fill_value;
mss_val_cmp_flt = *(float *)fill_value;
else
mss_val_cmp_flt = NC_FILL_FLOAT;
mss_val_cmp_flt = NC_FILL_FLOAT;
}
}
else
{
{
/* Determine the fill value. */
if (fill_value)
mss_val_cmp_dbl = *(double *)fill_value;
mss_val_cmp_dbl = *(double *)fill_value;
else
mss_val_cmp_dbl = NC_FILL_DOUBLE;
mss_val_cmp_dbl = NC_FILL_DOUBLE;
}
}
/* Parameters BitGroom needs to be set once */
if (quantize_mode == NC_QUANTIZE_BITGROOM)
/* Set parameters used by BitGroom and BitRound here, outside value loop.
Equivalent parameters used by GranularBR are set inside value loop,
since keep bits and thus masks can change for every value. */
if (quantize_mode == NC_QUANTIZE_BITGROOM ||
quantize_mode == NC_QUANTIZE_BITROUND )
{
/* How many bits to preserve? Being conservative, we round up the
* exact binary digits of precision. Add one because the first bit
* is implicit not explicit but corner cases prevent our taking
* advantage of this. */
prc_bnr_xpl_rqr = (unsigned short)ceil(nsd * bit_per_dgt) + 1;
if (dest_type == NC_DOUBLE)
prc_bnr_xpl_rqr++; /* Seems necessary for double-precision
* ppc=array(1.234567,1.0e-6,$dmn) */
if (quantize_mode == NC_QUANTIZE_BITGROOM){
/* BitGroom interprets nsd as number of significant decimal digits
* Must convert that to number of significant bits to preserve
* How many bits to preserve? Being conservative, we round up the
* exact binary digits of precision. Add one because the first bit
* is implicit not explicit but corner cases prevent our taking
* advantage of this. */
prc_bnr_xpl_rqr = (unsigned short)ceil(nsd * bit_per_dgt) + 1;
}else if (quantize_mode == NC_QUANTIZE_BITROUND){
/* BitRound interprets nsd as number of significant binary digits (bits) */
prc_bnr_xpl_rqr = nsd;
}
if (dest_type == NC_FLOAT)
{
@ -622,13 +627,17 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
msk_f32_u32_zro = 0u; /* Zero all bits */
msk_f32_u32_zro = ~msk_f32_u32_zro; /* Turn all bits to ones */
/* Bit Shave mask for AND: Left shift zeros into bits to be
/* BitShave mask for AND: Left shift zeros into bits to be
* rounded, leave ones in untouched bits. */
msk_f32_u32_zro <<= bit_xpl_nbr_zro;
/* Bit Set mask for OR: Put ones into bits to be set, zeros in
/* BitSet mask for OR: Put ones into bits to be set, zeros in
* untouched bits. */
msk_f32_u32_one = ~msk_f32_u32_zro;
/* BitRound mask for ADD: Set one bit: the MSB of LSBs */
msk_f32_u32_hshv=msk_f32_u32_one & (msk_f32_u32_zro >> 1);
}
else
{
@ -638,20 +647,23 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
msk_f64_u64_zro = 0ul; /* Zero all bits. */
msk_f64_u64_zro = ~msk_f64_u64_zro; /* Turn all bits to ones. */
/* Bit Shave mask for AND: Left shift zeros into bits to be
/* BitShave mask for AND: Left shift zeros into bits to be
* rounded, leave ones in untouched bits. */
msk_f64_u64_zro <<= bit_xpl_nbr_zro;
/* Bit Set mask for OR: Put ones into bits to be set, zeros in
/* BitSet mask for OR: Put ones into bits to be set, zeros in
* untouched bits. */
msk_f64_u64_one =~ msk_f64_u64_zro;
/* BitRound mask for ADD: Set one bit: the MSB of LSBs */
msk_f64_u64_hshv = msk_f64_u64_one & (msk_f64_u64_zro >> 1);
}
}
} /* endif quantize */
} /* endif quantize */
/* OK, this is ugly. If you can think of anything better, I'm open
to suggestions!
@ -1414,7 +1426,7 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
{
if (dest_type == NC_FLOAT)
{
/* Bit-Groom: alternately shave and set LSBs */
/* BitGroom: alternately shave and set LSBs */
op1.fp = (float *)dest;
u32_ptr = op1.ui32p;
for (idx = 0L; idx < len; idx += 2L)
@ -1426,7 +1438,7 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
else
{
/* Bit-Groom: alternately shave and set LSBs. */
/* BitGroom: alternately shave and set LSBs. */
op1.dp = (double *)dest;
u64_ptr = op1.ui64p;
for (idx = 0L; idx < len; idx += 2L)
@ -1438,6 +1450,34 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
} /* endif BitGroom */
if (quantize_mode == NC_QUANTIZE_BITROUND)
{
if (dest_type == NC_FLOAT)
{
/* BitRound: Quantize to user-specified NSB with IEEE-rounding */
op1.fp = (float *)dest;
u32_ptr = op1.ui32p;
for (idx = 0L; idx < len; idx++){
if (op1.fp[idx] != mss_val_cmp_flt){
u32_ptr[idx] += msk_f32_u32_hshv; /* Add 1 to the MSB of LSBs, carry 1 to mantissa or even exponent */
u32_ptr[idx] &= msk_f32_u32_zro; /* Shave it */
}
}
}
else
{
/* BitRound: Quantize to user-specified NSB with IEEE-rounding */
op1.dp = (double *)dest;
u64_ptr = op1.ui64p;
for (idx = 0L; idx < len; idx++){
if (op1.dp[idx] != mss_val_cmp_dbl){
u64_ptr[idx] += msk_f64_u64_hshv; /* Add 1 to the MSB of LSBs, carry 1 to mantissa or even exponent */
u64_ptr[idx] &= msk_f64_u64_zro; /* Shave it */
}
}
}
} /* endif BitRound */
if (quantize_mode == NC_QUANTIZE_GRANULARBR)
{
if (dest_type == NC_FLOAT)

31
nc_test4/tst_quantize.c
View File

@ -10,6 +10,7 @@
Dennis Heimbigner, 1/16/22
*/
#include <math.h> /* Define fabs(), powf(), round() */
#include <nc_tests.h>
#include "err_macros.h"
#include "netcdf.h"
@ -66,7 +67,7 @@ pd(double myd)
uint64_t u;
} du;
du.d = myd;
sprintf(pf_str, "0x%lx", (unsigned long)du.u);
sprintf(pf_str, "0x%llx", (unsigned long long)du.u);
return pf_str;
}
@ -116,10 +117,10 @@ main(int argc, char **argv)
if (nc_def_var_quantize(ncid, NC_GLOBAL, NC_QUANTIZE_BITGROOM, NSD_3) != NC_EGLOBAL) ERR;
if (nc_def_var_quantize(ncid, varid2 + 1, NC_QUANTIZE_BITGROOM, NSD_3) != NC_ENOTVAR) ERR;
/* Invalid values. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_GRANULARBR + 1, NSD_3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITROUND + 1, NSD_3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, -1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NC_QUANTIZE_MAX_FLOAT_NSD + 1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_GRANULARBR + 1, 3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITROUND + 1, 3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, -1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NC_QUANTIZE_MAX_DOUBLE_NSD + 1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, 0) != NC_EINVAL) ERR;
@ -292,8 +293,8 @@ main(int argc, char **argv)
/* printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[0], fout.u, float_data[0], fin.u); */
if (fin.u != 0x3f8e3000) ERR;
/* printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", */
/* double_data[0], dfout.u, double_data[0], dfin.u); */
/* printf ("\ndouble_data: %15g : 0x%16llx double_data_in: %15g : 0x%llx\n", */
/* double_data[0], dfout.u, double_data[0], dfin.u);*/
if (dfin.u != 0x3ff1c60000000000) ERR;
/* Close the file again. */
@ -352,7 +353,7 @@ main(int argc, char **argv)
/* printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[0], fout.u, float_data[0], fin.u); */
if (fin.u != 0x3f8e3000) ERR;
/* printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", */
/* printf ("\ndouble_data: %15g : 0x%16llx double_data_in: %15g : 0x%llx\n", */
/* double_data[0], dfout.u, double_data[0], dfin.u); */
if (dfin.u != 0x3ff1c60000000000) ERR;
@ -433,8 +434,8 @@ main(int argc, char **argv)
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
/*printf("double_data: %15g : 0x%16llx double_data_in: %15g : 0x%16llx\n",*/
/* double_data[x], dfout.u, double_data[x], dfin.u);*/
if (dfin.u != double_xpect[x].u) ERR;
}
@ -549,7 +550,7 @@ main(int argc, char **argv)
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU dfout;
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x3f800fff;
@ -584,8 +585,8 @@ main(int argc, char **argv)
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
/* printf("double_data: %15g : 0x%16llx double_data_in: %15g : 0x%16llx\n",*/
/* double_data[x], dfout.u, double_data[x], dfin.u);*/
if (dfin.u != double_xpect[x].u) ERR;
}
@ -872,8 +873,8 @@ main(int argc, char **argv)
dfin.d = double_data_in[x];
/* printf ("%d float_data_in : %08.8f : 0x%x expected %08.8f : 0x%x\n", */
/* x, float_data_in[x], fin.u, xpect[x].f, xpect[x].u); */
/* printf ("%d double_data_in : %15g : 0x%lx expected %15g : 0x%lx\n", */
/* x, double_data_in[x], dfin.u, double_xpect[x].d, double_xpect[x].u); */
/* printf ("%d double_data_in : %15g : 0x%llx expected %15g : 0x%llx\n",*/
/* x, double_data_in[x], dfin.u, double_xpect[x].d, double_xpect[x].u);*/
if (fin.u != xpect[x].u)
ERR;
if (dfin.u != double_xpect[x].u)
@ -956,9 +957,9 @@ main(int argc, char **argv)
for (i = 0; i < DIM_LEN_SIMPLE; i++)
{
if (abs(float_data_in[i] - float_data[i]) > EPSILON)
if (fabs(float_data_in[i] - float_data[i]) > EPSILON)
ERR;
if (abs(double_data_in[i] - double_data[i]) > EPSILON)
if (fabs(double_data_in[i] - double_data[i]) > EPSILON)
ERR;
}

2
ncgen/genbin.c
View File

@ -267,7 +267,7 @@ genbin_definespecialattributes(Symbol* var)
}
CHECK_ERR(stat);
}
if(special->flags & (_QUANTIZEBG_FLAG | _QUANTIZEBR_FLAG)) {
if(special->flags & (_QUANTIZEBG_FLAG | _QUANTIZEGBR_FLAG | _QUANTIZEBR_FLAG)) {
stat = nc_def_var_quantize(var->container->nc_id,
var->nc_id, special->_Quantizer, special->_NSD);
CHECK_ERR(stat);

2
ncgen/genc.c
View File

@ -539,7 +539,7 @@ genc_definespecialattributes(Symbol* vsym)
codelined(1,"CHECK_ERR(stat);");
}
}
if(special->flags & (_QUANTIZEBG_FLAG | _QUANTIZEBR_FLAG)) {
if(special->flags & (_QUANTIZEBG_FLAG | _QUANTIZEGBR_FLAG)) {
const char* alg = NULL;
switch(special->_Quantizer) {
case NC_QUANTIZE_BITGROOM: alg = "NC_QUANTIZE_BITGROOM";

3
ncgen/ncgen.h
View File

@ -81,7 +81,8 @@ various C global variables
#define _FILTER_FLAG 0x1000
#define _CODECS_FLAG 0x2000
#define _QUANTIZEBG_FLAG 0x4000
#define _QUANTIZEBR_FLAG 0x8000
#define _QUANTIZEGBR_FLAG 0x8000
#define _QUANTIZEBR_FLAG 0x10000
extern struct Specialtoken {
char* name;

3
ncgen/ncgen.l
View File

@ -139,6 +139,7 @@ struct Specialtoken specials[] = {
{"_Filter",_FILTER,_FILTER_FLAG},
{"_Codecs",_CODECS,_CODECS_FLAG},
{"_QuantizeBitGroomNumberOfSignificantDigits",_QUANTIZEBG,_QUANTIZEBG_FLAG},
{"_QuantizeGranularBitRoundNumberOfSignificantDigits",_QUANTIZEGBR,_QUANTIZEGBR_FLAG},
{"_QuantizeBitRoundNumberOfSignificantDigits",_QUANTIZEBR,_QUANTIZEBR_FLAG},
{NULL,0} /* null terminate */
};
@ -218,7 +219,7 @@ NUMBER [+-]?[0-9][0-9]*[Uu]?([BbSs]|[Ll]|[Ll][Ll])?
DBLNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[LlDd]?|[+-]?[0-9]*{exp}[LlDd]?
FLTNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[Ff]|[+-]?[0-9]*{exp}[Ff]
SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"|"_Codecs"|"_QuantizeBitgroomNumberOfSignificantDIgits"|"_QuantizeGranularBitRoundNumberOfSignificantDigits"
SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"|"_Codecs"|"_QuantizeBitgroomNumberOfSignificantDIgits"|"_QuantizeGranularBitRoundNumberOfSignificantDigits"|"_QuantizeBitRoundNumberOfSignificantDigits"
USASCII [\x01-\x7F]

11
ncgen/ncgen.y
View File

@ -217,6 +217,7 @@ NCConstant* constant;
_FILTER
_CODECS
_QUANTIZEBG
_QUANTIZEGBR
_QUANTIZEBR
DATASETID
@ -776,6 +777,8 @@ attrdecl:
{$$ = makespecial(_CODECS_FLAG,$1,NULL,(void*)$5,ISCONST);}
| ambiguous_ref ':' _QUANTIZEBG '=' constint
{$$ = makespecial(_QUANTIZEBG_FLAG,$1,NULL,(void*)$5,ISCONST);}
| ambiguous_ref ':' _QUANTIZEGBR '=' constint
{$$ = makespecial(_QUANTIZEGBR_FLAG,$1,NULL,(void*)$5,ISCONST);}
| ambiguous_ref ':' _QUANTIZEBR '=' constint
{$$ = makespecial(_QUANTIZEBR_FLAG,$1,NULL,(void*)$5,ISCONST);}
| ambiguous_ref ':' _NOFILL '=' constbool
@ -1247,6 +1250,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
case _SUPERBLOCK_FLAG:
case _DEFLATE_FLAG:
case _QUANTIZEBG_FLAG:
case _QUANTIZEGBR_FLAG:
case _QUANTIZEBR_FLAG:
tmp = nullconst();
tmp->nctype = NC_INT;
@ -1349,9 +1353,14 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
special->_NSD = idata;
special->flags |= _QUANTIZEBG_FLAG;
break;
case _QUANTIZEBR_FLAG:
case _QUANTIZEGBR_FLAG:
special->_Quantizer = NC_QUANTIZE_GRANULARBR;
special->_NSD = idata;
special->flags |= _QUANTIZEGBR_FLAG;
break;
case _QUANTIZEBR_FLAG:
special->_Quantizer = NC_QUANTIZE_BITROUND;
special->_NSD = idata;
special->flags |= _QUANTIZEBR_FLAG;
break;
case _SHUFFLE_FLAG:

860
ncgen/ncgenl.c
View File

File diff suppressed because it is too large Load Diff

1309
ncgen/ncgeny.c
View File

File diff suppressed because it is too large Load Diff

7
ncgen/ncgeny.h
View File

@ -108,8 +108,9 @@ extern int ncgdebug;
_FILTER = 309, /* _FILTER */
_CODECS = 310, /* _CODECS */
_QUANTIZEBG = 311, /* _QUANTIZEBG */
_QUANTIZEBR = 312, /* _QUANTIZEBR */
DATASETID = 313 /* DATASETID */
_QUANTIZEGBR = 312, /* _QUANTIZEGBR */
_QUANTIZEBR = 313, /* _QUANTIZEBR */
DATASETID = 314 /* DATASETID */
};
typedef enum yytokentype yytoken_kind_t;
#endif
@ -127,7 +128,7 @@ int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant* constant;
#line 131 "ncgeny.h"
#line 132 "ncgeny.h"
};
typedef union YYSTYPE YYSTYPE;