mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into zarrs3.dmh

Browse Source
This commit is contained in:
Dennis Heimbigner 2021-10-07 15:10:03 -06:00
commit 289103d2b1
42 changed files with 1940 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
CMakeLists.txt
View File

@ -37,7 +37,7 @@ SET(PACKAGE_VERSION ${VERSION})
# Version of the dispatch table. This must match the value in
# configure.ac.
SET(NC_DISPATCH_VERSION 3)
SET(NC_DISPATCH_VERSION 4)
# Get system configuration, Use it to determine osname, os release, cpu. These
# will be used when committing to CDash.
@ -745,6 +745,11 @@ IF(USE_HDF5)
# HDF5_C_LIBRARY, HDF5_HL_LIBRARY and HDF5_LIBRARIES.
###
IF(MSVC)
####
# Environmental variables in Windows when using MSVC
# are a hot mess between versions.
####
##
# HDF5 1.8.15 defined HDF5_LIBRARIES.
##
@ -764,16 +769,23 @@ IF(USE_HDF5)
ENDIF(${HDF5_VERSION} VERSION_GREATER "1.8.15")
ELSE(MSVC)
####
# Environmental variables in Windows when using MSVC
# are a hot mess between versions.
####
# Depending on the install, either HDF5_hdf_library or
# HDF5_C_LIBRARIES may be defined. We must check for either.
IF(HDF5_C_LIBRARIES AND NOT HDF5_hdf5_LIBRARY)
SET(HDF5_hdf5_LIBRARY ${HDF5_C_LIBRARIES})
ENDIF()
# Some versions of FIND_PACKAGE set HDF5_C_LIBRARIES, but not HDF5_C_LIBRARY
# We use HDF5_C_LIBRARY below, so need to make sure it is set.
IF(HDF5_C_LIBRARIES AND NOT HDF5_C_LIBRARY)
SET(HDF5_C_LIBRARY ${HDF5_C_LIBRARIES})
ENDIF()
# Same issue as above...
IF(HDF5_HL_LIBRARIES AND NOT HDF5_HL_LIBRARY)
SET(HDF5_HL_LIBRARY ${HDF5_HL_LIBRARIES})
ENDIF()
ENDIF(MSVC)
IF(NOT HDF5_C_LIBRARY)
@ -1507,6 +1519,9 @@ ENDIF()
# Always enable DISKLESS
OPTION(ENABLE_DISKLESS "Enable in-memory files" ON)
# Always enable quantization.
OPTION(ENABLE_QUANTIZE "Enable variable quantization" ON)
# By default, MSVC has a stack size of 1000000.
# Allow a user to override this.
IF(MSVC)
@ -2301,6 +2316,7 @@ is_enabled(ENABLE_NCZARR HAS_NCZARR)
is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS)
is_enabled(ENABLE_MULTIFILTERS HAS_MULTIFILTERS)
is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS)
is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE)
is_enabled(ENABLE_LOGGING HAS_LOGGING)
is_enabled(ENABLE_FILTER_TESTING DO_FILTER_TESTS)
is_enabled(ENABLE_BLOSC HAS_BLOSC)

3
configure.ac
View File

@ -1708,6 +1708,7 @@ AC_SUBST(HAS_NCZARR,[$enable_nczarr])
AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
AC_SUBST([HAS_QUANTIZE],[yes])
AC_SUBST(HAS_LOGGING,[$enable_logging])
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
AC_SUBST(HAVE_BLOSC,[$enable_blosc])
@ -1792,7 +1793,7 @@ AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes])
# dispatch table to submit. If this is changed, make sure the value in
# CMakeLists.txt also changes to match.
AC_SUBST([NC_DISPATCH_VERSION], [3])
AC_SUBST([NC_DISPATCH_VERSION], [4])
AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.])
#####

4
include/hdf5internal.h
View File

@ -202,6 +202,10 @@ int NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, cons
int NC4_hdf5_filter_freelist(NC_VAR_INFO_T* var);
int NC4_hdf5_find_missing_filter(NC_VAR_INFO_T* var, unsigned int* idp);
/* Add an attribute to the attribute list. */
int nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
size_t len, const void *data, nc_type mem_type, int force);
/* Support functions for provenance info (defined in nc4hdf.c) */
extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hdf.c*/
extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/

6
include/nc4dispatch.h
View File

@ -251,6 +251,12 @@ extern "C" {
EXTERNL int
NC4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
EXTERNL int
NC4_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd);
EXTERNL int
NC4_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp);
#if defined(__cplusplus)
}
#endif

9
include/nc4internal.h
View File

@ -205,7 +205,9 @@ typedef struct NC_VAR_INFO
nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */
size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
float chunk_cache_preemption; /**< Chunk cache preemption policy. */
float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */
int nsd; /**< Number of significant digits if quantization is used, 0 if not. */
void *format_var_info; /**< Pointer to any binary format info. */
void* filters; /**< Record of the list of filters to be applied to var data; format dependent */
} NC_VAR_INFO_T;
@ -343,8 +345,9 @@ extern int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep)
/* These functions convert between netcdf and HDF5 types. */
extern int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len);
extern int nc4_convert_type(const void *src, void *dest, const nc_type src_type,
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3);
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3, int quantize_mode,
int nsd);
/* These functions do HDF5 things. */
extern int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);

25
include/netcdf.h
View File

@ -324,6 +324,21 @@ there. */
#define NC_MIN_DEFLATE_LEVEL 0 /**< Minimum deflate level. */
#define NC_MAX_DEFLATE_LEVEL 9 /**< Maximum deflate level. */
#define NC_NOQUANTIZE 0 /**< No quantization in use. */
#define NC_QUANTIZE_BITGROOM 1 /**< Use bitgroom quantization. */
/** When quantization is used for a variable, an attribute of this
* name is added. */
#define NC_QUANTIZE_ATT_NAME "_QuantizeBitgroomNumberOfSignificantDigits"
/** For quantization, the allowed value of number of significant
* digits for float. */
#define NC_QUANTIZE_MAX_FLOAT_NSD (7)
/** For quantization, the allowed value of number of significant
* digits for double. */
#define NC_QUANTIZE_MAX_DOUBLE_NSD (15)
/** The netcdf version 3 functions all return integer error status.
* These are the possible values, in addition to certain values from
* the system errno.h.
@ -854,6 +869,16 @@ nc_get_varm(int ncid, int varid, const size_t *startp,
/* Extra netcdf-4 stuff. */
/* Set quantization settings for a variable. Quantizing data improves
* later compression. Must be called after nc_def_var and before
* nc_enddef. */
EXTERNL int
nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd);
/* Find out quantization settings of a var. */
EXTERNL int
nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp);
/* Set compression settings for a variable. Lower is faster, higher is
* better. Must be called after nc_def_var and before nc_enddef. */
EXTERNL int

7
include/netcdf_dispatch.h.in
View File

@ -147,6 +147,9 @@ struct NC_Dispatch
/* Version 3 Replace filteractions with more specific functions */
int (*inq_var_filter_ids)(int ncid, int varid, size_t* nfilters, unsigned int* filterids);
int (*inq_var_filter_info)(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params);
/* Version 4 Add quantization. */
int (*def_var_quantize)(int ncid, int varid, int quantize_mode, int nsd);
int (*inq_var_quantize)(int ncid, int varid, int *quantize_modep, int *nsdp);
};
#if defined(__cplusplus)
@ -223,7 +226,9 @@ extern "C" {
EXTERNL int NC_NOTNC4_inq_typeids(int, int *, int *);
EXTERNL int NC_NOTNC4_inq_user_type(int, nc_type, char *, size_t *,
nc_type *, size_t *, int *);
EXTERNL int NC_NOTNC4_def_var_quantize(int, int, int, int);
EXTERNL int NC_NOTNC4_inq_var_quantize(int, int, int *, int *);
/* These functions are for dispatch layers that don't implement
* the enhanced model, but want to succeed anyway.
* They return NC_NOERR plus properly set the out parameters.

1
include/netcdf_meta.h.in
View File

@ -63,5 +63,6 @@
#define NC_HAS_NCZARR @NC_HAS_NCZARR@ /*!< Parallel I/O with filter support. */
#define NC_HAS_MULTIFILTERS @NC_HAS_MULTIFILTERS@ /*!< Nczarr support. */
#define NC_HAS_LOGGING @NC_HAS_LOGGING@ /*!< Logging support. */
#define NC_HAS_QUANTIZE @NC_HAS_QUANTIZE@
#endif

3
libdap2/ncd2dispatch.c
View File

@ -178,6 +178,9 @@ NCD2_get_var_chunk_cache,
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
};
const NC_Dispatch* NCD2_dispatch_table = NULL; /* moved here from ddispatch.c */

3
libdap4/ncd4dispatch.c
View File

@ -972,4 +972,7 @@ NCD4_get_var_chunk_cache,
NC_NOTNC4_inq_var_filter_ids,
NC_NOTNC4_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
};

35
libdispatch/dnotnc4.c
View File

@ -19,6 +19,41 @@
#include "ncdispatch.h"
#include "nc4internal.h"
/**
* @internal Not implemented in some dispatch tables
*
* @param ncid Ignored.
* @param varid Ignored.
* @param quantize_mode Ignored.
* @param nsd Ignored.
*
* @return ::NC_ENOTNC4 Not implemented for a dispatch table
* @author Ed Hartnett
*/
int
NC_NOTNC4_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
{
return NC_ENOTNC4;
}
/**
* @internal Not implemented in some dispatch tables
*
* @param ncid Ignored.
* @param varid Ignored.
* @param quantize_modep Ignored.
* @param nsdp Ignored.
*
* @return ::NC_ENOTNC4 Not implemented for a dispatch table
* @author Ed Hartnett
*/
int
NC_NOTNC4_inq_var_quantize(int ncid, int varid, int *quantize_modep,
int *nsdp)
{
return NC_ENOTNC4;
}
/**
* @internal Not implemented in some dispatch tables
*

94
libdispatch/dvar.c
View File

@ -461,6 +461,100 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
return ncp->dispatch->def_var_deflate(ncid,varid,shuffle,deflate,deflate_level);
}
/**
Turn on quantization for a variable.
The data data are quantized by setting unneeded bits alternately to
1/0, so that they may compress well. Quantization is lossy (data
are irretrievably altered), and it improves the compression ratio
provided by a subsequent lossless compression filter. Quantization
alone will not reduce the size of the data - lossless compression
like zlib must also be used (see nc_def_var_deflate()).
Producers of large datasets may find that using quantize with
compression will result in significant improvent in the final data
size.
This data quantization used the bitgroom algorithm. A notable
feature of BitGroom is that the data it processes remain in IEEE754
format after quantization. Therefore the BitGroom algorithm does
nothing when data are read.
Quantization is only available for variables of type NC_FLOAT or
NC_DOUBLE. Attempts to set quantization for other variable
types return an error (NC_EINVAL).
Variables which use quantize will have added an attribute with name
::NC_QUANTIZE_ATT_NAME, which will contain the number of
significant digits. Users should not delete or change this
attribute. This is the only record that quantize has been applied
to the data.
Quantization is not applied to values equal to the value of the
_FillValue attribute, if any. If the _FillValue attribute is not
set, then quantization is not applied to values matching the
default fill value.
Quantization may be applied to scalar variables.
When type conversion takes place during a write, the it occurs
before quantization is applied. For example, if nc_put_var_double()
is called on a variable of type NC_FLOAT, which has quantizze
turned on, then the data are first converted from dounle to float,
then quantization is applied to the float values.
As with the deflate settings, quantize settings may only be
modified before the first call to nc_enddef(). Once nc_enddef() is
called for the file, quantize settings for any variable in the file
may not be changed.
Use of quantization is fully backwards compatible with existing
versions and packages that can read compressed netCDF data. A
variable which has been quantized is readable to older versions of
the netCDF libraries, and to netCDF-Java.
For more information about quantization and the bitgroom filter, see
Zender, C. S. (2016), Bit Grooming: Statistically accurate
precision-preserving quantization with compression, evaluated in
the netCDF Operators (NCO, v4.4.8+), Geosci. Model Dev., 9,
3199-3211, doi:10.5194/gmd-9-3199-2016 Retrieved on Sep 21, 2020
from
https://www.researchgate.net/publication/301575383_Bit_Grooming_Statistically_accurate_precision-preserving_quantization_with_compression_evaluated_in_the_netCDF_Operators_NCO_v448.
@param ncid File ID.
@param varid Variable ID. ::NC_GLOBAL may not be used.
@param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
::NC_QUANTIZE_BITGROOM.
@param nsd Number of significant digits. May be any integer from 1
to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT)
or ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type
::NC_DOUBLE). Ignored if quantize_mode = NC_NOQUANTIZE.
@return ::NC_NOERR No error.
@return ::NC_EGLOBAL Can't use ::NC_GLOBAL with this function.
@return ::NC_EBADID Bad ncid.
@return ::NC_ENOTVAR Invalid variable ID.
@return ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is
not netCDF-4/HDF5.
@return ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3
netcdf-4 file.
@return ::NC_ELATEDEF Too late to change settings for this variable.
@return ::NC_EINVAL Invalid input.
@author Charlie Zender, Ed Hartnett
*/
int
nc_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
/* Using NC_GLOBAL is illegal. */
if (varid == NC_GLOBAL) return NC_EGLOBAL;
return ncp->dispatch->def_var_quantize(ncid,varid,quantize_mode,nsd);
}
/**
Set checksum for a var.

30
libdispatch/dvarinq.c
View File

@ -527,6 +527,36 @@ nc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
);
}
/** @ingroup variables
* Learn whether BitGroom quantization is on for a variable, and, if so,
* the NSD setting.
*
* @param ncid File ID.
* @param varid Variable ID. Must not be NC_GLOBAL.
* @param quantize_modep Pointer that gets a 0 if BitGroom is not in
* use for this var, and a 1 if it is. Ignored if NULL.
* @param nsdp Pointer that gets the NSD setting (from 1 to 15), if
* BitGroom is in use. Ignored if NULL.
*
* @return 0 for success, error code otherwise.
* @author Charlie Zender, Ed Hartnett
*/
int
nc_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_quantize);
/* Using NC_GLOBAL is illegal. */
if (varid == NC_GLOBAL) return NC_EGLOBAL;
return ncp->dispatch->inq_var_quantize(ncid, varid,
quantize_modep, nsdp);
}
/** \ingroup variables
Find the endianness of a variable.

2
libhdf4/hdf4var.c
View File

@ -88,7 +88,7 @@ NC_HDF4_get_vara(int ncid, int varid, const size_t *startp,
if (var->type_info->hdr.id != memtype)
{
if ((retval = nc4_convert_type(data, ip, var->type_info->hdr.id, memtype, nelem,
&range_error, NULL, 0)))
&range_error, NULL, 0, NC_NOQUANTIZE, 0)))
return retval;
free(data);
if (range_error)

3
libhdf5/hdf5attr.c
View File

@ -716,7 +716,8 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Data types are like religions, in that one can convert. */
if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,
len, &range_error, NULL,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
}
}

4
libhdf5/hdf5dispatch.c
View File

@ -105,6 +105,10 @@ static const NC_Dispatch HDF5_dispatcher = {
NC4_hdf5_inq_var_filter_ids,
NC4_hdf5_inq_var_filter_info,
NC4_def_var_quantize,
NC4_inq_var_quantize,
};
const NC_Dispatch* HDF5_dispatch_table = NULL; /* moved here from ddispatch.c */

3
libhdf5/hdf5internal.c
View File

@ -928,7 +928,10 @@ nc4_hdf5_find_grp_var_att(int ncid, int varid, const char *name, int attnum,
/* Give the people what they want. */
if (norm_name)
{
strncpy(norm_name, my_norm_name, NC_MAX_NAME);
norm_name[NC_MAX_NAME] = 0;
}
if (h5)
*h5 = my_h5;
if (grp)

44
libhdf5/hdf5open.c
View File

@ -1155,6 +1155,46 @@ static int get_fill_info(hid_t propid, NC_VAR_INFO_T *var)
return NC_NOERR;
}
/**
* @internal Learn if quantize has been applied to this var. If so,
* find the mode and the number of significant digit settings.
*
* @param var Pointer to NC_VAR_INFO_T for this variable.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENOMEM Out of memory.
* @return ::NC_EHDFERR HDF5 returned error.
* @author Dennis Heimbigner, Ed Hartnett
*/
static int get_quantize_info(NC_VAR_INFO_T *var)
{
hid_t attid;
hid_t datasetid;
/* Try to open an attribute of the correct name for quantize
* info. */
datasetid = ((NC_HDF5_VAR_INFO_T *)var->format_var_info)->hdf_datasetid;
attid = H5Aopen_by_name(datasetid, ".", NC_QUANTIZE_ATT_NAME,
H5P_DEFAULT, H5P_DEFAULT);
/* If there is an attribute, read it for the nsd. */
if (attid > 0)
{
var->quantize_mode = NC_QUANTIZE_BITGROOM;
if (H5Aread(attid, H5T_NATIVE_INT, &var->nsd) < 0)
return NC_EHDFERR;
if (H5Aclose(attid) < 0)
return NC_EHDFERR;
}
else
{
var->quantize_mode = NC_NOQUANTIZE;
var->nsd = 0;
}
return NC_NOERR;
}
/**
* @internal Learn the storage and (if chunked) chunksizes of a var.
*
@ -1392,6 +1432,10 @@ nc4_get_var_meta(NC_VAR_INFO_T *var)
if ((retval = nc4_adjust_var_cache(var->container, var)))
BAIL(retval);
/* Is there an attribute which means quantization was used? */
if ((retval = get_quantize_info(var)))
BAIL(retval);
if (var->coords_read && !hdf5_var->dimscale)
if ((retval = get_attached_info(var, hdf5_var, var->ndims, hdf5_var->hdf_datasetid)))
return retval;

182
libhdf5/hdf5var.c
View File

@ -465,6 +465,8 @@ exit:
* @param no_fill Pointer to no_fill setting.
* @param fill_value Pointer to fill value.
* @param endianness Pointer to endianness setting.
* @param quantize_mode Pointer to quantization mode.
* @param nsd Pointer to number of significant digits.
*
* @returns ::NC_NOERR for success
* @returns ::NC_EBADID Bad ncid.
@ -484,7 +486,8 @@ static int
nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
int *unused2, int *fletcher32, int *storage,
const size_t *chunksizes, int *no_fill,
const void *fill_value, int *endianness)
const void *fill_value, int *endianness,
int *quantize_mode, int *nsd)
{
NC_GRP_INFO_T *grp;
NC_FILE_INFO_T *h5;
@ -706,6 +709,46 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
var->endianness = *endianness;
}
/* Remember quantization settings. They will be used when data are
* written. */
if (quantize_mode)
{
/* Only two valid mode settings. */
if (*quantize_mode != NC_NOQUANTIZE &&
*quantize_mode != NC_QUANTIZE_BITGROOM)
return NC_EINVAL;
if (*quantize_mode == NC_QUANTIZE_BITGROOM)
{
/* Only float and double types can have quantization. */
if (var->type_info->hdr.id != NC_FLOAT &&
var->type_info->hdr.id != NC_DOUBLE)
return NC_EINVAL;
/* For bitgroom, number of significant digits is required. */
if (!nsd)
return NC_EINVAL;
/* NSD must be in range. */
if (*nsd <= 0)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_FLOAT &&
*nsd > NC_QUANTIZE_MAX_FLOAT_NSD)
return NC_EINVAL;
if (var->type_info->hdr.id == NC_DOUBLE &&
*nsd > NC_QUANTIZE_MAX_DOUBLE_NSD)
return NC_EINVAL;
var->nsd = *nsd;
}
var->quantize_mode = *quantize_mode;
/* If quantization is turned off, then set nsd to 0. */
if (*quantize_mode == NC_NOQUANTIZE)
var->nsd = 0;
}
return NC_NOERR;
}
@ -737,7 +780,8 @@ NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
int stat = NC_NOERR;
unsigned int level = (unsigned int)deflate_level;
/* Set shuffle first */
if((stat = nc_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done;
if((stat = nc_def_var_extra(ncid, varid, &shuffle, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL))) goto done;
if(deflate) {
if((stat = nc_def_var_filter(ncid, varid, H5Z_FILTER_DEFLATE,1,&level))) goto done;
} /* else ignore */
@ -746,6 +790,113 @@ done:
return stat;
}
/**
* @internal Set quantization settings on a variable. This is
* called by nc_def_var_quantize().
*
* Quantization allows the user to specify a number of significant
* digits for variables of type ::NC_FLOAT or ::NC_DOUBLE. (Attempting
* to set quantize for other types will result in an ::NC_EINVAL
* error.)
*
* When quantize is turned on, and the number of significant digits
* has been specified, then the netCDF library will apply all zeros or
* all ones (alternating) to bits which are not needed to specify the
* value to the number of significant digits. This will change the
* value of the data, but will make it more compressable.
*
* Quantizing the data does not reduce the size of the data on disk,
* but combining quantize with compression will allow for better
* compression. Since the data values are changed, the use of quantize
* and compression such as deflate constitute lossy compression.
*
* Producers of large datasets may find that using quantize with
* compression will result in significant improvent in the final data
* size.
*
* Variables which use quantize will have added an attribute with name
* ::NC_QUANTIZE_ATT_NAME, which will contain the number of
* significant digits. Users should not delete or change this
* attribute. This is the only record that quantize has been applied
* to the data.
*
* As with the deflate settings, quantize settings may only be
* modified before the first call to nc_enddef(). Once nc_enddef() is
* called for the file, quantize settings for any variable in the file
* may not be changed.
*
* Use of quantization is fully backwards compatible with existing
* versions and packages that can read compressed netCDF data. A
* variable which has been quantized is readable to older versions of
* the netCDF libraries, and to netCDF-Java.
*
* @param ncid File ID.
* @param varid Variable ID. NC_GLOBAL may not be used.
* @param quantize_mode Quantization mode. May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM.
* @param nsd Number of significant digits. May be any integer from 1
* to ::NC_QUANTIZE_MAX_FLOAT_NSD (for variables of type ::NC_FLOAT) or
* ::NC_QUANTIZE_MAX_DOUBLE_NSD (for variables of type ::NC_DOUBLE).
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Invalid variable ID.
* @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is
* not netCDF-4/HDF5.
* @returns ::NC_ELATEDEF Too late to change settings for this variable.
* @returns ::NC_ENOTINDEFINE Not in define mode.
* @returns ::NC_EINVAL Invalid input
* @author Ed Hartnett, Dennis Heimbigner
*/
int
NC4_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd)
{
return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL,
&quantize_mode, &nsd);
}
/**
* @internal Get quantize information about a variable. Pass NULL for
* whatever you don't care about.
*
* @param ncid File ID.
* @param varid Variable ID.
* @param quantize_modep Gets quantize mode.
* @param nsdp Gets Number of Significant Digits if quantize is in use.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.
* @returns ::NC_ENOTVAR Bad varid.
* @returns ::NC_EINVAL Invalid input.
* @author Ed Hartnett
*/
int
NC4_inq_var_quantize(int ncid, int varid, int *quantize_modep,
int *nsdp)
{
NC_VAR_INFO_T *var;
int retval;
LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid));
/* Find info for this file and group, and set pointer to each. */
/* Get pointer to the var. */
if ((retval = nc4_hdf5_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
return retval;
if (!var)
return NC_ENOTVAR;
assert(var->hdr.id == varid);
/* Copy the data to the user's data buffers. */
if (quantize_modep)
*quantize_modep = var->quantize_mode;
if (nsdp)
*nsdp = var->nsd;
return 0;
}
#if 0
/**
* @internal Remove a filter from filter list for a variable
@ -803,7 +954,7 @@ int
NC4_def_var_fletcher32(int ncid, int varid, int fletcher32)
{
return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, &fletcher32,
NULL, NULL, NULL, NULL, NULL);
NULL, NULL, NULL, NULL, NULL, NULL, NULL);
}
/**
@ -832,7 +983,7 @@ int
NC4_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
{
return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL,
&storage, chunksizesp, NULL, NULL, NULL);
&storage, chunksizesp, NULL, NULL, NULL, NULL, NULL);
}
/**
@ -878,7 +1029,7 @@ nc_def_var_chunking_ints(int ncid, int varid, int storage, int *chunksizesp)
cs[i] = chunksizesp[i];
retval = nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL,
&storage, cs, NULL, NULL, NULL);
&storage, cs, NULL, NULL, NULL, NULL, NULL);
if (var->ndims)
free(cs);
@ -912,7 +1063,7 @@ int
NC4_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
{
return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL,
NULL, &no_fill, fill_value, NULL);
NULL, &no_fill, fill_value, NULL, NULL, NULL);
}
/**
@ -941,7 +1092,7 @@ int
NC4_def_var_endian(int ncid, int varid, int endianness)
{
return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, &endianness);
NULL, NULL, NULL, &endianness, NULL, NULL);
}
/**
@ -1456,9 +1607,11 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
}
/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if (mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE)
* opaque types.) We also need to call this code if we are doing
* quantization. */
if ((mem_nc_type != var->type_info->hdr.id &&
mem_nc_type != NC_COMPOUND && mem_nc_type != NC_OPAQUE) ||
var->quantize_mode)
{
size_t file_type_size;
@ -1585,7 +1738,8 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode,
var->nsd)))
BAIL(retval);
}
@ -1986,8 +2140,8 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, var->nsd)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE
@ -2054,6 +2208,8 @@ exit:
* @param nparamsp Pointer to memory to store filter parameter count.
* @param params Pointer to vector of unsigned integers into which
* to store filter parameters.
* @param quantize_modep Gets quantization mode.
* @param nsdp Gets number of significant digits, if quantization is in use.
*
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADID Bad ncid.

13
libhdf5/nc4hdf.c
View File

@ -990,11 +990,11 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
var->is_new_var = NC_FALSE;
/* Always write the hidden coordinates attribute, which lists the
* dimids of this var. When present, this speeds opens. When no
* dimids of this var. When present, this speeds opens. When not
* present, dimscale matching is used. */
if (var->ndims)
if ((retval = write_coord_dimids(var)))
BAIL(retval);
BAIL(retval);
/* If this is a dimscale, mark it as such in the HDF5 file. Also
* find the dimension info and store the dataset id of the dimscale
@ -1016,9 +1016,18 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
BAIL(retval);
}
/* If quantization is in use, write an attribute indicating it, a
* single integer which is the number of significant digits. */
if (var->quantize_mode == NC_QUANTIZE_BITGROOM)
if ((retval = nc4_put_att(var->container, var->hdr.id, NC_QUANTIZE_ATT_NAME, NC_INT, 1,
&var->nsd, NC_INT, 0)))
BAIL(retval);
/* Write attributes for this var. */
if ((retval = write_attlist(var->att, var->hdr.id, grp)))
BAIL(retval);
/* The file is now up-to-date with all settings for this var. */
var->attr_dirty = NC_FALSE;
exit:

3
libnczarr/zattr.c
View File

@ -736,7 +736,8 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Data types are like religions, in that one can convert. */
if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,
len, &range_error, NULL,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
}
}

2
libnczarr/zdispatch.c
View File

@ -104,6 +104,8 @@ static const NC_Dispatch NCZ_dispatcher = {
NC4_get_var_chunk_cache,
NCZ_inq_var_filter_ids,
NCZ_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
};
const NC_Dispatch* NCZ_dispatch_table = NULL; /* moved here from ddispatch.c */

6
libnczarr/zvar.c
View File

@ -1527,7 +1527,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
var->quantize_mode, var->nsd)))
BAIL(retval);
}
@ -1902,7 +1903,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode,
var->nsd)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */

1
libnetcdf.settings.in
View File

@ -45,4 +45,5 @@ SZIP Write Support: @HAS_SZLIB_WRITE@
Parallel Filters: @HAS_PAR_FILTERS@
NCZarr Support: @HAS_NCZARR@
Multi-Filter Support: @HAS_MULTIFILTERS@
Quantization: @HAS_QUANTIZE@
Logging: @HAS_LOGGING@

3
libsrc/nc3dispatch.c
View File

@ -166,6 +166,9 @@ NC3_get_var_chunk_cache,
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
};
const NC_Dispatch* NC3_dispatch_table = NULL; /*!< NC3 Dispatch table, moved here from ddispatch.c */

3
libsrc4/nc4attr.c
View File

@ -116,7 +116,8 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
need_to_convert++;
if ((retval = nc4_convert_type(att->data, bufr, att->nc_typeid,
mem_type, (size_t)att->len, &range_error,
NULL, (h5->cmode & NC_CLASSIC_MODEL))))
NULL, (h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE

171
libsrc4/nc4var.c
View File

@ -21,6 +21,43 @@
/** @internal Default size for unlimited dim chunksize. */
#define DEFAULT_1D_UNLIM_SIZE (4096)
/** @internal Minimum number of explicit significand bits to preserve
* when zeroing/bit-masking floating point values. Codes will preserve
* at least two explicit bits, IEEE significand representation
* contains one implicit bit Thus preserve a least three bits which is
* approximately one sigificant decimal digit Used in
* nco_ppc_bitmask() and nco_ppc_bitmask_scl() */
#define NCO_PPC_BIT_XPL_NBR_MIN 2
/* Define log_e for 10 and 2. Prefer constants defined in math.h,
* however, GCC environments can have hard time defining M_LN10/M_LN2
* despite finding math.h */
#ifndef M_LN10
# define M_LN10 2.30258509299404568402 /**< log_e 10 */
#endif /* M_LN10 */
#ifndef M_LN2
# define M_LN2 0.69314718055994530942 /**< log_e 2 */
#endif /* M_LN2 */
/** Used in quantize code. Number of explicit bits in significand for
* floats. Bits 0-22 of SP significands are explicit. Bit 23 is
* implicitly 1. */
#define BIT_XPL_NBR_SGN_FLT (23)
/** Used in quantize code. Number of explicit bits in significand for
* doubles. Bits 0-52 of DP significands are explicit. Bit 53 is
* implicitly 1. */
#define BIT_XPL_NBR_SGN_DBL (53)
/** Pointer union for floating point and bitmask types. */
typedef union { /* ptr_unn */
float *fp;
double *dp;
unsigned int *ui32p;
unsigned long long *ui64p;
void *vp;
} ptr_unn;
/**
* @internal This is called by nc_get_var_chunk_cache(). Get chunk
* cache size for a variable.
@ -448,6 +485,11 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* value used (or the default fill value if none is supplied) for
* values that overflow the type.
*
* This function applies quantization to float and double data, if
* desired. The code to do this is derived from the bitgroom filter in
* the CCR project (see
* https://github.com/ccr/ccr/blob/master/hdf5_plugins/BITGROOM/src/H5Zbitgroom.c).
*
* @param src Pointer to source of data.
* @param dest Pointer that gets data.
* @param src_type Type ID of source data.
@ -456,16 +498,36 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* @param range_error Pointer that gets 1 if there was a range error.
* @param fill_value The fill value.
* @param strict_nc3 Non-zero if strict model in effect.
* @param quantize_mode May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM.
* @param nsd Number of significant diggits for quantizize. Ignored
* unless quantize_mode is ::NC_QUANTIZE_BITGROOM.
*
* @returns NC_NOERR No error.
* @returns NC_EBADTYPE Type not found.
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADTYPE Type not found.
* @author Ed Hartnett, Dennis Heimbigner
*/
int
nc4_convert_type(const void *src, void *dest, const nc_type src_type,
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3)
const void *fill_value, int strict_nc3, int quantize_mode,
int nsd)
{
/* These vars are used with quantize feature. */
const double bit_per_dcm_dgt_prc = M_LN10 / M_LN2; /* 3.32 [frc] Bits per decimal digit of precision */
double mss_val_cmp_dbl; /* Missing value for comparison to double precision values */
float mss_val_cmp_flt; /* Missing value for comparison to single precision values */
int bit_xpl_nbr_zro; /* [nbr] Number of explicit bits to zero */
size_t idx;
unsigned int *u32_ptr;
unsigned int msk_f32_u32_zro;
unsigned int msk_f32_u32_one;
unsigned long long int *u64_ptr;
unsigned long long int msk_f64_u64_zro;
unsigned long long int msk_f64_u64_one;
unsigned short prc_bnr_xpl_rqr; /* [nbr] Explicitly represented binary digits required to retain */
ptr_unn op1; /* I/O [frc] Values to quantize */
char *cp, *cp1;
float *fp, *fp1;
double *dp, *dp1;
@ -483,6 +545,66 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
LOG((3, "%s: len %d src_type %d dest_type %d", __func__, len, src_type,
dest_type));
/* If quantize is in use, set up some values. Quantize can only be
* used when the destination type is NC_FLOAT or NC_DOUBLE. */
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
assert(dest_type == NC_FLOAT || dest_type == NC_DOUBLE);
/* How many bits to preserve? Being conservative, we round up the
* exact binary digits of precision. Add one because the first bit
* is implicit not explicit but corner cases prevent our taking
* advantage of this. */
prc_bnr_xpl_rqr = (unsigned short)ceil(nsd * bit_per_dcm_dgt_prc) + 1;
if (dest_type == NC_DOUBLE)
prc_bnr_xpl_rqr++; /* Seems necessary for double-precision
* ppc=array(1.234567,1.0e-6,$dmn) */
/* Determine masks, copy the data, do the quantization. */
if (dest_type == NC_FLOAT)
{
/* Determine the fill value. */
if (fill_value)
mss_val_cmp_flt = *(float *)fill_value;
else
mss_val_cmp_flt = NC_FILL_FLOAT;
bit_xpl_nbr_zro = BIT_XPL_NBR_SGN_FLT - prc_bnr_xpl_rqr;
/* Create mask */
msk_f32_u32_zro = 0u; /* Zero all bits */
msk_f32_u32_zro = ~msk_f32_u32_zro; /* Turn all bits to ones */
/* Bit Shave mask for AND: Left shift zeros into bits to be
* rounded, leave ones in untouched bits. */
msk_f32_u32_zro <<= bit_xpl_nbr_zro;
/* Bit Set mask for OR: Put ones into bits to be set, zeros in
* untouched bits. */
msk_f32_u32_one = ~msk_f32_u32_zro;
}
else
{
bit_xpl_nbr_zro = BIT_XPL_NBR_SGN_DBL - prc_bnr_xpl_rqr;
assert(bit_xpl_nbr_zro <= BIT_XPL_NBR_SGN_DBL - NCO_PPC_BIT_XPL_NBR_MIN);
if (fill_value)
mss_val_cmp_dbl = *(double *)fill_value;
else
mss_val_cmp_dbl = NC_FILL_DOUBLE;
/* Create mask. */
msk_f64_u64_zro = 0ul; /* Zero all bits. */
msk_f64_u64_zro = ~msk_f64_u64_zro; /* Turn all bits to ones. */
/* Bit Shave mask for AND: Left shift zeros into bits to be
* rounded, leave ones in untouched bits. */
msk_f64_u64_zro <<= bit_xpl_nbr_zro;
/* Bit Set mask for OR: Put ones into bits to be set, zeros in
* untouched bits. */
msk_f64_u64_one =~ msk_f64_u64_zro;
}
} /* endif quantize */
/* OK, this is ugly. If you can think of anything better, I'm open
to suggestions!
@ -557,8 +679,8 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
break;
case NC_FLOAT:
for (bp = (signed char *)src, fp = dest; count < len; count++)
*fp++ = *bp++;
for (bp = (signed char *)src, fp = dest; count < len; count++)
*fp++ = *bp++;
break;
case NC_DOUBLE:
for (bp = (signed char *)src, dp = dest; count < len; count++)
@ -1134,11 +1256,7 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
break;
case NC_FLOAT:
for (fp = (float *)src, fp1 = dest; count < len; count++)
{
/* if (*fp > X_FLOAT_MAX || *fp < X_FLOAT_MIN)
(*range_error)++;*/
*fp1++ = *fp++;
}
break;
case NC_DOUBLE:
for (fp = (float *)src, dp = dest; count < len; count++)
@ -1228,11 +1346,7 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
break;
case NC_DOUBLE:
for (dp = (double *)src, dp1 = dest; count < len; count++)
{
/* if (*dp > X_DOUBLE_MAX || *dp < X_DOUBLE_MIN) */
/* (*range_error)++; */
*dp1++ = *dp++;
}
break;
default:
LOG((0, "%s: unexpected dest type. src_type %d, dest_type %d",
@ -1246,6 +1360,37 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
__func__, src_type, dest_type));
return NC_EBADTYPE;
}
/* If quantize is in use, determine masks, copy the data, do the
* quantization. */
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
if (dest_type == NC_FLOAT)
{
/* Bit-Groom: alternately shave and set LSBs */
op1.fp = (float *)dest;
u32_ptr = op1.ui32p;
for (idx = 0L; idx < len; idx += 2L)
if (op1.fp[idx] != mss_val_cmp_flt)
u32_ptr[idx] &= msk_f32_u32_zro;
for (idx = 1L; idx < len; idx += 2L)
if (op1.fp[idx] != mss_val_cmp_flt && u32_ptr[idx] != 0U) /* Never quantize upwards floating point values of zero */
u32_ptr[idx] |= msk_f32_u32_one;
}
else
{
/* Bit-Groom: alternately shave and set LSBs. */
op1.dp = (double *)dest;
u64_ptr = op1.ui64p;
for (idx = 0L; idx < len; idx += 2L)
if (op1.dp[idx] != mss_val_cmp_dbl)
u64_ptr[idx] &= msk_f64_u64_zro;
for (idx = 1L; idx < len; idx += 2L)
if (op1.dp[idx] != mss_val_cmp_dbl && u64_ptr[idx] != 0ULL) /* Never quantize upwards floating point values of zero */
u64_ptr[idx] |= msk_f64_u64_one;
}
} /* endif quantize */
return NC_NOERR;
}

3
libsrcp/ncpdispatch.c
View File

@ -1463,6 +1463,9 @@ NC_NOTNC4_get_var_chunk_cache,
NC_NOOP_inq_var_filter_ids,
NC_NOOP_inq_var_filter_info,
NC_NOTNC4_def_var_quantize,
NC_NOTNC4_inq_var_quantize,
};
const NC_Dispatch *NCP_dispatch_table = NULL; /* moved here from ddispatch.c */

4
nc_perf/Makefile.am
View File

@ -29,6 +29,7 @@ tst_files2 tst_files3 tst_mem tst_mem1 tst_knmi bm_netcdf4_recs \
tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando
bm_file_SOURCES = bm_file.c tst_utils.c
bm_file_LDFLAGS = -no-install
bm_netcdf4_recs_SOURCES = bm_netcdf4_recs.c tst_utils.c
bm_many_atts_SOURCES = bm_many_atts.c tst_utils.c
bm_many_objs_SOURCES = bm_many_objs.c tst_utils.c
@ -60,7 +61,8 @@ run_bm_test2.log: tst_create_files.log
# This will run parallel I/O benchmarks for parallel builds.
if TEST_PARALLEL4
check_PROGRAMS += tst_gfs_data_1
check_PROGRAMS += tst_compress_par
tst_compress_par_LDFLAGS = -no-install
TESTS += run_par_bm_test.sh run_gfs_test.sh
run_par_bm_test.log: tst_create_files.log run_bm_test1.log
endif # TEST_PARALLEL4

51
nc_perf/tst_gfs_data_1.c → nc_perf/tst_compress_par.c
View File

@ -2,9 +2,11 @@
Copyright 2020, UCAR/Unidata See COPYRIGHT file for copying and
redistribution conditions.
This program tests and benchmarks netcdf-4 parallel I/O using the
same access pattern as is used by NOAA's GFS when writing and
reading model data. See:
This program tests and benchmarks netcdf-4 parallel I/O doing
compression.
This program tries to use the same access pattern as is used by
NOAA's GFS when writing and reading model data. See:
https://github.com/Unidata/netcdf-fortran/issues/264.
Also see the file gfs_sample.cdl to see what is being produced by
@ -28,7 +30,7 @@
#include <netcdf_par.h>
#include <netcdf_meta.h>
#define TEST_NAME "tst_gfs_data_1"
#define TEST_NAME "tst_compress_par"
#define NUM_META_VARS 7
#define NUM_META_TRIES 2
#define NDIM2 2
@ -39,6 +41,7 @@
/* #define NUM_DEFLATE_LEVELS 3 */
#define NUM_DEFLATE_LEVELS 3
#define NUM_UNLIM_TRIES 1
#define NUM_NSD_SETTINGS 2
#define THOUSAND 1000
#define NUM_DATA_VARS 3
#define ERR_AWFUL 1
@ -61,6 +64,7 @@
#define MAX_COMPRESSION_FILTERS 4
char compression_filter_name[MAX_COMPRESSION_FILTERS][NC_MAX_NAME + 1];
int deflate_level[MAX_COMPRESSION_FILTERS][NUM_DEFLATE_LEVELS];
int nsd[NUM_NSD_SETTINGS] = {0, 4};
char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull",
"phalf", "time"};
@ -176,7 +180,7 @@ check_meta(int ncid, int *data_varid, int s, int f, int deflate, int u,
/* Write all the metadata, including coordinate variable data. */
int
write_meta(int ncid, int *data_varid, int s, int f, int deflate, int u,
write_meta(int ncid, int *data_varid, int s, int f, int nsd, int deflate, int u,
size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start,
size_t *data_count, size_t pfull_start, size_t pfull_size, float *pfull,
size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start,
@ -290,6 +294,9 @@ write_meta(int ncid, int *data_varid, int s, int f, int deflate, int u,
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR;
if (nsd)
if (nc_def_var_quantize(ncid, data_varid[dv], NC_QUANTIZE_BITGROOM, nsd)) ERR;
/* Setting any filter only will work for HDF5-1.10.3 and later */
/* versions. Do nothing for "none". */
if (!strcmp(compression_filter_name[f], "zlib"))
@ -537,7 +544,6 @@ main(int argc, char **argv)
MPI_Info info = MPI_INFO_NULL;
/* For timing. */
double meta_start_time, meta_stop_time;
double data_start_time, data_stop_time;
int ncid;
@ -561,7 +567,7 @@ main(int argc, char **argv)
/* Compression filter info. */
int num_compression_filters;
int f, s, u;
int f, s, n;
int i, j, k, dv, dl;
int ret;
@ -604,7 +610,8 @@ main(int argc, char **argv)
{
for(i = 0; i < data_count[3]; i++)
{
value_data[cnt] = my_rank * 1000 + cnt / sqrt(my_rank + cnt + 1);
/* value_data[cnt] = (-1 * i%2) * my_rank * 1000 + cnt / sqrt(my_rank + cnt + 1) - (-1 * i%3 * i); */
value_data[cnt] = (-1 * i%2) * my_rank * 1000 + cnt / sqrt(my_rank + cnt + 1) - (-1 * i%2 * i);
/* printf("%d: value_data[%ld] %g\n", my_rank, cnt, value_data[cnt]); */
cnt++;
}
@ -614,15 +621,15 @@ main(int argc, char **argv)
if (my_rank == 0)
{
printf("Benchmarking creation of file similar to one produced by the UFS.\n");
printf("unlim, comp, level, shuffle, meta wr time (s), data wr rate (MB/s), "
printf("comp, level, nsd, shuffle, data wr rate (MB/s), "
"file size (MB)\n");
}
for (u = 0; u < NUM_UNLIM_TRIES; u++)
for (f = 0; f < num_compression_filters; f++)
{
for (f = 0; f < num_compression_filters; f++)
{
for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
{
for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
{
for (n = 0; n < NUM_NSD_SETTINGS; n++)
{
for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++)
{
size_t file_size;
@ -638,10 +645,9 @@ main(int argc, char **argv)
/* nc_set_log_level(3); */
/* Create a parallel netcdf-4 file. */
meta_start_time = MPI_Wtime();
if (nc_create_par(file_name, NC_NETCDF4, comm, info,
&ncid)) ERR;
if (write_meta(ncid, data_varid, s, f, deflate_level[f][dl], u,
if (write_meta(ncid, data_varid, s, f, nsd[n], deflate_level[f][dl], 0,
phalf_size, phalf_start, phalf,
data_start, data_count, pfull_start, pfull_size, pfull, grid_xt_start,
grid_xt_size, grid_xt, grid_yt_start,
@ -650,7 +656,6 @@ main(int argc, char **argv)
/* Stop the timer for metadata writes. */
MPI_Barrier(MPI_COMM_WORLD);
meta_stop_time = MPI_Wtime();
data_start_time = MPI_Wtime();
/* Write one record each of the data variables. */
@ -676,7 +681,7 @@ main(int argc, char **argv)
/* Check the file metadata for correctness. */
if (nc_open_par(file_name, NC_NOWRITE, comm, info, &ncid)) ERR;
if (check_meta(ncid, data_varid, s, f, deflate_level[f][dl], u,
if (check_meta(ncid, data_varid, s, f, deflate_level[f][dl], 0,
phalf_size, phalf_start, phalf,
data_start, data_count, pfull_start, pfull_size,
pfull, grid_xt_start, grid_xt_size, grid_xt,
@ -692,15 +697,15 @@ main(int argc, char **argv)
dim_len[4] * sizeof(float))/MILLION;
/* printf("data_size %f (data_stop_time - data_start_time) %g\n", data_size, (data_stop_time - data_start_time)); */
data_rate = data_size / (data_stop_time - data_start_time);
printf("%d, %s, %d, %d, %g, %g, %g\n", u, compression_filter_name[f],
deflate_level[f][dl], s, meta_stop_time - meta_start_time,
printf("%s, %d, %d, %d, %g, %g\n", compression_filter_name[f],
deflate_level[f][dl], nsd[n], s,
data_rate, (float)file_size/MILLION);
}
MPI_Barrier(MPI_COMM_WORLD);
} /* next deflate level */
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
} /* next unlim_try */
} /* next nsd */
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
/* Free resources. */
if (grid_xt)

3
nc_test4/CMakeLists.txt
View File

@ -22,7 +22,7 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4
tst_files6 tst_sync tst_h_strbug tst_h_refs tst_h_scalar tst_rename
tst_rename2 tst_rename3 tst_h5_endians tst_atts_string_rewrite tst_put_vars_two_unlim_dim
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_types tst_bug324
tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442)
tst_atts3 tst_put_vars tst_elatefill tst_udf tst_bug1442 tst_quantize)
# Note, renamegroup needs to be compiled before run_grp_rename
@ -109,5 +109,6 @@ IF(TEST_PARALLEL4)
build_bin_test(tst_nc4perf)
build_bin_test(tst_mode)
build_bin_test(tst_simplerw_coll_r)
build_bin_test(tst_quantize_par)
add_sh_test(nc_test4 run_par_test)
ENDIF()

4
nc_test4/Makefile.am
View File

@ -35,7 +35,7 @@ tst_h_scalar tst_rename tst_rename2 tst_rename3 tst_h5_endians \
tst_atts_string_rewrite tst_hdf5_file_compat tst_fill_attr_vanish \
tst_rehash tst_filterparser tst_bug324 tst_types tst_atts3 \
tst_put_vars tst_elatefill tst_udf tst_put_vars_two_unlim_dim \
tst_bug1442
tst_bug1442 tst_quantize
# Temporary I hoped, but hoped in vain.
if !ISCYGWIN
@ -89,7 +89,7 @@ endif # BUILD_UTILITIES
if TEST_PARALLEL4
check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \
tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \
tst_mode tst_parallel_zlib tst_parallel_compress
tst_mode tst_parallel_zlib tst_parallel_compress tst_quantize_par
TESTS += run_par_test.sh
endif # TEST_PARALLEL4

6
nc_test4/run_par_test.sh.in
View File

@ -59,5 +59,9 @@ if test "@HAS_PAR_FILTERS@" = "yes"; then
echo "Parallel I/O more tests with zlib and szip (if present in HDF5)."
@MPIEXEC@ -n 1 ./tst_parallel_compress
@MPIEXEC@ -n 4 ./tst_parallel_compress
fi
echo
echo "Parallel I/O test for quantize feature."
@MPIEXEC@ -n 4 ./tst_quantize_par

840
nc_test4/tst_quantize.c Normal file
View File

@ -0,0 +1,840 @@
/* This is part of the netCDF package.
Copyright 2021 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
Test quantization of netcdf-4 variables. Quantization is the
zeroing-out of bits in float or double data beyond a desired
precision.
Ed Hartnett, 8/19/21
*/
#include <nc_tests.h>
#include "err_macros.h"
#include "netcdf.h"
#define TEST "tst_quantize"
#define FILE_NAME "tst_quantize.nc"
#define NDIM1 1
#define DIM_NAME_1 "meters_along_canal"
#define DIM_LEN_3 3
#define DIM_LEN_1 1
#define DIM_LEN_5 5
#define DIM_LEN_8 8
#define VAR_NAME_1 "Amsterdam_houseboat_location"
#define VAR_NAME_2 "Amsterdam_street_noise_decibels"
#define NSD_3 3
#define NSD_9 9
/* This var used to help print a float in hex. */
char pf_str[20];
/* This struct allows us to treat float as uint32_t
* types. */
union FU {
float f;
uint32_t u;
};
/* This struct allows us to treat double points as uint64_t
* types. */
union DU {
double d;
uint64_t u;
};
/* This function prints a float as hex. */
char *
pf(float myf)
{
union {
float f;
uint32_t u;
} fu;
fu.f = myf;
sprintf(pf_str, "0x%x", fu.u);
return pf_str;
}
/* This function prints a double as hex. */
char *
pd(double myd)
{
union {
double d;
uint64_t u;
} du;
du.d = myd;
sprintf(pf_str, "0x%lx", du.u);
return pf_str;
}
int
main(int argc, char **argv)
{
printf("\n*** Testing netcdf-4 variable quantization functions.\n");
printf("**** testing quantization setting and error conditions...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
/* Create a netcdf classic file with one var. Attempt
* quantization. It will not work. */
if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_3, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3) != NC_ENOTNC4) ERR;
if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in) != NC_ENOTNC4) ERR;
if (nc_close(ncid)) ERR;
/* Create a netcdf-4 file with two vars. Attempt
* quantization. It will work, eventually... */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_3, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Bad varid. */
if (nc_def_var_quantize(ncid, NC_GLOBAL, NC_QUANTIZE_BITGROOM, NSD_3) != NC_EGLOBAL) ERR;
if (nc_def_var_quantize(ncid, varid2 + 1, NC_QUANTIZE_BITGROOM, NSD_3) != NC_ENOTVAR) ERR;
/* Invalid values. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM + 1, NSD_3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, -1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NC_QUANTIZE_MAX_FLOAT_NSD + 1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM + 1, 3) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, -1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NC_QUANTIZE_MAX_DOUBLE_NSD + 1) != NC_EINVAL) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, 0) != NC_EINVAL) ERR;
/* This will work. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR;
if (nsd_in != NSD_3) ERR;
/* Wait, I changed my mind! Let's turn off quantization. */
if (nc_def_var_quantize(ncid, varid1, NC_NOQUANTIZE, 0)) ERR;
if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_NOQUANTIZE) ERR;
if (nsd_in != 0) ERR;
/* Changed my mind again, turn it on. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* I changed my mind again! Turn it off! */
if (nc_def_var_quantize(ncid, varid1, NC_NOQUANTIZE, 0)) ERR;
if (nc_inq_var_quantize(ncid, varid1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_NOQUANTIZE) ERR;
if (nsd_in != 0) ERR;
/* Changed my mind again, turn it on. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* This also will work for double. */
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_9)) ERR;
if (nc_inq_var_quantize(ncid, varid2, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR;
if (nsd_in != NSD_9) ERR;
/* End define mode. */
if (nc_enddef(ncid)) ERR;
/* This will not work, it's too late! */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3) != NC_ELATEDEF) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR;
if (nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM) ERR;
if (nsd_in != NSD_9) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#define NX_BIG 100
#define NY_BIG 100
#define NTYPES 9
#define VAR_NAME "Amsterdam_coffeeshop_location"
#define X_NAME "distance_from_center"
#define Y_NAME "distance_along_canal"
#define NDIM2 2
printf("**** testing quantization handling of non-floats...");
{
int ncid;
int dimid[NDIM2];
int varid;
int nsd_in, quantize_mode;
int nsd_out = 3;
char file_name[NC_MAX_NAME + 1];
int xtype[NTYPES] = {NC_CHAR, NC_SHORT, NC_INT, NC_BYTE, NC_UBYTE,
NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
int t;
for (t = 0; t < NTYPES; t++)
{
sprintf(file_name, "%s_bitgroom_type_%d.nc", TEST, xtype[t]);
/* Create file. */
if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, X_NAME, NX_BIG, &dimid[0])) ERR;
if (nc_def_dim(ncid, Y_NAME, NY_BIG, &dimid[1])) ERR;
if (nc_def_var(ncid, VAR_NAME, xtype[t], NDIM2, dimid, &varid)) ERR;
/* Bitgroom filter returns NC_EINVAL because this is not an
* NC_FLOAT or NC_DOULBE. */
if (nc_def_var_quantize(ncid, varid, NC_QUANTIZE_BITGROOM, nsd_out) != NC_EINVAL) ERR;
if (nc_close(ncid)) ERR;
/* Check file. */
{
if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in)) ERR;
if (quantize_mode) ERR;
if (nc_close(ncid)) ERR;
}
}
}
SUMMARIZE_ERR;
printf("**** testing quantization of scalars...");
{
int ncid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_1] = {1.1111111};
double double_data[DIM_LEN_1] = {1.111111111111};
/* Create a netcdf-4 file with two scalar vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, 0, NULL, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, 0, NULL, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_float(ncid, varid1, float_data)) ERR;
if (nc_put_var_double(ncid, varid2, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in;
double double_in;
union FU fin;
int nsd_att_in;
/* union FU fout; */
union DU dfin;
/* union DU dfout; */
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Each var now has an attribute describing the quantize settings. */
if (nc_get_att_int(ncid, 0, NC_QUANTIZE_ATT_NAME, &nsd_att_in)) ERR;
if (nsd_att_in != NSD_3) ERR;
if (nc_get_att_int(ncid, 1, NC_QUANTIZE_ATT_NAME, &nsd_att_in)) ERR;
if (nsd_att_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, &float_in)) ERR;
if (nc_get_var(ncid, varid2, &double_in)) ERR;
/* fout.f = float_data[0]; */
fin.f = float_in;
/* dfout.d = double_data[0]; */
dfin.d = double_in;
/* printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[0], fout.u, float_data[0], fin.u); */
if (fin.u != 0x3f8e3000) ERR;
/* printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", */
/* double_data[0], dfout.u, double_data[0], dfin.u); */
if (dfin.u != 0x3ff1c60000000000) ERR;
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing quantization of one value...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_1] = {1.1111111};
double double_data[DIM_LEN_1] = {1.111111111111};
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_float(ncid, varid1, float_data)) ERR;
if (nc_put_var_double(ncid, varid2, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in;
double double_in;
union FU fin;
/* union FU fout; */
union DU dfin;
/* union DU dfout; */
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, &float_in)) ERR;
if (nc_get_var(ncid, varid2, &double_in)) ERR;
/* fout.f = float_data[0]; */
fin.f = float_in;
/* dfout.d = double_data[0]; */
dfin.d = double_in;
/* printf ("\nfloat_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[0], fout.u, float_data[0], fin.u); */
if (fin.u != 0x3f8e3000) ERR;
/* printf ("\ndouble_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", */
/* double_data[0], dfout.u, double_data[0], dfin.u); */
if (dfin.u != 0x3ff1c60000000000) ERR;
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing more quantization values...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567};
double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0};
int x;
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_float(ncid, varid1, float_data)) ERR;
if (nc_put_var_double(ncid, varid2, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in[DIM_LEN_5];
double double_in[DIM_LEN_5];
union FU {
float f;
uint32_t u;
};
union FU fin;
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x3f800fff;
xpect[2].u = 0x41200000;
xpect[3].u = 0x4640efff;
xpect[4].u = 0x3dfcd000;
double_xpect[0].u = 0x3ff1c60000000000;
double_xpect[1].u = 0x3ff001ffffffffff;
double_xpect[2].u = 0x4023fe0000000000;
double_xpect[3].u = 0x41d265ffffffffff;
double_xpect[4].u = 0x42dc120000000000;
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, float_in)) ERR;
if (nc_get_var(ncid, varid2, double_in)) ERR;
/* printf("\n"); */
for (x = 0; x < DIM_LEN_5; x++)
{
/* fout.f = float_data[x]; */
fin.f = float_in[x];
/* printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[x], fout.u, float_data[x], fin.u); */
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
if (dfin.u != double_xpect[x].u) ERR;
}
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing quantization of one value with type conversion...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_1] = {1.1111111};
double double_data[DIM_LEN_1] = {1.111111111111};
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_1, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some double data to float var. */
if (nc_put_var_double(ncid, varid1, double_data)) ERR;
/* Write some float data to double var. */
if (nc_put_var_float(ncid, varid2, float_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in;
double double_in;
union FU fin;
/* union FU fout; */
union DU dfin;
/* union DU dfout; */
int nsd_att_in;
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Each var now has an attribute describing the quantize settings. */
if (nc_get_att_int(ncid, 0, NC_QUANTIZE_ATT_NAME, &nsd_att_in)) ERR;
if (nsd_att_in != NSD_3) ERR;
if (nc_get_att_int(ncid, 1, NC_QUANTIZE_ATT_NAME, &nsd_att_in)) ERR;
if (nsd_att_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, &float_in)) ERR;
if (nc_get_var(ncid, varid2, &double_in)) ERR;
/* fout.f = (float)double_data[0]; */
fin.f = float_in;
/* dfout.d = float_data[0]; */
dfin.d = double_in;
/* printf ("\ndouble_data: %15g : 0x%x float_data_in: %10f : 0x%x\n", */
/* double_data[0], fout.u, float_in, fin.u); */
if (fin.u != 0x3f8e3000) ERR;
/* printf ("\nfloat_data: %15g : 0x%16lx double_data_in: %15g : 0x%lx\n", */
/* float_data[0], dfout.u, double_in, dfin.u); */
if (dfin.u != 0x3ff1c60000000000) ERR;
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing more quantization values with type conversion...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567};
double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0};
int x;
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_double(ncid, varid1, double_data)) ERR;
if (nc_put_var_float(ncid, varid2, float_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in[DIM_LEN_5];
double double_in[DIM_LEN_5];
union FU {
float f;
uint32_t u;
};
union FU fin;
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x3f800fff;
xpect[2].u = 0x41200000;
xpect[3].u = 0x4e932fff;
xpect[4].u = 0x56e09000;
double_xpect[0].u = 0x3ff1c60000000000;
double_xpect[1].u = 0x3ff001ffffffffff;
double_xpect[2].u = 0x4024000000000000;
double_xpect[3].u = 0x40c81dffffffffff;
double_xpect[4].u = 0x3fbf9a0000000000;
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, float_in)) ERR;
if (nc_get_var(ncid, varid2, double_in)) ERR;
/* printf("\n"); */
for (x = 0; x < DIM_LEN_5; x++)
{
/* fout.f = float_data[x]; */
fin.f = float_in[x];
/* printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[x], fout.u, float_data[x], fin.u); */
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
if (dfin.u != double_xpect[x].u) ERR;
}
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing more quantization values with default fill values...");
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_5] = {1.11111111, NC_FILL_FLOAT, 9.99999999, 12345.67, NC_FILL_FLOAT};
double double_data[DIM_LEN_5] = {1.1111111, NC_FILL_DOUBLE, 9.999999999, 1234567890.12345, NC_FILL_DOUBLE};
int x;
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_float(ncid, varid1, float_data)) ERR;
if (nc_put_var_double(ncid, varid2, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in[DIM_LEN_5];
double double_in[DIM_LEN_5];
union FU {
float f;
uint32_t u;
};
union FU fin;
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x7cf00000;
xpect[2].u = 0x41200000;
xpect[3].u = 0x4640efff;
xpect[4].u = 0x7cf00000;
double_xpect[0].u = 0x3ff1c60000000000;
double_xpect[1].u = 0x479e000000000000;
double_xpect[2].u = 0x4023fe0000000000;
double_xpect[3].u = 0x41d265ffffffffff;
double_xpect[4].u = 0x479e000000000000;
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, float_in)) ERR;
if (nc_get_var(ncid, varid2, double_in)) ERR;
/* printf("\n"); */
for (x = 0; x < DIM_LEN_5; x++)
{
/* fout.f = float_data[x]; */
fin.f = float_in[x];
/* printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[x], fout.u, float_data[x], fin.u); */
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
if (dfin.u != double_xpect[x].u) ERR;
}
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("**** testing more quantization values with custom fill values...");
{
#define CUSTOM_FILL_FLOAT 99.99999
#define CUSTOM_FILL_DOUBLE -99999.99999
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_5] = {1.11111111, CUSTOM_FILL_FLOAT, 9.99999999, 12345.67, CUSTOM_FILL_FLOAT};
double double_data[DIM_LEN_5] = {1.1111111, CUSTOM_FILL_DOUBLE, 9.999999999, 1234567890.12345, CUSTOM_FILL_DOUBLE};
float custom_fill_float = CUSTOM_FILL_FLOAT;
double custom_fill_double = CUSTOM_FILL_DOUBLE;
int x;
/* Create a netcdf-4 file with two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_5, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_put_att_float(ncid, varid1, _FillValue, NC_FLOAT, 1, &custom_fill_float)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
if (nc_put_att_double(ncid, varid2, _FillValue, NC_DOUBLE, 1, &custom_fill_double)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. */
if (nc_put_var_float(ncid, varid1, float_data)) ERR;
if (nc_put_var_double(ncid, varid2, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in[DIM_LEN_5];
double double_in[DIM_LEN_5];
union FU {
float f;
uint32_t u;
};
union FU fin;
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x42c7ffff;
xpect[2].u = 0x41200000;
xpect[3].u = 0x4640efff;
xpect[4].u = 0x42c7ffff;
double_xpect[0].u = 0x3ff1c60000000000;
double_xpect[1].u = 0xc0f869fffff583a5;
double_xpect[2].u = 0x4023fe0000000000;
double_xpect[3].u = 0x41d265ffffffffff;
double_xpect[4].u = 0xc0f869fffff583a5;
/* Open the file and check metadata. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
if (nc_get_var(ncid, varid1, float_in)) ERR;
if (nc_get_var(ncid, varid2, double_in)) ERR;
/* printf("\n"); */
for (x = 0; x < DIM_LEN_5; x++)
{
/* fout.f = float_data[x]; */
fin.f = float_in[x];
/* printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[x], fout.u, float_data[x], fin.u); */
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
if (dfin.u != double_xpect[x].u) ERR;
}
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
printf("*** Checking BitGroom values with type conversion between ints and floats...");
{
int ncid;
int dimid;
int varid1, varid2;
unsigned char uc = 99;
signed char sc = -99;
unsigned short us = 9999;
signed short ss = -9999;
unsigned int ui = 9999999;
signed int si = -9999999;
unsigned long long int ull = 999999999;
signed long long int sll = -999999999;
size_t index;
/* Create file. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
/* Create dims. */
if (nc_def_dim(ncid, X_NAME, DIM_LEN_8, &dimid)) ERR;
/* Create the variables. */
if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Set up quantization. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write data. */
index = 0;
if (nc_put_var1_uchar(ncid, varid1, &index, &uc)) ERR;
if (nc_put_var1_uchar(ncid, varid2, &index, &uc)) ERR;
index = 1;
if (nc_put_var1_schar(ncid, varid1, &index, &sc)) ERR;
if (nc_put_var1_schar(ncid, varid2, &index, &sc)) ERR;
index = 2;
if (nc_put_var1_ushort(ncid, varid1, &index, &us)) ERR;
if (nc_put_var1_ushort(ncid, varid2, &index, &us)) ERR;
index = 3;
if (nc_put_var1_short(ncid, varid1, &index, &ss)) ERR;
if (nc_put_var1_short(ncid, varid2, &index, &ss)) ERR;
index = 4;
if (nc_put_var1_uint(ncid, varid1, &index, &ui)) ERR;
if (nc_put_var1_uint(ncid, varid2, &index, &ui)) ERR;
index = 5;
if (nc_put_var1_int(ncid, varid1, &index, &si)) ERR;
if (nc_put_var1_int(ncid, varid2, &index, &si)) ERR;
index = 6;
if (nc_put_var1_ulonglong(ncid, varid1, &index, &ull)) ERR;
if (nc_put_var1_ulonglong(ncid, varid2, &index, &ull)) ERR;
index = 7;
if (nc_put_var1_longlong(ncid, varid1, &index, &sll)) ERR;
if (nc_put_var1_longlong(ncid, varid2, &index, &sll)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_data_in[DIM_LEN_8];
double double_data_in[DIM_LEN_8];
int x;
/* Now reopen the file and check. */
if (nc_open(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
/* Read the data. */
if (nc_get_var_float(ncid, varid1, float_data_in)) ERR;
if (nc_get_var_double(ncid, varid2, double_data_in)) ERR;
union FU xpect[DIM_LEN_8];
union DU double_xpect[DIM_LEN_8];
/* This test comes up with different answers to this than
* the corresponding bitgroom filter test, but that's
* OK. In netcdf-c quantization is applied as the data are
* written by the user, but in HDF5 filters, the bitgroom
* filter is applied to all data values as they are
* written to disk. See
* https://github.com/ccr/ccr/issues/194 for a full
* explanation. */
xpect[0].u = 0x42c60000;
xpect[1].u = 0xc2c60000;
xpect[2].u = 0x461c3000;
xpect[3].u = 0xc61c3000;
xpect[4].u = 0x4b189000;
xpect[5].u = 0xcb189000;
xpect[6].u = 0x4e6e6b28;
xpect[6].u = 0x4e6e6000;
xpect[7].u = 0xce6e6000;
double_xpect[0].u = 0x4058c00000000000;
double_xpect[1].u = 0xc058c00000000000;
double_xpect[2].u = 0x40c3860000000000;
double_xpect[3].u = 0xc0c3860000000000;
double_xpect[4].u = 0x4163120000000000;
double_xpect[5].u = 0xc163120000000000;
double_xpect[6].u = 0x41cdcc0000000000;
double_xpect[7].u = 0xc1cdcc0000000000;
for (x = 0; x < DIM_LEN_8; x++)
{
union FU fin;
union DU dfin;
fin.f = float_data_in[x];
dfin.d = double_data_in[x];
/* printf ("%d float_data_in : %08.8f : 0x%x expected %08.8f : 0x%x\n", */
/* x, float_data_in[x], fin.u, xpect[x].f, xpect[x].u); */
/* printf ("%d double_data_in : %15g : 0x%lx expected %15g : 0x%lx\n", */
/* x, double_data_in[x], dfin.u, double_xpect[x].d, double_xpect[x].u); */
if (fin.u != xpect[x].u)
ERR;
if (dfin.u != double_xpect[x].u)
ERR;
}
/* Close the file. */
if (nc_close(ncid)) ERR;
}
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}

190
nc_test4/tst_quantize_par.c Normal file
View File

@ -0,0 +1,190 @@
/* This is part of the netCDF package.
Copyright 2021 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
Test quantization of netcdf-4 variables with parallel
I/O. Quantization is the zeroing-out of bits in float or double
data beyond a desired precision.
Ed Hartnett, 9/2/21
*/
#include <nc_tests.h>
#include "err_macros.h"
#include "netcdf.h"
#define TEST "tst_quantize"
#define FILE_NAME "tst_quantize_par.nc"
#define NDIM1 1
#define DIM_NAME_1 "DIM_1"
#define DIM_LEN_5 5
#define DIM_LEN_20 20
#define VAR_NAME_1 "VAR_1"
#define VAR_NAME_2 "VAR_2"
#define NSD_3 3
#define NSD_9 9
/* This var used to help print a float in hex. */
char pf_str[20];
/* This struct allows us to treat float as uint32_t
* types. */
union FU {
float f;
uint32_t u;
};
/* This struct allows us to treat double points as uint64_t
* types. */
union DU {
double d;
uint64_t u;
};
/* This function prints a float as hex. */
char *
pf(float myf)
{
union {
float f;
uint32_t u;
} fu;
fu.f = myf;
sprintf(pf_str, "0x%x", fu.u);
return pf_str;
}
/* This function prints a double as hex. */
char *
pd(double myd)
{
union {
double d;
uint64_t u;
} du;
du.d = myd;
sprintf(pf_str, "0x%lx", du.u);
return pf_str;
}
int
main(int argc, char **argv)
{
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
/* Initialize MPI. */
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
/* Must be run on exactly 4 processors. */
if (mpi_size != 4)
{
if (mpi_rank == 0)
printf("Test must be run on 4 processors.\n");
return 2;
}
if (mpi_rank == 0)
{
printf("\n*** Testing netcdf-4 variable quantization with parallel I/O.\n");
printf("**** testing quantization on four processors...");
}
{
int ncid, dimid, varid1, varid2;
int quantize_mode_in, nsd_in;
float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567};
double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0};
size_t start[NDIM1], count[NDIM1] = {DIM_LEN_5};
int x;
/* Create a netcdf-4 file with two vars. */
if (nc_create_par(FILE_NAME, NC_NETCDF4|NC_CLOBBER, comm, info, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME_1, DIM_LEN_20, &dimid)) ERR;
if (nc_def_var(ncid, VAR_NAME_1, NC_FLOAT, NDIM1, &dimid, &varid1)) ERR;
if (nc_def_var(ncid, VAR_NAME_2, NC_DOUBLE, NDIM1, &dimid, &varid2)) ERR;
/* Turn on quantize for both vars. */
if (nc_def_var_quantize(ncid, varid1, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
if (nc_def_var_quantize(ncid, varid2, NC_QUANTIZE_BITGROOM, NSD_3)) ERR;
/* Write some data. Each of the 4 processes writes the same 5
* values, writing 20 in all. */
start[0] = mpi_rank * DIM_LEN_5;
if (nc_put_vara_float(ncid, varid1, start, count, float_data)) ERR;
if (nc_put_vara_double(ncid, varid2, start, count, double_data)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
{
float float_in[DIM_LEN_5];
double double_in[DIM_LEN_5];
union FU {
float f;
uint32_t u;
};
union FU fin;
/* union FU fout; */
union FU xpect[DIM_LEN_5];
union DU dfin;
/* union DU dfout; */
union DU double_xpect[DIM_LEN_5];
xpect[0].u = 0x3f8e3000;
xpect[1].u = 0x3f800fff;
xpect[2].u = 0x41200000;
xpect[3].u = 0x4640efff;
xpect[4].u = 0x3dfcd000;
double_xpect[0].u = 0x3ff1c60000000000;
double_xpect[1].u = 0x3ff001ffffffffff;
double_xpect[2].u = 0x4023fe0000000000;
double_xpect[3].u = 0x41d265ffffffffff;
double_xpect[4].u = 0x42dc120000000000;
/* Open the file and check metadata. */
if (nc_open_par(FILE_NAME, NC_WRITE, comm, info, &ncid)) ERR;
if (nc_inq_var_quantize(ncid, 0, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
if (nc_inq_var_quantize(ncid, 1, &quantize_mode_in, &nsd_in)) ERR;
if (quantize_mode_in != NC_QUANTIZE_BITGROOM || nsd_in != NSD_3) ERR;
/* Check the data. */
start[0] = mpi_rank * DIM_LEN_5;
if (nc_get_vara_float(ncid, varid1, start, count, float_in)) ERR;
if (nc_get_vara_double(ncid, varid2, start, count, double_in)) ERR;
/* printf("\n"); */
for (x = 0; x < DIM_LEN_5; x++)
{
/* fout.f = float_data[x]; */
fin.f = float_in[x];
/* printf ("float_data: %10f : 0x%x float_data_in: %10f : 0x%x\n", */
/* float_data[x], fout.u, float_data[x], fin.u); */
if (fin.u != xpect[x].u) ERR;
/* dfout.d = double_data[x]; */
dfin.d = double_in[x];
/* printf("double_data: %15g : 0x%16lx double_data_in: %15g : 0x%16lx\n", */
/* double_data[x], dfout.u, double_data[x], dfin.u); */
if (dfin.u != double_xpect[x].u) ERR;
}
/* Close the file again. */
if (nc_close(ncid)) ERR;
}
}
/* Shut down MPI. */
MPI_Finalize();
if (mpi_rank == 0)
{
SUMMARIZE_ERR;
FINAL_RESULTS;
}
return 0;
}

2
ncdump/Makefile.am
View File

@ -79,7 +79,7 @@ TESTS = tst_inttags.sh run_tests.sh tst_64bit.sh ref_ctest \
ref_ctest64 tst_output.sh tst_lengths.sh tst_calendars.sh \
run_utf8_tests.sh tst_nccopy3.sh tst_nccopy3_subset.sh \
tst_charfill.sh tst_iter.sh tst_formatx3.sh tst_bom.sh \
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh test_rcmerge.sh
tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh #test_rcmerge.sh
# The tst_nccopy3.sh test uses output from a bunch of other
# tests. This records the dependency so parallel builds work.

2
ncdump/cdl/ref_tst_econst.cdl
View File

@ -9,5 +9,5 @@ variables:
data:
test1=pass;
test2=e2.undefined;
test3=/e1.fail;
test3=e1.fail;
}

46
ncdump/cdl/ref_tst_econst2.cdl Normal file
View File

@ -0,0 +1,46 @@
netcdf foo { // an example netCDF specification in CDL
types:
ubyte enum enum_t {Clear = 0, Cumulonimbus = 1, Stratus = 2};
opaque(11) opaque_t;
int(*) vlen_t;
dimensions:
lat = 10, lon = 5, time = unlimited ;
variables:
long lat(lat), lon(lon), time(time);
float Z(time,lat,lon), t(time,lat,lon);
double p(time,lat,lon);
long rh(time,lat,lon);
string country(time,lat,lon);
ubyte tag;
// variable attributes
lat:long_name = "latitude";
lat:units = "degrees_north";
lon:long_name = "longitude";
lon:units = "degrees_east";
time:units = "seconds since 1992-1-1 00:00:00";
// typed variable attributes
string Z:units = "geopotential meters";
float Z:valid_range = 0., 5000.;
double p:_FillValue = -9999.;
long rh:_FillValue = -1;
vlen_t :globalatt = {17, 18, 19};
data:
lat = 0, 10, 20, 30, 40, 50, 60, 70, 80, 90;
lon = -140, -118, -96, -84, -52;
group: g {
types:
compound cmpd_t { vlen_t f1; enum_t f2; enum_t f3;};
} // group g
group: h {
variables:
/g/cmpd_t compoundvar;
data:
compoundvar = { {3,4,5}, Stratus, enum_t.Clear } ;
} // group h
}

56
ncdump/expected/ref_tst_econst2.dmp Normal file
View File

@ -0,0 +1,56 @@
netcdf ref_tst_econst2 {
types:
ubyte enum enum_t {Clear = 0, Cumulonimbus = 1, Stratus = 2} ;
opaque(11) opaque_t ;
int(*) vlen_t ;
dimensions:
lat = 10 ;
lon = 5 ;
time = UNLIMITED ; // (0 currently)
variables:
int lat(lat) ;
lat:long_name = "latitude" ;
lat:units = "degrees_north" ;
int lon(lon) ;
lon:long_name = "longitude" ;
lon:units = "degrees_east" ;
int time(time) ;
time:units = "seconds since 1992-1-1 00:00:00" ;
float Z(time, lat, lon) ;
string Z:units = "geopotential meters" ;
Z:valid_range = 0.f, 5000.f ;
float t(time, lat, lon) ;
double p(time, lat, lon) ;
p:_FillValue = -9999. ;
int rh(time, lat, lon) ;
rh:_FillValue = -1 ;
string country(time, lat, lon) ;
ubyte tag ;
// global attributes:
vlen_t :globalatt = {17, 18, 19} ;
data:
lat = 0, 10, 20, 30, 40, 50, 60, 70, 80, 90 ;
lon = -140, -118, -96, -84, -52 ;
tag = 255 ;
group: g {
types:
compound cmpd_t {
vlen_t f1 ;
enum_t f2 ;
enum_t f3 ;
}; // cmpd_t
} // group g
group: h {
variables:
/g/cmpd_t compoundvar ;
data:
compoundvar = {{3, 4, 5}, Stratus, Clear} ;
} // group h
}

6
ncdump/tst_ncgen4_cycle.sh
View File

@ -31,9 +31,15 @@ for x in ${TESTSET} ; do
for t in ${XFAILTESTS} ; do
if test "x${t}" = "x${x}" ; then isxfail=1; fi
done
isnocycle=0
for t in ${NOCYCLE} ; do
if test "x${t}" = "x${x}" ; then isnocycle=1; fi
done
if test "${isxfail}" = "1"; then
echo "xfail test: ${x}: ignored"
xfailcount=`expr $xfailcount + 1`
elif test "${isnocycle}" = "1"; then
echo "test: ${x}: ignored for cycle test"
else
rm -f ${x}_$$.nc ${x}_$$.dmp
# step 1: use original cdl to build the .nc

5
ncdump/tst_ncgen_shared.sh
View File

@ -97,12 +97,17 @@ unlimtest2 \
ref_niltest \
ref_tst_h_scalar \
ref_tst_nul4 \
ref_tst_econst \
ref_tst_econst2 \
"
if test "x$NC_VLEN_NOTEST" = x ; then
TESTS4="$TESTS4 ref_tst_vlen_data ref_tst_vlen_data2"
fi
# These tests should not be cycle tested because ncdump loses information
NOCYCLE="ref_tst_econst"
SPECIALTESTS3="ref_tst_special_atts3"
SPECIALTESTS="${SPECIALTESTS3} ref_tst_special_atts"

157
ncgen/semantics.c
View File

@ -34,7 +34,6 @@ static void inferattributetype(Symbol* asym);
static void validateNIL(Symbol* sym);
static void checkconsistency(void);
static int tagvlentypes(Symbol* tsym);
static void computefqns(void);
static Symbol* uniquetreelocate(Symbol* refsym, Symbol* root);
static char* createfilename(void);
@ -468,61 +467,124 @@ processeconstrefsR(Symbol* avsym, Datalist* data)
if(con->nctype == NC_COMPOUND) {
/* Iterate over the sublists */
processeconstrefsR(avsym,con->value.compoundv);
} else if(con->nctype == NC_ECONST || con->nctype == NC_FILLVALUE) {
} else if(con->nctype == NC_ECONST) {
fixeconstref(avsym,con);
}
}
}
/*
Collect all types in all groups using preorder depth-first traversal.
*/
static void
typewalk(Symbol* root, nc_type typ, List* list)
{
unsigned long i;
for(i=0;i<listlength(root->subnodes);i++) {
Symbol* sym = (Symbol*)listget(root->subnodes,i);
if(sym->objectclass == NC_GRP) {
typewalk(sym,typ,list);
} else if(sym->objectclass == NC_TYPE && (typ == NC_NAT || typ == sym->subclass)) {
if(!listcontains(list,sym))
listpush(list,sym);
}
}
}
/* Find all user-define types of type typ in access order */
static void
orderedtypes(Symbol* avsym, nc_type typ, List* types)
{
Symbol* container = NULL;
listclear(types);
/* find innermost containing group */
if(avsym->objectclass == NC_VAR) {
container = avsym->container;
} else {
ASSERT(avsym->objectclass == NC_ATT);
container = avsym->container;
if(container->objectclass == NC_VAR)
container = container->container;
}
/* walk up the containing groups and collect type */
for(;container!= NULL;container = container->container) {
int i;
/* Walk types in the container */
for(i=0;i<listlength(container->subnodes);i++) {
Symbol* sym = (Symbol*)listget(container->subnodes,i);
if(sym->objectclass == NC_TYPE && (typ == NC_NAT || sym->subclass == typ))
listpush(types,sym);
}
}
/* Now do all-tree search */
typewalk(rootgroup,typ,types);
}
static Symbol*
locateeconst(Symbol* enumt, const char* ecname)
{
int i;
for(i=0;i<listlength(enumt->subnodes);i++) {
Symbol* esym = (Symbol*)listget(enumt->subnodes,i);
ASSERT(esym->subclass == NC_ECONST);
if(strcmp(esym->name,ecname)==0)
return esym;
}
return NULL;
}
static Symbol*
findeconstenum(Symbol* avsym, NCConstant* con)
{
int i;
Symbol* refsym = con->value.enumv;
List* typdefs = listnew();
Symbol* enumt = NULL;
Symbol* candidate = NULL; /* possible enum type */
Symbol* econst = NULL;
char* path = NULL;
char* name = NULL;
/* get all enum types */
orderedtypes(avsym,NC_ENUM,typdefs);
/* It is possible that the enum const is prefixed with the type name */
path = strchr(refsym->name,'.');
if(path != NULL) {
path = strdup(refsym->name);
name = strchr(path,'.');
*name++ = '\0';
} else
name = refsym->name;
/* See if we can find the enum type */
for(i=0;i<listlength(typdefs);i++) {
Symbol* sym = (Symbol*)listget(typdefs,i);
ASSERT(sym->objectclass == NC_TYPE && sym->subclass == NC_ENUM);
if(path != NULL && strcmp(sym->name,path)==0) {enumt = sym; break;}
/* See if enum has a matching econst */
econst = locateeconst(sym,name);
if(candidate == NULL && econst != NULL) candidate = sym; /* remember this */
}
if(enumt != NULL) goto done;
/* otherwise use the candidate */
enumt = candidate;
done:
if(enumt) econst = locateeconst(enumt,name);
nullfree(path);
if(econst == NULL)
semerror(con->lineno,"Undefined enum constant: %s",refsym->name);
return econst;
}
static void
fixeconstref(Symbol* avsym, NCConstant* con)
{
Symbol* basetype = NULL;
Symbol* refsym = con->value.enumv;
Symbol* varsym = NULL;
int i;
Symbol* econst = NULL;
/* Figure out the proper type associated with avsym */
ASSERT(avsym->objectclass == NC_VAR || avsym->objectclass == NC_ATT);
if(avsym->objectclass == NC_VAR) {
basetype = avsym->typ.basetype;
varsym = avsym;
} else { /*(avsym->objectclass == NC_ATT)*/
basetype = avsym->typ.basetype;
varsym = avsym->container;
if(varsym->objectclass == NC_GRP)
varsym = NULL;
}
/* If this is a non-econst fillvalue, then ignore it */
if(con->nctype == NC_FILLVALUE && basetype->subclass != NC_ENUM)
return;
/* If this is an econst then validate against type */
if(con->nctype == NC_ECONST && basetype->subclass != NC_ENUM)
semerror(con->lineno,"Enumconstant associated with a non-econst type");
if(con->nctype == NC_FILLVALUE) {
Datalist* filllist = NULL;
NCConstant* filler = NULL;
filllist = getfiller(varsym == NULL?basetype:varsym);
if(filllist == NULL)
semerror(con->lineno, "Cannot determine enum constant fillvalue");
filler = datalistith(filllist,0);
con->value.enumv = filler->value.enumv;
return;
}
for(i=0;i<listlength(basetype->subnodes);i++) {
Symbol* econst = listget(basetype->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
if(strcmp(econst->name,refsym->name)==0) {
con->value.enumv = econst;
return;
}
}
semerror(con->lineno,"Undefined enum or enum constant reference: %s",refsym->name);
econst = findeconstenum(avsym,con);
assert(econst != NULL && econst->subclass == NC_ECONST);
con->value.enumv = econst;
}
/* Compute type sizes and compound offsets*/
@ -1131,7 +1193,6 @@ processunlimiteddims(void)
#endif
}
/* Rules for specifying the dataset name:
1. use -o name
2. use the datasetname from the .cdl file