mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

perparing to apply bitgroom algorithm

Browse Source
This commit is contained in:
Edward Hartnett 2021-08-25 01:31:26 -06:00
parent c9eca4bbca
commit 0f26083f4d
8 changed files with 87 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
include/nc4internal.h
View File

@ -343,8 +343,9 @@ extern int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep)
/* These functions convert between netcdf and HDF5 types. */
extern int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len);
extern int nc4_convert_type(const void *src, void *dest, const nc_type src_type,
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3);
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3, int quantize_mode,
int nsd);
/* These functions do HDF5 things. */
extern int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);

2
libhdf4/hdf4var.c
View File

@ -88,7 +88,7 @@ NC_HDF4_get_vara(int ncid, int varid, const size_t *startp,
if (var->type_info->hdr.id != memtype)
{
if ((retval = nc4_convert_type(data, ip, var->type_info->hdr.id, memtype, nelem,
&range_error, NULL, 0)))
&range_error, NULL, 0, NC_NOQUANTIZE, 0)))
return retval;
free(data);
if (range_error)

3
libhdf5/hdf5attr.c
View File

@ -716,7 +716,8 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Data types are like religions, in that one can convert. */
if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,
len, &range_error, NULL,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
}
}

7
libhdf5/hdf5var.c
View File

@ -1718,7 +1718,8 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode,
var->nsd)))
BAIL(retval);
}
@ -2119,8 +2120,8 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, var->nsd)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE

3
libnczarr/zattr.c
View File

@ -714,7 +714,8 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
/* Data types are like religions, in that one can convert. */
if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,
len, &range_error, NULL,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
}
}

6
libnczarr/zvar.c
View File

@ -1527,7 +1527,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL),
var->quantize_mode, var->nsd)))
BAIL(retval);
}
@ -1902,7 +1903,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp,
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
len, &range_error, var->fill_value,
(h5->cmode & NC_CLASSIC_MODEL))))
(h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode,
var->nsd)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */

3
libsrc4/nc4attr.c
View File

@ -116,7 +116,8 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var,
need_to_convert++;
if ((retval = nc4_convert_type(att->data, bufr, att->nc_typeid,
mem_type, (size_t)att->len, &range_error,
NULL, (h5->cmode & NC_CLASSIC_MODEL))))
NULL, (h5->cmode & NC_CLASSIC_MODEL),
NC_NOQUANTIZE, 0)))
BAIL(retval);
/* For strict netcdf-3 rules, ignore erange errors between UBYTE

94
libsrc4/nc4var.c
View File

@ -448,6 +448,11 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* value used (or the default fill value if none is supplied) for
* values that overflow the type.
*
* This function applies quantization to float and double data, if
* desired. The code to do this is derived from the bitgroom filter in
* the CCR project (see
* https://github.com/ccr/ccr/blob/master/hdf5_plugins/BITGROOM/src/H5Zbitgroom.c).
*
* @param src Pointer to source of data.
* @param dest Pointer that gets data.
* @param src_type Type ID of source data.
@ -456,15 +461,20 @@ NC4_var_par_access(int ncid, int varid, int par_access)
* @param range_error Pointer that gets 1 if there was a range error.
* @param fill_value The fill value.
* @param strict_nc3 Non-zero if strict model in effect.
* @param quantize_mode May be ::NC_NOQUANTIZE or
* ::NC_QUANTIZE_BITGROOM.
* @param nsd Number of significant diggits for quantizize. Ignored
* otherwise.
*
* @returns NC_NOERR No error.
* @returns NC_EBADTYPE Type not found.
* @returns ::NC_NOERR No error.
* @returns ::NC_EBADTYPE Type not found.
* @author Ed Hartnett, Dennis Heimbigner
*/
int
nc4_convert_type(const void *src, void *dest, const nc_type src_type,
const nc_type dest_type, const size_t len, int *range_error,
const void *fill_value, int strict_nc3)
const void *fill_value, int strict_nc3, int quantize_mode,
int nsd)
{
char *cp, *cp1;
float *fp, *fp1;
@ -557,8 +567,8 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
break;
case NC_FLOAT:
for (bp = (signed char *)src, fp = dest; count < len; count++)
*fp++ = *bp++;
for (bp = (signed char *)src, fp = dest; count < len; count++)
*fp++ = *bp++;
break;
case NC_DOUBLE:
for (bp = (signed char *)src, dp = dest; count < len; count++)
@ -1133,16 +1143,34 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
break;
case NC_FLOAT:
for (fp = (float *)src, fp1 = dest; count < len; count++)
{
/* if (*fp > X_FLOAT_MAX || *fp < X_FLOAT_MIN)
(*range_error)++;*/
*fp1++ = *fp++;
}
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
for (fp = (float *)src, fp1 = dest; count < len; count++)
{
*fp1 = *fp;
/* Move to next float. */
fp1++;
fp++;
}
}
else
{
for (fp = (float *)src, fp1 = dest; count < len; count++)
*fp1++ = *fp++;
}
break;
case NC_DOUBLE:
for (fp = (float *)src, dp = dest; count < len; count++)
*dp++ = *fp++;
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
for (fp = (float *)src, dp = dest; count < len; count++)
*dp++ = *fp++;
}
else
{
for (fp = (float *)src, dp = dest; count < len; count++)
*dp++ = *fp++;
}
break;
default:
LOG((0, "%s: unexpected dest type. src_type %d, dest_type %d",
@ -1219,20 +1247,36 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type,
}
break;
case NC_FLOAT:
for (dp = (double *)src, fp = dest; count < len; count++)
{
if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN))
(*range_error)++;
*fp++ = *dp++;
}
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
for (dp = (double *)src, fp = dest; count < len; count++)
{
if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN))
(*range_error)++;
*fp++ = *dp++;
}
}
else
{
for (dp = (double *)src, fp = dest; count < len; count++)
{
if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN))
(*range_error)++;
*fp++ = *dp++;
}
}
break;
case NC_DOUBLE:
for (dp = (double *)src, dp1 = dest; count < len; count++)
{
/* if (*dp > X_DOUBLE_MAX || *dp < X_DOUBLE_MIN) */
/* (*range_error)++; */
*dp1++ = *dp++;
}
if (quantize_mode == NC_QUANTIZE_BITGROOM)
{
for (dp = (double *)src, dp1 = dest; count < len; count++)
*dp1++ = *dp++;
}
else
{
for (dp = (double *)src, dp1 = dest; count < len; count++)
*dp1++ = *dp++;
}
break;
default:
LOG((0, "%s: unexpected dest type. src_type %d, dest_type %d",