diff --git a/include/nc4internal.h b/include/nc4internal.h index e2686c0e9..8310f54f0 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -343,8 +343,9 @@ extern int NC4_lookup_atomic_type(const char *name, nc_type* idp, size_t *sizep) /* These functions convert between netcdf and HDF5 types. */ extern int nc4_get_typelen_mem(NC_FILE_INFO_T *h5, nc_type xtype, size_t *len); extern int nc4_convert_type(const void *src, void *dest, const nc_type src_type, - const nc_type dest_type, const size_t len, int *range_error, - const void *fill_value, int strict_nc3); + const nc_type dest_type, const size_t len, int *range_error, + const void *fill_value, int strict_nc3, int quantize_mode, + int nsd); /* These functions do HDF5 things. */ extern int nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); diff --git a/libhdf4/hdf4var.c b/libhdf4/hdf4var.c index 25a419079..a8923df77 100644 --- a/libhdf4/hdf4var.c +++ b/libhdf4/hdf4var.c @@ -88,7 +88,7 @@ NC_HDF4_get_vara(int ncid, int varid, const size_t *startp, if (var->type_info->hdr.id != memtype) { if ((retval = nc4_convert_type(data, ip, var->type_info->hdr.id, memtype, nelem, - &range_error, NULL, 0))) + &range_error, NULL, 0, NC_NOQUANTIZE, 0))) return retval; free(data); if (range_error) diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 56b5edb37..d8418519a 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -716,7 +716,8 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* Data types are like religions, in that one can convert. */ if ((retval = nc4_convert_type(data, att->data, mem_type, file_type, len, &range_error, NULL, - (h5->cmode & NC_CLASSIC_MODEL)))) + (h5->cmode & NC_CLASSIC_MODEL), + NC_NOQUANTIZE, 0))) BAIL(retval); } } diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 9ceef3504..4a7177915 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1718,7 +1718,8 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, { if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id, len, &range_error, var->fill_value, - (h5->cmode & NC_CLASSIC_MODEL)))) + (h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, + var->nsd))) BAIL(retval); } @@ -2119,8 +2120,8 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, if (need_to_convert) { if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type, - len, &range_error, var->fill_value, - (h5->cmode & NC_CLASSIC_MODEL)))) + len, &range_error, var->fill_value, + (h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, var->nsd))) BAIL(retval); /* For strict netcdf-3 rules, ignore erange errors between UBYTE diff --git a/libnczarr/zattr.c b/libnczarr/zattr.c index 679a3f860..f41249faa 100644 --- a/libnczarr/zattr.c +++ b/libnczarr/zattr.c @@ -714,7 +714,8 @@ ncz_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type, /* Data types are like religions, in that one can convert. */ if ((retval = nc4_convert_type(data, att->data, mem_type, file_type, len, &range_error, NULL, - (h5->cmode & NC_CLASSIC_MODEL)))) + (h5->cmode & NC_CLASSIC_MODEL), + NC_NOQUANTIZE, 0))) BAIL(retval); } } diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 7ba84f968..cc16e7954 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -1527,7 +1527,8 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, { if ((retval = nc4_convert_type(data, bufr, mem_nc_type, var->type_info->hdr.id, len, &range_error, var->fill_value, - (h5->cmode & NC_CLASSIC_MODEL)))) + (h5->cmode & NC_CLASSIC_MODEL), + var->quantize_mode, var->nsd))) BAIL(retval); } @@ -1902,7 +1903,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, { if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type, len, &range_error, var->fill_value, - (h5->cmode & NC_CLASSIC_MODEL)))) + (h5->cmode & NC_CLASSIC_MODEL), var->quantize_mode, + var->nsd))) BAIL(retval); /* For strict netcdf-3 rules, ignore erange errors between UBYTE * and BYTE types. */ diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index 6c9459071..b84b51819 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -116,7 +116,8 @@ nc4_get_att_ptrs(NC_FILE_INFO_T *h5, NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, need_to_convert++; if ((retval = nc4_convert_type(att->data, bufr, att->nc_typeid, mem_type, (size_t)att->len, &range_error, - NULL, (h5->cmode & NC_CLASSIC_MODEL)))) + NULL, (h5->cmode & NC_CLASSIC_MODEL), + NC_NOQUANTIZE, 0))) BAIL(retval); /* For strict netcdf-3 rules, ignore erange errors between UBYTE diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index 559e84dea..9adf7c50c 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -448,6 +448,11 @@ NC4_var_par_access(int ncid, int varid, int par_access) * value used (or the default fill value if none is supplied) for * values that overflow the type. * + * This function applies quantization to float and double data, if + * desired. The code to do this is derived from the bitgroom filter in + * the CCR project (see + * https://github.com/ccr/ccr/blob/master/hdf5_plugins/BITGROOM/src/H5Zbitgroom.c). + * * @param src Pointer to source of data. * @param dest Pointer that gets data. * @param src_type Type ID of source data. @@ -456,15 +461,20 @@ NC4_var_par_access(int ncid, int varid, int par_access) * @param range_error Pointer that gets 1 if there was a range error. * @param fill_value The fill value. * @param strict_nc3 Non-zero if strict model in effect. + * @param quantize_mode May be ::NC_NOQUANTIZE or + * ::NC_QUANTIZE_BITGROOM. + * @param nsd Number of significant diggits for quantizize. Ignored + * otherwise. * - * @returns NC_NOERR No error. - * @returns NC_EBADTYPE Type not found. + * @returns ::NC_NOERR No error. + * @returns ::NC_EBADTYPE Type not found. * @author Ed Hartnett, Dennis Heimbigner */ int nc4_convert_type(const void *src, void *dest, const nc_type src_type, const nc_type dest_type, const size_t len, int *range_error, - const void *fill_value, int strict_nc3) + const void *fill_value, int strict_nc3, int quantize_mode, + int nsd) { char *cp, *cp1; float *fp, *fp1; @@ -557,8 +567,8 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type, } break; case NC_FLOAT: - for (bp = (signed char *)src, fp = dest; count < len; count++) - *fp++ = *bp++; + for (bp = (signed char *)src, fp = dest; count < len; count++) + *fp++ = *bp++; break; case NC_DOUBLE: for (bp = (signed char *)src, dp = dest; count < len; count++) @@ -1133,16 +1143,34 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type, } break; case NC_FLOAT: - for (fp = (float *)src, fp1 = dest; count < len; count++) - { - /* if (*fp > X_FLOAT_MAX || *fp < X_FLOAT_MIN) - (*range_error)++;*/ - *fp1++ = *fp++; - } + if (quantize_mode == NC_QUANTIZE_BITGROOM) + { + for (fp = (float *)src, fp1 = dest; count < len; count++) + { + *fp1 = *fp; + + /* Move to next float. */ + fp1++; + fp++; + } + } + else + { + for (fp = (float *)src, fp1 = dest; count < len; count++) + *fp1++ = *fp++; + } break; case NC_DOUBLE: - for (fp = (float *)src, dp = dest; count < len; count++) - *dp++ = *fp++; + if (quantize_mode == NC_QUANTIZE_BITGROOM) + { + for (fp = (float *)src, dp = dest; count < len; count++) + *dp++ = *fp++; + } + else + { + for (fp = (float *)src, dp = dest; count < len; count++) + *dp++ = *fp++; + } break; default: LOG((0, "%s: unexpected dest type. src_type %d, dest_type %d", @@ -1219,20 +1247,36 @@ nc4_convert_type(const void *src, void *dest, const nc_type src_type, } break; case NC_FLOAT: - for (dp = (double *)src, fp = dest; count < len; count++) - { - if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN)) - (*range_error)++; - *fp++ = *dp++; - } + if (quantize_mode == NC_QUANTIZE_BITGROOM) + { + for (dp = (double *)src, fp = dest; count < len; count++) + { + if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN)) + (*range_error)++; + *fp++ = *dp++; + } + } + else + { + for (dp = (double *)src, fp = dest; count < len; count++) + { + if (isgreater(*dp, X_FLOAT_MAX) || isless(*dp, X_FLOAT_MIN)) + (*range_error)++; + *fp++ = *dp++; + } + } break; case NC_DOUBLE: - for (dp = (double *)src, dp1 = dest; count < len; count++) - { - /* if (*dp > X_DOUBLE_MAX || *dp < X_DOUBLE_MIN) */ - /* (*range_error)++; */ - *dp1++ = *dp++; - } + if (quantize_mode == NC_QUANTIZE_BITGROOM) + { + for (dp = (double *)src, dp1 = dest; count < len; count++) + *dp1++ = *dp++; + } + else + { + for (dp = (double *)src, dp1 = dest; count < len; count++) + *dp1++ = *dp++; + } break; default: LOG((0, "%s: unexpected dest type. src_type %d, dest_type %d",