mirror of
https://github.com/Unidata/netcdf-c.git
synced 2025-03-25 17:40:27 +08:00
Fix some bugs in the blosc filter wrapper
re: Issue https://github.com/Unidata/netcdf-c/issues/2458 The above Github Issue revealed some bugs in the file netcdf-c/plugins/H5Zblosc.c. Fixed and added a testcase. Also discovered that the Blosc LZ sub-compressors do not work well with small datasets. Misc. Other Change(s): I noticed that the file "dap4_test/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds" is still causing tar errors during "make distcheck", so I made some changes to do rename at test-time.
This commit is contained in:
Dennis Heimbigner
parent
64033e367d
commit
3623e17920
@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
|
||||
|
||||
## 4.9.1 - T.B.D.
|
||||
|
||||
* [Bug Fix] Fix blosc plugin errors. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
|
||||
* [Bug Fix] Fix quantize with CLASSIC_MODEL files. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2445).
|
||||
* [Enhancement] Add `--disable-quantize` option to `configure`.
|
||||
* [Bug Fix] Fix CMakeLists.txt to handle all acceptable boolean values for -DPLUGIN_INSTALL_DIR. See [Github #2430](https://github.com/Unidata/netcdf-c/pull/2430).
|
||||
|
||||
@ -95,7 +95,6 @@ AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat3.nc:nc_test4/ref_hdf5_compat3.nc])
|
||||
|
||||
AC_CONFIG_LINKS([hdf4_test/ref_chunked.hdf4:hdf4_test/ref_chunked.hdf4])
|
||||
AC_CONFIG_LINKS([hdf4_test/ref_contiguous.hdf4:hdf4_test/ref_contiguous.hdf4])
|
||||
AC_CONFIG_LINKS([dap4_test/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds:dap4_test/baselinethredds/GOES16_TEST1.nc4.thredds])
|
||||
AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects])
|
||||
|
||||
# Check for the existence of this file before proceeding.
|
||||
|
||||
@ -59,7 +59,8 @@ EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
|
||||
CLEANFILES = *.exe
|
||||
# This should only be left behind if using parallel io
|
||||
CLEANFILES += tmp_*
|
||||
|
||||
CLEANFILES += ${execdir}/baselinethredds/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds
|
||||
|
||||
DISTCLEANFILES = findtestserver4.c pingurl4.c
|
||||
|
||||
# One last thing
|
||||
|
||||
@ -5,17 +5,19 @@
|
||||
|
||||
# See netcdf-c/COPYRIGHT file for more info.
|
||||
|
||||
|
||||
FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
|
||||
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
|
||||
#FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
#FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
|
||||
#FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
|
||||
|
||||
##
|
||||
# Rename file in support of https://github.com/Unidata/netcdf-c/issues/2077
|
||||
##
|
||||
IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_TEST1.nc4.thredds DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
ENDIF()
|
||||
#IF(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
#FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/GOES16_TEST1.nc4.thredds DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds)
|
||||
#ENDIF()
|
||||
|
||||
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
|
||||
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
|
||||
|
||||
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
|
||||
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt)
|
||||
|
||||
@ -13,6 +13,14 @@ echo "test_thredds.sh:"
|
||||
|
||||
FRAG="#checksummode=ignore"
|
||||
|
||||
# The name GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4 is too long during distcheck,
|
||||
# so rename on the fly
|
||||
GOESSHORT=${srcdir}/baselinethredds/GOES16_TEST1.nc4.thredds
|
||||
GOESLONG=${BASELINETH}/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4.thredds
|
||||
if ! test -f "${GOESLONG}" ; then
|
||||
cp "${GOESSHORT}" "${GOESLONG}"
|
||||
fi
|
||||
|
||||
F="\
|
||||
harvey/goes16/CONUS/Channel01/20170821/GOES16_CONUS_20170821_020218_0.47_1km_33.3N_91.4W.nc4?dap4.ce=y \
|
||||
"
|
||||
@ -44,6 +52,8 @@ for f in $F ; do
|
||||
fi
|
||||
done
|
||||
|
||||
#rm -fr "${GOESLONG}"
|
||||
|
||||
echo "*** Pass"
|
||||
exit 0
|
||||
|
||||
|
||||
@ -377,13 +377,10 @@ ncz_sync_netcdf4_file(NC_FILE_INFO_T* file, int isclose)
|
||||
LOG((3, "%s", __func__));
|
||||
ZTRACE(2,"file=%s",file->hdr.name);
|
||||
|
||||
/* If we're in define mode, that's an error, for strict nc3 rules,
|
||||
* otherwise, end define mode. */
|
||||
/* End depend mode if needed. (Error checking for classic mode has
|
||||
* already happened). */
|
||||
if (file->flags & NC_INDEF)
|
||||
{
|
||||
if (file->cmode & NC_CLASSIC_MODEL)
|
||||
return NC_EINDEFINE;
|
||||
|
||||
/* Turn define mode off. */
|
||||
file->flags ^= NC_INDEF;
|
||||
|
||||
|
||||
@ -52,9 +52,7 @@ IF(USE_HDF5 AND ENABLE_FILTER_TESTING)
|
||||
ADD_SH_TEST(nc_test4 tst_filter)
|
||||
ADD_SH_TEST(nc_test4 tst_unknown)
|
||||
ADD_SH_TEST(nc_test4 tst_specific_filters)
|
||||
IF(ENABLE_CLIENTSIDE_FILTERS)
|
||||
add_bin_test(nc_test4 test_filter_reg)
|
||||
ENDIF(ENABLE_CLIENTSIDE_FILTERS)
|
||||
ADD_SH_TEST(nc_test4 tst_bloscfail)
|
||||
ENDIF(USE_HDF5 AND ENABLE_FILTER_TESTING)
|
||||
|
||||
ENDIF(BUILD_UTILITIES)
|
||||
|
||||
@ -85,6 +85,7 @@ check_PROGRAMS += tst_multifilter tst_filter_avail
|
||||
TESTS += tst_filter.sh
|
||||
TESTS += tst_unknown.sh
|
||||
TESTS += tst_specific_filters.sh
|
||||
TESTS += tst_bloscfail.sh
|
||||
endif
|
||||
endif # USE_HDF5
|
||||
endif # BUILD_UTILITIES
|
||||
@ -112,7 +113,7 @@ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
|
||||
ref_filter_order_create.txt ref_filter_order_read.txt \
|
||||
ref_any.cdl tst_specific_filters.sh tst_unknown.sh \
|
||||
tst_virtual_datasets.c noop1.cdl unknown.cdl \
|
||||
tst_broken_files.c
|
||||
tst_broken_files.c ref_bloscx.cdl tst_bloscfail.sh
|
||||
|
||||
# The tst_filterinstall test can only be run after an install
|
||||
# occurred with --with-plugin-dir enabled. So there is no point
|
||||
|
||||
35
nc_test4/ref_bloscx.cdl
Normal file
35
nc_test4/ref_bloscx.cdl
Normal file
@ -0,0 +1,35 @@
|
||||
netcdf x3 {
|
||||
dimensions:
|
||||
time = 10 ;
|
||||
lat = 2 ;
|
||||
lon = 4 ;
|
||||
variables:
|
||||
float three_dmn_rec_var(time, lat, lon) ;
|
||||
three_dmn_rec_var:long_name = "three dimensional record variable" ;
|
||||
three_dmn_rec_var:units = "watt meter-2" ;
|
||||
three_dmn_rec_var:_FillValue = -99.f ;
|
||||
data:
|
||||
|
||||
three_dmn_rec_var =
|
||||
1, 2, 3, 4,
|
||||
5, 6, 7, 8,
|
||||
9, 10, 11, 12,
|
||||
13, 14, 15, 16,
|
||||
17, 18, 19, 20,
|
||||
21, 22, 23, 24,
|
||||
25, 26, 27, 28,
|
||||
29, 30, 31, 32,
|
||||
33, 34, 35, 36,
|
||||
37, 38, 39, 40,
|
||||
41, 42, 43, 44,
|
||||
45, 46, 47, 48,
|
||||
49, 50, 51, 52,
|
||||
53, 54, 55, 56,
|
||||
57, 58, 59, 60,
|
||||
61, 62, 63, 64,
|
||||
65, 66, 67, 68,
|
||||
69, 70, 71, 72,
|
||||
73, 74, 75, 76,
|
||||
77, 78, 79, 80 ;
|
||||
|
||||
}
|
||||
70
nc_test4/tst_bloscfail.sh
Executable file
70
nc_test4/tst_bloscfail.sh
Executable file
@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -e
|
||||
|
||||
# Load the findplugins function
|
||||
. ${builddir}/findplugin.sh
|
||||
echo "findplugin.sh loaded"
|
||||
|
||||
# Function to remove selected -s attributes from file;
|
||||
# These attributes might be platform dependent
|
||||
sclean() {
|
||||
cat $1 \
|
||||
| sed -e '/:_IsNetcdf4/d' \
|
||||
| sed -e '/:_Endianness/d' \
|
||||
| sed -e '/_NCProperties/d' \
|
||||
| sed -e '/_SuperblockVersion/d' \
|
||||
| cat > $2
|
||||
}
|
||||
|
||||
# Function to extract _Filter attribute from a file
|
||||
# These attributes might be platform dependent
|
||||
getfilterattr() {
|
||||
sed -e '/var.*:_Filter/p' -ed <$1 >$2
|
||||
}
|
||||
|
||||
trimleft() {
|
||||
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
|
||||
}
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
. "$srcdir/test_nczarr.sh"
|
||||
fi
|
||||
|
||||
if ! avail blosc; then echo "Blosc compressor not found"; exit 0; fi
|
||||
|
||||
# Locate the plugin dir and the library names; argument order is critical
|
||||
# Find bzip2 and capture
|
||||
findplugin h5blosc
|
||||
BLOSCLIB="${HDF5_PLUGIN_LIB}"
|
||||
BLOSCDIR="${HDF5_PLUGIN_DIR}/${BZIP2LIB}"
|
||||
|
||||
echo "final HDF5_PLUGIN_DIR=${HDF5_PLUGIN_DIR}"
|
||||
export HDF5_PLUGIN_DIR
|
||||
export HDF5_PLUGIN_PATH="$HDF5_PLUGIN_DIR"
|
||||
|
||||
localclean() {
|
||||
rm -f ./tmp_bloscx3.nc ./tmp_bloscx4.nc ./tmp_bloscx4_fail.nc
|
||||
}
|
||||
|
||||
# Execute the specified tests
|
||||
|
||||
echo "*** Testing dynamic filters using API"
|
||||
localclean
|
||||
${NCGEN} -3 -o tmp_bloscx3.nc ${srcdir}/ref_bloscx.cdl
|
||||
# This should pass
|
||||
${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,1,0 ./tmp_bloscx3.nc ./tmp_bloscx4.nc
|
||||
# This should fail because shuffle is off
|
||||
if ${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,0,0 ./tmp_bloscx3.nc ./tmp_bloscx4_fail.nc ; then
|
||||
echo "*** not xfail: nccopy "
|
||||
exit 1;
|
||||
else
|
||||
echo "*** xfail: nccopy "
|
||||
fi
|
||||
|
||||
localclean
|
||||
|
||||
exit 0
|
||||
@ -8,17 +8,13 @@
|
||||
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
|
||||
. ../test_common.sh
|
||||
|
||||
set -x
|
||||
set -e
|
||||
|
||||
if test "x$TESTNCZARR" = x1 ; then
|
||||
. "$srcdir/test_nczarr.sh"
|
||||
BLOSCARGS="32001,0,0,0,256,5,1,1"
|
||||
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
|
||||
else
|
||||
BLOSCARGS="32001,2,2,4,256,5,1,1"
|
||||
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
|
||||
fi
|
||||
BLOSCARGS="32001,0,0,4,256,5,1,1"
|
||||
BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]'
|
||||
|
||||
# Load the findplugins function
|
||||
. ${builddir}/findplugin.sh
|
||||
|
||||
@ -418,7 +418,7 @@ main(int argc, char **argv)
|
||||
if (nc_close(ncid)) ERR;
|
||||
}
|
||||
SUMMARIZE_ERR;
|
||||
#ifdef HAVE_SZ
|
||||
#ifdef HAVE_H5Z_SZIP
|
||||
printf("**** testing simple szip filter setup...");
|
||||
{
|
||||
int ncid;
|
||||
@ -748,9 +748,9 @@ main(int argc, char **argv)
|
||||
params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */
|
||||
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;
|
||||
{ int stat;
|
||||
if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN,
|
||||
params)) != NC_ENOFILTER)
|
||||
ERR;
|
||||
stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, params);
|
||||
if(stat != NC_ENOFILTER)
|
||||
ERR;
|
||||
}
|
||||
if (nc_def_var_szip(ncid, varid, NC_SZIP_NN,
|
||||
NC_SZIP_EC_BPP_IN) != NC_ENOFILTER) ERR;
|
||||
|
||||
@ -87,6 +87,18 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
*/
|
||||
|
||||
/*
|
||||
HDF5 Blosc filter takes 7 parameters as follows:
|
||||
param[0] -- FILTER_BLOSC_VERSION (ignored)
|
||||
param[1] -- BLOSC_VERSION_FORMAT (ignored)
|
||||
param[2] -- typesize in bytes
|
||||
param[3] -- chunksize in bytes (ignored)
|
||||
param[4] -- compression level
|
||||
param[5] -- shuffle enabled
|
||||
param[6] -- compressor to use
|
||||
*/
|
||||
|
||||
|
||||
#ifdef HAVE_CONFIG_H
|
||||
#include "config.h"
|
||||
#endif
|
||||
@ -105,25 +117,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
#include <H5Epubgen.h>
|
||||
#endif
|
||||
|
||||
#define BLOSCCTX
|
||||
#undef BLOSC_DEBUG
|
||||
|
||||
#ifdef USE_HDF5
|
||||
#if defined(__GNUC__)
|
||||
#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str)
|
||||
#elif defined(_MSC_VER)
|
||||
#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str)
|
||||
#else
|
||||
/* This version is portable but it's better to use compiler-supported
|
||||
approaches for handling the trailing comma issue when possible. */
|
||||
#define PUSH_ERR(func, minor, str, ...) H5Epush1(__FILE__, func, __LINE__, H5_VERS_MAJOR, minor, str
|
||||
)
|
||||
#endif /* defined(__GNUC__) */
|
||||
|
||||
#define GET_FILTER(a, b, c, d, e, f, g) H5Pget_filter_by_id1(a,b,c,d,e,f,g)
|
||||
|
||||
#else /*!USE_HDF5*/
|
||||
#define PUSH_ERR(f,m,s,...) fprintf(stderr,"%s\n",s)
|
||||
#endif /*USE_HDF5*/
|
||||
#undef BLOSCCTX
|
||||
|
||||
static size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
const unsigned cd_values[], size_t nbytes,
|
||||
@ -147,37 +143,40 @@ static herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space);
|
||||
static
|
||||
herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
|
||||
{
|
||||
int ndims;
|
||||
int i;
|
||||
herr_t r;
|
||||
|
||||
int i,ndims;
|
||||
unsigned int typesize, basetypesize;
|
||||
unsigned int bufsize;
|
||||
hsize_t chunkdims[32];
|
||||
unsigned int flags;
|
||||
size_t nelements = 8;
|
||||
unsigned int values[] = {0, 0, 0, 0, 0, 0, 0, 0};
|
||||
unsigned int flags = 0;
|
||||
size_t nelements = 7;
|
||||
unsigned int values[] = {0, 0, 0, 0, 0, 0, 0};
|
||||
hid_t super_type;
|
||||
H5T_class_t classt;
|
||||
hsize_t chunkdims[32];
|
||||
|
||||
r = GET_FILTER(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL);
|
||||
if (r < 0) return -1;
|
||||
assert(sizeof(hid_t) == 8);
|
||||
|
||||
if(H5Pget_filter_by_id1(dcpl, H5Z_FILTER_BLOSC, &flags, &nelements, values, 0, NULL) < 0) goto failed;
|
||||
|
||||
if (nelements < 4) nelements = 4; /* First 4 slots reserved. */
|
||||
assert(nelements == 7);
|
||||
|
||||
/* Set Blosc info in first two slots */
|
||||
values[0] = FILTER_BLOSC_VERSION;
|
||||
values[1] = BLOSC_VERSION_FORMAT;
|
||||
#ifdef BLOSC_DEBUG
|
||||
fprintf(stderr, "Blosc: initial parameters =(%u) [%u %u %u %u %u %u %u]\n",
|
||||
(unsigned)nelements,values[0],values[1],values[2],values[3],values[4],values[5],values[6]); fflush(stderr);
|
||||
#endif
|
||||
|
||||
/* Ignore first two slots */
|
||||
values[0] = 0; /* FILTER_BLOSC_VERSION*/
|
||||
values[1] = 0; /*BLOSC_VERSION_FORMAT*/
|
||||
|
||||
ndims = H5Pget_chunk(dcpl, 32, chunkdims);
|
||||
if (ndims < 0) return -1;
|
||||
if (ndims < 0) goto failed;
|
||||
if (ndims > 32) {
|
||||
PUSH_ERR("blosc_set_local", H5E_CALLBACK, "Chunk rank exceeds limit");
|
||||
return -1;
|
||||
fprintf(stderr,"Blosc Filter Error: blosc_set_local: Chunk rank exceeds limit\n");
|
||||
goto failed;
|
||||
}
|
||||
|
||||
typesize = H5Tget_size(type);
|
||||
if (typesize == 0) return -1;
|
||||
if (typesize == 0) goto failed;
|
||||
/* Get the size of the base type, even for ARRAY types */
|
||||
classt = H5Tget_class(type);
|
||||
if (classt == H5T_ARRAY) {
|
||||
@ -196,21 +195,32 @@ herr_t blosc_set_local(hid_t dcpl, hid_t type, hid_t space)
|
||||
if (basetypesize > BLOSC_MAX_TYPESIZE) basetypesize = 1;
|
||||
values[2] = basetypesize;
|
||||
|
||||
/* Get the size of the chunk */
|
||||
bufsize = typesize;
|
||||
for (i = 0; i < ndims; i++) {
|
||||
bufsize *= chunkdims[i];
|
||||
}
|
||||
values[3] = bufsize;
|
||||
|
||||
{
|
||||
/* Get the size of the chunk */
|
||||
bufsize = typesize;
|
||||
#ifdef BLOSC_DEBUG
|
||||
fprintf(stderr, "Blosc: Computed buffer size %d\n", bufsize);
|
||||
fprintf(stderr,">>> set_local: |dims|=%d dims=(",(int)ndims);
|
||||
for (i = 0; i < ndims; i++) {
|
||||
fprintf(stderr,"%s%u",(i==0?"":","),(unsigned)chunkdims[i]);
|
||||
}
|
||||
fprintf(stderr,")\n");
|
||||
#endif
|
||||
for (i = 0; i < ndims; i++) {
|
||||
bufsize *= chunkdims[i];
|
||||
}
|
||||
values[3] = bufsize;
|
||||
}
|
||||
#ifdef BLOSC_DEBUG
|
||||
fprintf(stderr, "Blosc: working parameters =(%u) [%u %u %u %u %u %u %u]\n",
|
||||
(unsigned)nelements,values[0],values[1],values[2],values[3],values[4],values[5],values[6]); fflush(stderr);
|
||||
#endif
|
||||
|
||||
r = H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values);
|
||||
if (r < 0) return -1;
|
||||
if(H5Pmodify_filter(dcpl, H5Z_FILTER_BLOSC, flags, nelements, values) < 0) goto failed;
|
||||
|
||||
return 1;
|
||||
|
||||
failed:
|
||||
return -1;
|
||||
}
|
||||
|
||||
#endif /*USE_HDF5*/
|
||||
@ -222,7 +232,6 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
size_t* buf_size, void** buf)
|
||||
{
|
||||
void* outbuf = NULL;
|
||||
int status = 0; /* Return code from Blosc routines */
|
||||
size_t typesize;
|
||||
size_t outbuf_size;
|
||||
int clevel = 5; /* Compression level default */
|
||||
@ -230,37 +239,27 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
int compcode; /* Blosc compressor */
|
||||
int code;
|
||||
const char* compname = "blosclz"; /* The compressor by default */
|
||||
const char* complist = NULL;
|
||||
int bloscsize = 0;
|
||||
|
||||
#ifdef BLOSC_DEBUG
|
||||
fprintf(stderr, "Blosc: compressor =(%u) [%u %u %u %u %u %u %u]\n",
|
||||
(unsigned)cd_nelmts,cd_values[0],cd_values[1],cd_values[2],cd_values[3],cd_values[4],cd_values[5],cd_values[6]); fflush(stderr);
|
||||
#endif
|
||||
|
||||
/* Filter params that are always set */
|
||||
typesize = cd_values[2]; /* The datatype size */
|
||||
outbuf_size = cd_values[3]; /* Precomputed buffer guess */
|
||||
/* Optional params */
|
||||
if (cd_nelmts >= 5) {
|
||||
clevel = cd_values[4]; /* The compression level */
|
||||
}
|
||||
if (cd_nelmts >= 6) {
|
||||
doshuffle = cd_values[5]; /* BLOSC_SHUFFLE, BLOSC_BITSHUFFLE */
|
||||
/* bitshuffle is only meant for production in >= 1.8.0 */
|
||||
#if ((BLOSC_VERSION_MAJOR <= 1) && (BLOSC_VERSION_MINOR < 8))
|
||||
if (doshuffle == BLOSC_BITSHUFFLE) {
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK,
|
||||
"this Blosc library version is not supported. Please update to >= 1.8");
|
||||
goto failed;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
if (cd_nelmts >= 7) {
|
||||
compcode = cd_values[6]; /* The Blosc compressor used */
|
||||
/* Check that we actually have support for the compressor code */
|
||||
complist = blosc_list_compressors();
|
||||
code = blosc_compcode_to_compname(compcode, &compname);
|
||||
if (code == -1) {
|
||||
char s[4096];
|
||||
snprintf(s,sizeof(s),"this Blosc library does not have support for "
|
||||
"the '%s' compressor, but only for: %s",
|
||||
compname, complist);
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK,s);
|
||||
fprintf(stderr,"Blosc Filter Error: this Blosc library does not have support for the '%s' compressor\n",compname);
|
||||
goto failed;
|
||||
}
|
||||
}
|
||||
@ -277,33 +276,33 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
outbuf_size = (*buf_size);
|
||||
|
||||
#ifdef BLOSC_DEBUG
|
||||
fprintf(stderr, "Blosc: Compress %zd chunk w/buffer %zd\n",
|
||||
nbytes, outbuf_size);
|
||||
fprintf(stderr, "Blosc: Compress %zd chunk w/chunksize %zd\n",
|
||||
nbytes, outbuf_size);
|
||||
#endif
|
||||
|
||||
outbuf = malloc(outbuf_size);
|
||||
|
||||
if (outbuf == NULL) {
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK,
|
||||
"Can't allocate compression buffer");
|
||||
fprintf(stderr,"Blosc Filter Error: blosc_filter: Can't allocate compression buffer\n");
|
||||
goto failed;
|
||||
}
|
||||
|
||||
#ifdef BLOSCCTX
|
||||
blosc_set_compressor(compname);
|
||||
status = blosc_compress(clevel, doshuffle, typesize, nbytes, *buf, outbuf, nbytes);
|
||||
#else
|
||||
status = blosc_compress_ctx(clevel, doshuffle, typesize, nbytes, *buf, outbut, nbytes,
|
||||
bloscsize = blosc_compress_ctx(clevel, doshuffle, typesize, nbytes, *buf, outbut, nbytes,
|
||||
compname, /*blocksize*/0, /*no. thredds*/0);
|
||||
#else
|
||||
blosc_set_compressor(compname);
|
||||
bloscsize = blosc_compress(clevel, doshuffle, typesize, nbytes, *buf, outbuf, nbytes);
|
||||
#endif
|
||||
|
||||
if (status < 0) {
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc compression error");
|
||||
if(bloscsize == 0) {
|
||||
fprintf(stderr,"Blosc_FIlter Error: blosc_filter: Buffer is uncompressible.\n");
|
||||
goto failed;
|
||||
} else if(bloscsize < 0) {
|
||||
fprintf(stderr,"Blosc Filter Error: blosc_filter: blosc compression error\n");
|
||||
goto failed;
|
||||
}
|
||||
|
||||
/* We're decompressing */
|
||||
} else {
|
||||
} else { /* We're decompressing */
|
||||
/* declare dummy variables */
|
||||
size_t cbytes, blocksize;
|
||||
|
||||
@ -328,31 +327,31 @@ size_t blosc_filter(unsigned flags, size_t cd_nelmts,
|
||||
outbuf = malloc(outbuf_size);
|
||||
|
||||
if (outbuf == NULL) {
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK, "Can't allocate decompression buffer");
|
||||
fprintf(stderr,"Blosc Filter Error: blosc_filter: can't allocate decompression buffer\n");
|
||||
goto failed;
|
||||
}
|
||||
|
||||
#ifdef BLOSCCTX
|
||||
status = blosc_decompress(*buf, outbuf, outbuf_size);
|
||||
bloscsize = blosc_decompress_ctx(*buf, outbuf, outbuf_size, /*no. thredds*/0);
|
||||
#else
|
||||
status = blosc_decompress_ctx(*buf, outbuf, outbuf_size, /*no. thredds*/0);
|
||||
bloscsize = blosc_decompress(*buf, outbuf, outbuf_size);
|
||||
#endif
|
||||
|
||||
if (status <= 0) { /* decompression failed */
|
||||
PUSH_ERR("blosc_filter", H5E_CALLBACK, "Blosc decompression error");
|
||||
if(bloscsize <= 0) { /* decompression failed */
|
||||
fprintf(stderr,"Blosc Filter Error: blosc_filter: blosc decompression error\n");
|
||||
goto failed;
|
||||
} /* if !status */
|
||||
|
||||
} /* compressing vs decompressing */
|
||||
|
||||
if (status != 0) {
|
||||
{
|
||||
free(*buf);
|
||||
*buf = outbuf;
|
||||
*buf_size = outbuf_size;
|
||||
return status; /* Size of compressed/decompressed data */
|
||||
return bloscsize; /* Size of compressed/decompressed data */
|
||||
}
|
||||
|
||||
failed:
|
||||
failed:
|
||||
free(outbuf);
|
||||
return 0;
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
#! /bin/sh
|
||||
# test-driver - basic testsuite driver script.
|
||||
|
||||
scriptversion=2018-06-26.01; # UTC
|
||||
scriptversion=2022-07-12.01; # UTC
|
||||
|
||||
# Copyright (C) 2011-2013 Free Software Foundation, Inc.
|
||||
#
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user