hdf5/src/H5Z.c
Robb Matzke e455040749 [svn-r606] Debugging can be enabled/disabled at runtime though code is still
conditionally included at compile time.  Tracing and debugging share
the same environment variable.  All is documented in Debugging.html.
1998-08-19 17:46:31 -05:00

612 lines
16 KiB
C
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/*
* Copyright (C) 1998 NCSA
* All rights reserved.
*
* Programmer: Robb Matzke <matzke@llnl.gov>
* Thursday, April 16, 1998
*
* Purpose: Functions for data filters.
*/
#include <H5private.h>
#include <H5Eprivate.h>
#include <H5MMprivate.h>
#include <H5Oprivate.h>
#include <H5Zprivate.h>
#ifdef HAVE_ZLIB_H
# include <zlib.h>
#endif
/* Interface initialization */
#define PABLO_MASK H5Z_mask
#define INTERFACE_INIT H5Z_init_interface
static intn interface_initialize_g = FALSE;
static herr_t H5Z_init_interface (void);
static void H5Z_term_interface (void);
static size_t H5Z_table_alloc_g = 0;
static size_t H5Z_table_used_g = 0;
static H5Z_class_t *H5Z_table_g = NULL;
/* Predefined filters */
static size_t H5Z_filter_deflate(uintn flags, size_t cd_nelmts,
const uintn cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
/*-------------------------------------------------------------------------
* Function: H5Z_init_interface
*
* Purpose: Initializes the data filter layer.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5Z_init_interface (void)
{
FUNC_ENTER (H5Z_init_interface, FAIL);
H5_add_exit (H5Z_term_interface);
H5Z_register (H5Z_FILTER_DEFLATE, "deflate",
H5Z_filter_deflate);
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Z_term_interface
*
* Purpose: Terminate the H5Z layer.
*
* Return: void
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static void
H5Z_term_interface (void)
{
size_t i;
#ifdef H5Z_DEBUG
int dir, nprint=0;
char comment[16], bandwidth[32];
if (H5DEBUG(Z)) {
for (i=0; i<H5Z_table_used_g; i++) {
for (dir=0; dir<2; dir++) {
if (0==H5Z_table_g[i].stats[dir].total) continue;
if (0==nprint++) {
/* Print column headers */
HDfprintf (H5DEBUG(Z), "H5Z: filter statistics "
"accumulated over life of library:\n");
HDfprintf (H5DEBUG(Z),
" %-16s %10s %10s %8s %8s %8s %10s\n",
"Filter", "Total", "Errors", "User",
"System", "Elapsed", "Bandwidth");
HDfprintf (H5DEBUG(Z),
" %-16s %10s %10s %8s %8s %8s %10s\n",
"------", "-----", "------", "----",
"------", "-------", "---------");
}
/* Truncate the comment to fit in the field */
strncpy(comment, H5Z_table_g[i].name, sizeof comment);
comment[sizeof(comment)-1] = '\0';
/*
* Format bandwidth to have four significant digits and units
* of `B/s', `kB/s', `MB/s', `GB/s', or `TB/s' or the word
* `Inf' if the elapsed time is zero.
*/
H5_bandwidth(bandwidth,
(double)(H5Z_table_g[i].stats[dir].total),
H5Z_table_g[i].stats[dir].timer.etime);
/* Print the statistics */
HDfprintf (H5DEBUG(Z),
" %s%-15s %10Hd %10Hd %8.2f %8.2f %8.2f "
"%10s\n", dir?"<":">", comment,
H5Z_table_g[i].stats[dir].total,
H5Z_table_g[i].stats[dir].errors,
H5Z_table_g[i].stats[dir].timer.utime,
H5Z_table_g[i].stats[dir].timer.stime,
H5Z_table_g[i].stats[dir].timer.etime,
bandwidth);
}
}
}
#endif
/* Free the table */
for (i=0; i<H5Z_table_used_g; i++) {
H5MM_xfree(H5Z_table_g[i].name);
}
H5Z_table_g = H5MM_xfree(H5Z_table_g);
H5Z_table_used_g = H5Z_table_alloc_g = 0;
}
/*-------------------------------------------------------------------------
* Function: H5Zregister
*
* Purpose: This function registers new filter. The COMMENT argument is
* used for debugging and may be the null pointer.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Zregister (H5Z_filter_t id, const char *comment, H5Z_func_t func)
{
FUNC_ENTER (H5Zregister, FAIL);
H5TRACE3("e","Zfsx",id,comment,func);
/* Check args */
if (id<0 || id>H5Z_FILTER_MAX) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"invalid filter identification number");
}
if (id<256) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"unable to modify predefined filters");
}
if (!func) {
HRETURN_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
"no function specified");
}
/* Do it */
if (H5Z_register (id, comment, func)<0) {
HRETURN_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL,
"unable to register filter");
}
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Z_register
*
* Purpose: Same as the public version except this one allows filters
* to be set for predefined method numbers <256
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Z_register (H5Z_filter_t id, const char *comment, H5Z_func_t func)
{
size_t i;
FUNC_ENTER (H5Z_register, FAIL);
assert (id>=0 && id<=H5Z_FILTER_MAX);
/* Is the filter already registered? */
for (i=0; i<H5Z_table_used_g; i++) {
if (H5Z_table_g[i].id==id) break;
}
if (i>=H5Z_table_used_g) {
if (H5Z_table_used_g>=H5Z_table_alloc_g) {
size_t n = MAX(32, 2*H5Z_table_alloc_g);
H5Z_class_t *table = H5MM_realloc(H5Z_table_g,
n*sizeof(H5Z_class_t));
if (!table) {
HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"unable to extend filter table");
}
H5Z_table_g = table;
H5Z_table_alloc_g = n;
}
/* Initialize */
i = H5Z_table_used_g++;
HDmemset(H5Z_table_g+i, 0, sizeof(H5Z_class_t));
H5Z_table_g[i].id = id;
H5Z_table_g[i].name = H5MM_xstrdup(comment);
H5Z_table_g[i].func = func;
} else {
/* Replace old contents */
H5MM_xfree(H5Z_table_g[i].name);
H5Z_table_g[i].name = H5MM_xstrdup(comment);
H5Z_table_g[i].func = func;
}
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Z_append
*
* Purpose: Append another filter to the specified pipeline.
*
* Return: Success: SUCCEED
*
* Failure: FAIL
*
* Programmer: Robb Matzke
* Tuesday, August 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, uintn flags,
size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/])
{
size_t idx, i;
FUNC_ENTER(H5Z_append, FAIL);
assert(pline);
assert(filter>=0 && filter<=H5Z_FILTER_MAX);
assert(0==(flags & ~H5Z_FLAG_DEFMASK));
assert(0==cd_nelmts || cd_values);
/*
* Check filter limit. We do it here for early warnings although we may
* decide to relax this restriction in the future.
*/
if (pline->nfilters>=32) {
HRETURN_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL,
"too many filters in pipeline");
}
/* Allocate additional space in the pipeline if it's full */
if (pline->nfilters>=pline->nalloc) {
H5O_pline_t x;
x.nalloc = MAX(32, 2*pline->nalloc);
x.filter = H5MM_realloc(pline->filter, x.nalloc*sizeof(x.filter[0]));
if (NULL==x.filter) {
HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"memory allocation failed for filter pipeline");
}
pline->nalloc = x.nalloc;
pline->filter = x.filter;
}
/* Add the new filter to the pipeline */
idx = pline->nfilters;
pline->filter[idx].id = filter;
pline->filter[idx].flags = flags;
pline->filter[idx].name = NULL; /*we'll pick it up later*/
pline->filter[idx].cd_nelmts = cd_nelmts;
if (cd_nelmts>0) {
pline->filter[idx].cd_values = H5MM_malloc(cd_nelmts*sizeof(uintn));
if (NULL==pline->filter[idx].cd_values) {
HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
"memory allocation failed for filter");
}
for (i=0; i<cd_nelmts; i++) {
pline->filter[idx].cd_values[i] = cd_values[i];
}
} else {
pline->filter[idx].cd_values = NULL;
}
pline->nfilters++;
FUNC_LEAVE(SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Z_find
*
* Purpose: Given a filter ID return a pointer to a global struct that
* defines the filter.
*
* Return: Success: Ptr to entry in global filter table.
*
* Failure: NULL
*
* Programmer: Robb Matzke
* Wednesday, August 5, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
H5Z_class_t *
H5Z_find(H5Z_filter_t id)
{
size_t i;
FUNC_ENTER(H5Z_find, NULL);
for (i=0; i<H5Z_table_used_g; i++) {
if (H5Z_table_g[i].id == id) {
return H5Z_table_g + i;
}
}
FUNC_LEAVE(NULL);
}
/*-------------------------------------------------------------------------
* Function: H5Z_pipeline
*
* Purpose: Process data through the filter pipeline. The FLAGS argument
* is the filter invocation flags (definition flags come from
* the PLINE->filter[].flags). The filters are processed in
* definition order unless the H5Z_FLAG_REVERSE is set. The
* FILTER_MASK is a bit-mask to indicate which filters to skip
* and on exit will indicate which filters failed. Each
* filter has an index number in the pipeline and that index
* number is the filter's bit in the FILTER_MASK. NBYTES is the
* number of bytes of data to filter and on exit should be the
* number of resulting bytes while BUF_SIZE holds the total
* allocated size of the buffer, which is pointed to BUF.
*
* If the buffer must grow during processing of the pipeline
* then the pipeline function should free the original buffer
* and return a fresh buffer, adjusting BUF_SIZE accordingly.
*
* Return: Success: 0
*
* Failure: -1
*
* Programmer: Robb Matzke
* Tuesday, August 4, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Z_pipeline(H5F_t *f, const H5O_pline_t *pline, uintn flags,
uintn *filter_mask/*in,out*/, size_t *nbytes/*in,out*/,
size_t *buf_size/*in,out*/, void **buf/*in,out*/)
{
size_t i, idx, new_nbytes;
H5Z_class_t *fclass=NULL;
uintn failed = 0;
#ifdef H5Z_DEBUG
H5_timer_t timer;
#endif
FUNC_ENTER(H5Z_pipeline, FAIL);
assert(f);
assert(0==(flags & ~H5Z_FLAG_INVMASK));
assert(filter_mask);
assert(nbytes && *nbytes>0);
assert(buf_size && *buf_size>0);
assert(buf && *buf);
assert(!pline || pline->nfilters<32);
if (pline && (flags & H5Z_FLAG_REVERSE)) {
for (i=pline->nfilters; i>0; --i) {
idx = i-1;
if (*filter_mask & ((unsigned)1<<idx)) {
failed |= (unsigned)1 << idx;
continue;/*filter excluded*/
}
if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) {
failed |= (unsigned)1 << idx;
HRETURN_ERROR(H5E_PLINE, H5E_READERROR, FAIL,
"required filter is not registered");
}
#ifdef H5Z_DEBUG
H5_timer_begin(&timer);
#endif
new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags),
pline->filter[idx].cd_nelmts,
pline->filter[idx].cd_values,
*nbytes, buf_size, buf);
#ifdef H5Z_DEBUG
H5_timer_end(&(fclass->stats[1].timer), &timer);
fclass->stats[1].total += MAX(*nbytes, new_nbytes);
if (0==new_nbytes) fclass->stats[1].errors += *nbytes;
#endif
if (0==new_nbytes) {
failed |= (unsigned)1 << idx;
HRETURN_ERROR(H5E_PLINE, H5E_READERROR, FAIL,
"filter returned failure");
}
*nbytes = new_nbytes;
}
} else if (pline) {
for (idx=0; idx<pline->nfilters; idx++) {
if (*filter_mask & ((unsigned)1<<idx)) {
failed |= (unsigned)1 << idx;
continue; /*filter excluded*/
}
if (NULL==(fclass=H5Z_find(pline->filter[idx].id))) {
failed |= (unsigned)1 << idx;
if (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) {
H5E_clear();
continue;
} else {
HRETURN_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL,
"required filter is not registered");
}
}
#ifdef H5Z_DEBUG
H5_timer_begin(&timer);
#endif
new_nbytes = (fclass->func)(flags|(pline->filter[idx].flags),
pline->filter[idx].cd_nelmts,
pline->filter[idx].cd_values,
*nbytes, buf_size, buf);
#ifdef H5Z_DEBUG
H5_timer_end(&(fclass->stats[0].timer), &timer);
fclass->stats[0].total += MAX(*nbytes, new_nbytes);
if (0==new_nbytes) fclass->stats[0].errors += *nbytes;
#endif
if (0==new_nbytes) {
failed |= (unsigned)1 << idx;
if (0==(pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) {
HRETURN_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL,
"filter returned failure");
} else {
H5E_clear();
}
} else {
*nbytes = new_nbytes;
}
}
}
*filter_mask = failed;
FUNC_LEAVE(SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Z_filter_deflate
*
* Purpose:
*
* Return: Success:
*
* Failure:
*
* Programmer: Robb Matzke
* Thursday, April 16, 1998
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static size_t
H5Z_filter_deflate (uintn flags, size_t cd_nelmts, const uintn cd_values[],
size_t nbytes, size_t *buf_size, void **buf)
{
size_t ret_value = 0;
int aggression = 6;
void *outbuf = NULL;
#if defined(HAVE_LIBZ) && defined(HAVE_ZLIB_H)
int status;
#endif
FUNC_ENTER (H5Z_filter_deflate, 0);
/* Get aggression level */
if (cd_nelmts!=1 || cd_values[0]>9) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
"invalid deflate aggression level");
}
aggression = cd_values[0];
#if defined(HAVE_LIBZ) && defined (HAVE_ZLIB_H)
if (flags & H5Z_FLAG_REVERSE) {
/* Input; uncompress */
z_stream z_strm;
size_t nalloc = *buf_size;
if (NULL==(outbuf = H5MM_malloc(nalloc))) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"memory allocation failed for deflate uncompression");
}
HDmemset(&z_strm, 0, sizeof(z_strm));
z_strm.next_in = *buf;
z_strm.avail_in = nbytes;
z_strm.next_out = outbuf;
z_strm.avail_out = nalloc;
if (Z_OK!=inflateInit(&z_strm)) {
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
}
while (1) {
status = inflate(&z_strm, Z_SYNC_FLUSH);
if (Z_STREAM_END==status) break; /*done*/
if (Z_OK!=status) {
inflateEnd(&z_strm);
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
}
if (Z_OK==status && 0==z_strm.avail_out) {
nalloc *= 2;
if (NULL==(outbuf = H5MM_realloc(outbuf, nalloc))) {
inflateEnd(&z_strm);
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"memory allocation failed for deflate "
"uncompression");
}
z_strm.next_out = (char*)outbuf + z_strm.total_out;
z_strm.avail_out = nalloc - z_strm.total_out;
}
}
H5MM_xfree(*buf);
*buf = outbuf;
outbuf = NULL;
*buf_size = nalloc;
ret_value = z_strm.total_out;
inflateEnd(&z_strm);
} else {
/*
* Output; compress but fail if the result would be larger than the
* input. The library doesn't provide in-place compression, so we
* must allocate a separate buffer for the result.
*/
const Bytef *z_src = (const Bytef*)(*buf);
Bytef *z_dst; /*destination buffer */
uLongf z_dst_nbytes = (uLongf)nbytes;
uLong z_src_nbytes = (uLong)nbytes;
if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"unable to allocate deflate destination buffer");
}
status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes,
aggression);
if (Z_BUF_ERROR==status) {
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow");
} else if (Z_MEM_ERROR==status) {
HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate memory error");
} else if (Z_OK!=status) {
HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, 0, "deflate error");
} else {
H5MM_xfree(*buf);
*buf = outbuf;
outbuf = NULL;
*buf_size = nbytes;
ret_value = z_dst_nbytes;
}
}
#else
HGOTO_ERROR (H5E_PLINE, H5E_UNSUPPORTED, 0,
"hdf5 was not compiled with zlib-1.0.2 or better");
#endif
done:
H5MM_xfree(outbuf);
FUNC_LEAVE (ret_value);
}