mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-13 16:47:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r6114] Purpose:

Browse Source 
Code Cleanup & New Feature

Description:
    H5config.h.in:
        Removed H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags.

        Added H5_HAVE_FILTER_DEFLATE flag.

    H5Z.c:
    H5Zprivate.h:
    H5Zpublic.h:
        Switched from using H5_HAVE_COMPRESSION flag in favor of
        H5_HAVE_FILTER_DEFLATE.

        Added H5Zunregister & H5Zfilter_avail API functions.

        Changed a numeric constant (256) to a symbolic constant
        (H5Z_FILTER_RESERVED).

        Automatically add the shuffling filter to the list of available filters
        (when it is enabled).

        Moved prototypes for H5Z_filter_deflate & H5Z_filter_shuffle from the
        public header into the private header.

    H5Zdeflate.c:
        Switched from using H5_HAVE_COMPRESSION & H5_HAVE_FILTER_GZIP flags in
        favor of H5_HAVE_FILTER_DEFLATE.

        Cleaned up formatting & error reporting a bit.

    H5Zshuffle.c:
        Rewrote shuffling algorithm to be more efficient.

        Added error checking & reporting.

        Added standard Pablo information.

        Added standard function header comment.

        Added FUNC_ENTER & FUNC_LEAVE macros.

Platforms tested:
    Tested h5committest {arabica (fortran), eirene (fortran, C++)
        modi4 (parallel, fortran)}
    FreeBSD 4.7 (sleipnir)
This commit is contained in:
Quincey Koziol 2002-11-20 08:15:18 -05:00
parent 892cc41777
commit ac48a23e2b
6 changed files with 253 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

133
src/H5Z.c
View File

@ -19,10 +19,12 @@
static int interface_initialize_g = 0;
static herr_t H5Z_init_interface (void);
/* Local variables */
static size_t H5Z_table_alloc_g = 0;
static size_t H5Z_table_used_g = 0;
static H5Z_class_t *H5Z_table_g = NULL;
/* Local functions */
/*-------------------------------------------------------------------------
@ -44,9 +46,12 @@ H5Z_init_interface (void)
{
FUNC_ENTER_NOINIT(H5Z_init_interface);
#ifdef H5_HAVE_FILTER_GZIP
#ifdef H5_HAVE_FILTER_DEFLATE
H5Z_register (H5Z_FILTER_DEFLATE, "deflate", H5Z_filter_deflate);
#endif /* H5_HAVE_FILTER_GZIP */
#endif /* H5_HAVE_FILTER_DEFLATE */
#ifdef H5_HAVE_FILTER_SHUFFLE
H5Z_register (H5Z_FILTER_SHUFFLE, "shuffle", H5Z_filter_shuffle);
#endif /* H5_HAVE_FILTER_SHUFFLE */
FUNC_LEAVE (SUCCEED);
}
@ -162,7 +167,7 @@ H5Zregister(H5Z_filter_t id, const char *comment, H5Z_func_t func)
/* Check args */
if (id<0 || id>H5Z_FILTER_MAX)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
if (id<256)
if (id<H5Z_FILTER_RESERVED)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "unable to modify predefined filters");
if (!func)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no function specified");
@ -180,7 +185,7 @@ done:
* Function: H5Z_register
*
* Purpose: Same as the public version except this one allows filters
* to be set for predefined method numbers <256
* to be set for predefined method numbers <H5Z_FILTER_RESERVED
*
* Return: Non-negative on success/Negative on failure
*
@ -233,6 +238,126 @@ done:
FUNC_LEAVE (ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5Zunregister
*
* Purpose: This function unregisters a filter.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* Thursday, November 14, 2002
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Zunregister(H5Z_filter_t id)
{
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_API(H5Zunregister, FAIL);
H5TRACE1("e","Zf",id);
/* Check args */
if (id<0 || id>H5Z_FILTER_MAX)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
if (id<H5Z_FILTER_RESERVED)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "unable to modify predefined filters");
/* Do it */
if (H5Z_unregister (id)<0)
HGOTO_ERROR (H5E_PLINE, H5E_CANTINIT, FAIL, "unable to unregister filter");
done:
FUNC_LEAVE (ret_value);
} /* end H5Zunregister() */
/*-------------------------------------------------------------------------
* Function: H5Z_unregister
*
* Purpose: Same as the public version except this one allows filters
* to be unset for predefined method numbers <H5Z_FILTER_RESERVED
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* Thursday, November 14, 2002
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Z_unregister (H5Z_filter_t id)
{
size_t i; /* Local index variable */
herr_t ret_value=SUCCEED; /* Return value */
FUNC_ENTER_NOINIT(H5Z_unregister);
assert (id>=0 && id<=H5Z_FILTER_MAX);
/* Is the filter already registered? */
for (i=0; i<H5Z_table_used_g; i++)
if (H5Z_table_g[i].id==id)
break;
/* Fail if filter not found */
if (i>=H5Z_table_used_g)
HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter is not registered");
/* Remove filter from table */
/* Don't worry about shrinking table size (for now) */
HDmemmove(&H5Z_table_g[i],&H5Z_table_g[i+1],sizeof(H5Z_class_t)*((H5Z_table_used_g-1)-i));
H5Z_table_used_g--;
done:
FUNC_LEAVE (ret_value);
} /* end H5Z_unregister() */
/*-------------------------------------------------------------------------
* Function: H5Zfilter_avail
*
* Purpose: Check if a filter is available
*
* Return: Non-negative (TRUE/FALSE) on success/Negative on failure
*
* Programmer: Quincey Koziol
* Thursday, November 14, 2002
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
htri_t
H5Zfilter_avail(H5Z_filter_t id)
{
size_t i; /* Local index variable */
htri_t ret_value=FALSE; /* Return value */
FUNC_ENTER_API(H5Zfilter_avail, FAIL);
H5TRACE1("b","Zf",id);
/* Check args */
if(id<0 || id>H5Z_FILTER_MAX)
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "invalid filter identification number");
/* Is the filter already registered? */
for(i=0; i<H5Z_table_used_g; i++)
if(H5Z_table_g[i].id==id) {
ret_value=TRUE;
break;
} /* end if */
done:
FUNC_LEAVE (ret_value);
} /* end H5Zfilter_avail() */
/*-------------------------------------------------------------------------
* Function: H5Z_append

55
src/H5Zdeflate.c
View File

@ -10,13 +10,10 @@
#include "H5MMprivate.h"
#include "H5Zprivate.h"
#ifdef H5_HAVE_FILTER_GZIP
#ifdef H5_HAVE_FILTER_DEFLATE
#ifdef H5_HAVE_ZLIB_H
# include "zlib.h"
#else
/* Make sure compression is disabled too. */
#undef H5_HAVE_COMPRESSION
#endif
/* Interface initialization */
@ -28,7 +25,8 @@ static int interface_initialize_g = 0;
/*-------------------------------------------------------------------------
* Function: H5Z_filter_deflate
*
* Purpose:
* Purpose: Implement an I/O filter around the 'deflate' algorithm in
* libz
*
* Return: Success:
*
@ -42,53 +40,40 @@ static int interface_initialize_g = 0;
*-------------------------------------------------------------------------
*/
size_t
#if defined(H5_HAVE_COMPRESSION)
H5Z_filter_deflate (unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
#else
H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
const unsigned cd_values[], size_t UNUSED nbytes,
size_t * UNUSED buf_size, void ** UNUSED buf)
#endif
{
size_t ret_value = 0;
void *outbuf = NULL;
#if defined(H5_HAVE_COMPRESSION)
int aggression = 6;
int status;
#endif
FUNC_ENTER_NOAPI(H5Z_filter_deflate, 0);
/* Check arguments */
if (cd_nelmts!=1 || cd_values[0]>9) {
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0,
"invalid deflate aggression level");
}
if (cd_nelmts!=1 || cd_values[0]>9)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level");
#if defined(H5_HAVE_COMPRESSION)
aggression = cd_values[0];
if (flags & H5Z_FLAG_REVERSE) {
/* Input; uncompress */
z_stream z_strm;
size_t nalloc = *buf_size;
if (NULL==(outbuf = H5MM_malloc(nalloc))) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"memory allocation failed for deflate uncompression");
}
if (NULL==(outbuf = H5MM_malloc(nalloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression");
HDmemset(&z_strm, 0, sizeof(z_strm));
z_strm.next_in = *buf;
H5_ASSIGN_OVERFLOW(z_strm.avail_in,nbytes,size_t,uInt);
z_strm.next_out = outbuf;
H5_ASSIGN_OVERFLOW(z_strm.avail_out,nalloc,size_t,uInt);
if (Z_OK!=inflateInit(&z_strm)) {
if (Z_OK!=inflateInit(&z_strm))
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflateInit() failed");
}
while (1) {
status = inflate(&z_strm, Z_SYNC_FLUSH);
if (Z_STREAM_END==status) break; /*done*/
if (Z_STREAM_END==status)
break; /*done*/
if (Z_OK!=status) {
inflateEnd(&z_strm);
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "inflate() failed");
@ -97,9 +82,7 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
nalloc *= 2;
if (NULL==(outbuf = H5MM_realloc(outbuf, nalloc))) {
inflateEnd(&z_strm);
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"memory allocation failed for deflate "
"uncompression");
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for deflate uncompression");
}
z_strm.next_out = (unsigned char*)outbuf + z_strm.total_out;
z_strm.avail_out = (uInt)(nalloc - z_strm.total_out);
@ -112,7 +95,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
*buf_size = nalloc;
ret_value = z_strm.total_out;
inflateEnd(&z_strm);
} else {
/*
* Output; compress but fail if the result would be larger than the
@ -124,12 +106,9 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
uLongf z_dst_nbytes = (uLongf)nbytes;
uLong z_src_nbytes = (uLong)nbytes;
if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes))) {
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0,
"unable to allocate deflate destination buffer");
}
status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes,
aggression);
if (NULL==(z_dst=outbuf=H5MM_malloc(nbytes)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate deflate destination buffer");
status = compress2 (z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression);
if (Z_BUF_ERROR==status) {
HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, 0, "overflow");
} else if (Z_MEM_ERROR==status) {
@ -144,10 +123,6 @@ H5Z_filter_deflate (unsigned UNUSED flags, size_t cd_nelmts,
ret_value = z_dst_nbytes;
}
}
#else
HGOTO_ERROR (H5E_PLINE, H5E_UNSUPPORTED, 0,
"hdf5 was not compiled with zlib-1.0.2 or better");
#endif
done:
if(outbuf)
@ -155,4 +130,4 @@ done:
FUNC_LEAVE (ret_value);
}
#endif /* H5_HAVE_FILTER_GZIP */
#endif /* H5_HAVE_FILTER_DEFLATE */

9
src/H5Zprivate.h
View File

@ -33,6 +33,7 @@ struct H5O_pline_t; /*forward decl*/
H5_DLL herr_t H5Z_register(H5Z_filter_t id, const char *comment,
H5Z_func_t filter);
H5_DLL herr_t H5Z_unregister (H5Z_filter_t id);
H5_DLL herr_t H5Z_append(struct H5O_pline_t *pline, H5Z_filter_t filter,
unsigned flags, size_t cd_nelmts,
const unsigned int cd_values[]);
@ -43,4 +44,12 @@ H5_DLL herr_t H5Z_pipeline(H5F_t *f, const struct H5O_pline_t *pline,
H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id);
/* Filter routines */
size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
#endif

11
src/H5Zpublic.h
View File

@ -20,6 +20,7 @@ typedef int H5Z_filter_t;
#define H5Z_FILTER_NONE 0 /*reserved indefinitely */
#define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */
#define H5Z_FILTER_SHUFFLE 2 /* shuffle the data */
#define H5Z_FILTER_RESERVED 256 /*filter ids below this value are reserved */
#define H5Z_FILTER_MAX 65535 /*maximum filter id */
/* Flags for filter definition */
@ -56,13 +57,11 @@ extern "C" {
H5_DLL herr_t H5Zregister(H5Z_filter_t id, const char *comment,
H5Z_func_t filter);
size_t H5Z_filter_deflate(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
H5_DLL herr_t H5Zunregister(H5Z_filter_t id);
H5_DLL htri_t H5Zfilter_avail(H5Z_filter_t id);
size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
#ifdef __cplusplus
}
#endif

153
src/H5Zshuffle.c
View File

@ -9,73 +9,106 @@
#include "H5Eprivate.h"
#include "H5MMprivate.h"
#include "H5Zprivate.h"
#include <stdio.h>
#ifdef H5_HAVE_FILTER_SHUFFLE
size_t H5Z_filter_shuffle(unsigned flags,
size_t cd_nelmts,
const unsigned cd_values[],
size_t nbytes,
size_t *buf_size,void **buf) {
size_t i;
size_t j;
size_t k;
size_t ret_value = 0;
size_t byte_pos;
size_t bytesoftype;
void* dest = NULL;
char* _src;
char* _des;
char* _dest;
size_t numofelements;
bytesoftype=cd_values[0];
numofelements=nbytes/bytesoftype;
_src =(char*)(*buf);
/* Interface initialization */
#define PABLO_MASK H5Z_shuffle_mask
#define INTERFACE_INIT NULL
static int interface_initialize_g = 0;
dest = malloc((size_t)nbytes);
_dest =(char*)dest;
/*-------------------------------------------------------------------------
* Function: H5Z_filter_shuffle
*
* Purpose: Implement an I/O filter which "de-interlaces" a block of data
* by putting all the bytes in a byte-position for each element
* together in the block. For example, for 4-byte elements stored
* as: 012301230123, shuffling will store them as: 000111222333
* Usually, the bytes in each byte position are more related to
* each other and putting them together will increase compression.
*
* Return: Success:
*
* Failure:
*
* Programmer: Kent Yang
* Wednesday, November 13, 2002
*
* Modifications:
* Quincey Koziol, November 13, 2002
* Cleaned up code.
*
*-------------------------------------------------------------------------
*/
size_t
H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts,
const unsigned cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */
unsigned char *_src; /* Alias for source buffer */
unsigned char *_dest; /* Alias for destination buffer */
unsigned bytesoftype; /* Number of bytes per element */
size_t numofelements; /* Number of elements in buffer */
size_t i,j; /* Local index variables */
size_t ret_value; /* Return value */
j = 0;
k = 0;
if(flags & H5Z_FLAG_REVERSE) {
for(i=0;i<nbytes;i++) {
byte_pos = 1 +j *numofelements;
if(byte_pos > nbytes) {
k++;
j=0;
byte_pos=1;
}
*(_dest+i)=*((char*)(_src+byte_pos-1+k));
j++;
}
free(*buf);
*buf = dest;
dest = NULL;
*buf_size=nbytes;
ret_value = nbytes;
}
FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0);
else {
for (i=0;i<nbytes;i++){
byte_pos = 1+j * bytesoftype;
if(byte_pos >nbytes) {
k++;
j=0;
byte_pos = 1;
}
/* Check arguments */
if (cd_nelmts!=1 || cd_values[0]==0)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid shuffle parameters");
*(_dest+i)=*(_src+byte_pos-1+k);
j++;
}
free(*buf);
*buf = dest;
dest = NULL;
*buf_size=nbytes;
ret_value = nbytes;
}
/* Get the number of bytes per element from the parameter block */
bytesoftype=cd_values[0];
/* Don't do anything for 1-byte elements */
if(bytesoftype>1) {
/* Compute the number of elements in buffer */
numofelements=nbytes/bytesoftype;
/* Allocate the destination buffer */
if (NULL==(dest = H5MM_malloc(nbytes)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer");
if(flags & H5Z_FLAG_REVERSE) {
/* Get the pointer to the source buffer */
_src =(unsigned char *)(*buf);
/* Input; unshuffle */
for(i=0; i<bytesoftype; i++) {
_dest=((unsigned char *)dest)+i;
for(j=0; j<numofelements; j++) {
*_dest=*_src++;
_dest+=bytesoftype;
} /* end for */
} /* end for */
} /* end if */
else {
/* Get the pointer to the destination buffer */
_dest =(unsigned char *)dest;
/* Output; shuffle */
for(i=0; i<bytesoftype; i++) {
_src=((unsigned char *)(*buf))+i;
for(j=0; j<numofelements; j++) {
*_dest++=*_src;
_src+=bytesoftype;
} /* end for */
} /* end for */
} /* end else */
/* Set the buffer information to return */
H5MM_xfree(*buf);
*buf = dest;
*buf_size=nbytes;
} /* end else */
/* Set the return value */
ret_value = nbytes;
done:
FUNC_LEAVE (ret_value);
}
#endif /*H5_HAVE_FILTER_SHUFFLE */

7
src/H5config.h.in
View File

@ -9,9 +9,6 @@
/* Define to 1 if you have the `BSDgettimeofday' function. */
#undef HAVE_BSDGETTIMEOFDAY
/* Define if we have zlib compression support */
#undef HAVE_COMPRESSION
/* Define to 1 if you have the `difftime' function. */
#undef HAVE_DIFFTIME
@ -24,8 +21,8 @@
/* Define to 1 if you have the <features.h> header file. */
#undef HAVE_FEATURES_H
/* Define if support for gzip filter is enabled */
#undef HAVE_FILTER_GZIP
/* Define if support for deflate filter is enabled */
#undef HAVE_FILTER_DEFLATE
/* Define if support for shuffle filter is enabled */
#undef HAVE_FILTER_SHUFFLE