mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-02-17 16:10:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r12806] Description:

Browse Source 
Reduce memory usage for common cases of I/O pipeline filter memory usage.

    Also, clean up some more code...

Tested on:
    Linux/32 2.6 (chicago)
    Linux/64 2.6 (chicago2)
This commit is contained in:
Quincey Koziol 2006-10-23 19:16:49 -05:00
parent 45ac464161
commit 5032b2e7df
6 changed files with 250 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/H5HFtest.c
View File

@ -205,7 +205,7 @@ HDfprintf(stderr, "%s: cparam2->pline.filter[%Zu].name = %s\n", "H5HF_cmp_cparam
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5HF_get_cparam_test() */
} /* H5HF_cmp_cparam_test() */
/*-------------------------------------------------------------------------

89
src/H5Opline.c
View File

@ -152,9 +152,22 @@ H5O_pline_decode(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const uint8_t *p)
/* Filter name, if there is one */
if(name_length) {
filter->name = H5MM_malloc(name_length);
HDmemcpy(filter->name, p, name_length);
HDassert(filter->name[name_length - 1] == '\0');
size_t actual_name_length; /* Actual length of name */
/* Determine actual name length (without padding, but with null terminator) */
actual_name_length = HDstrlen((const char *)p) + 1;
HDassert(actual_name_length <= name_length);
/* Allocate space for the filter name, or use the internal buffer */
if(actual_name_length > H5Z_COMMON_NAME_LEN) {
filter->name = H5MM_malloc(actual_name_length);
if(NULL == filter->name)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for filter name")
} /* end if */
else
filter->name = filter->_name;
HDstrcpy(filter->name, (const char *)p);
p += name_length;
} /* end if */
@ -162,12 +175,18 @@ H5O_pline_decode(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const uint8_t *p)
if(filter->cd_nelmts) {
size_t j; /* Local index variable */
/*
* Read the client data values and the padding
*/
/* Allocate space for the client data elements, or use the internal buffer */
if(filter->cd_nelmts > H5Z_COMMON_CD_VALUES) {
filter->cd_values = H5MM_malloc(filter->cd_nelmts * sizeof(unsigned));
if(NULL == filter->cd_values)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for client data")
} /* end if */
else
filter->cd_values = filter->_cd_values;
/*
* Read the client data values and the padding
*/
for(j = 0; j < filter->cd_nelmts; j++)
UINT32DECODE(p, filter->cd_values[j]);
if(version == H5O_PLINE_VERSION_1)
@ -181,14 +200,8 @@ H5O_pline_decode(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const uint8_t *p)
done:
if(NULL == ret_value && pline) {
if(pline->filter) {
for(i = 0; i < pline->nused; i++) {
H5MM_xfree(pline->filter[i].name);
H5MM_xfree(pline->filter[i].cd_values);
} /* end for */
H5MM_xfree(pline->filter);
} /* end if */
H5FL_FREE(H5O_pline_t,pline);
H5O_pline_reset(pline);
H5O_pline_free(pline);
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@ -346,17 +359,37 @@ H5O_pline_copy(const void *_src, void *_dst/*out*/, unsigned UNUSED update_flags
dst->filter[i] = src->filter[i];
/* Filter name */
if(src->filter[i].name)
if(NULL == (dst->filter[i].name = H5MM_xstrdup(src->filter[i].name)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
if(src->filter[i].name) {
size_t namelen; /* Length of source filter name, including null terminator */
namelen = HDstrlen(src->filter[i].name) + 1;
/* Allocate space for the filter name, or use the internal buffer */
if(namelen > H5Z_COMMON_NAME_LEN) {
dst->filter[i].name = H5MM_malloc(namelen);
if(NULL == dst->filter[i].name)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for filter name")
/* Copy name */
HDstrcpy(dst->filter[i].name, src->filter[i].name);
} /* end if */
else
dst->filter[i].name = dst->filter[i]._name;
} /* end if */
/* Filter parameters */
if(src->filter[i].cd_nelmts > 0) {
/* Allocate space for the client data elements, or use the internal buffer */
if(src->filter[i].cd_nelmts > H5Z_COMMON_CD_VALUES) {
if(NULL == (dst->filter[i].cd_values = H5MM_malloc(src->filter[i].cd_nelmts* sizeof(unsigned))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
HDmemcpy(dst->filter[i].cd_values, src->filter[i].cd_values,
src->filter[i].cd_nelmts * sizeof(unsigned));
} /* end if */
else
dst->filter[i].cd_values = dst->filter[i]._cd_values;
} /* end if */
} /* end for */
} /* end if */
else
@ -367,15 +400,9 @@ H5O_pline_copy(const void *_src, void *_dst/*out*/, unsigned UNUSED update_flags
done:
if(!ret_value && dst) {
if(dst->filter) {
for(i = 0; i < dst->nused; i++) {
H5MM_xfree(dst->filter[i].name);
H5MM_xfree(dst->filter[i].cd_values);
} /* end for */
H5MM_xfree(dst->filter);
} /* end if */
H5O_pline_reset(dst);
if(!_dst)
H5FL_FREE(H5O_pline_t,dst);
H5O_pline_free(dst);
} /* end if */
FUNC_LEAVE_NOAPI(ret_value)
@ -475,8 +502,14 @@ H5O_pline_reset(void *mesg)
/* Free information for each filter */
for(i = 0; i < pline->nused; i++) {
H5MM_xfree(pline->filter[i].name);
H5MM_xfree(pline->filter[i].cd_values);
if(pline->filter[i].name && pline->filter[i].name != pline->filter[i]._name)
HDassert((HDstrlen(pline->filter[i].name) + 1) > H5Z_COMMON_NAME_LEN);
if(pline->filter[i].name != pline->filter[i]._name)
pline->filter[i].name = H5MM_xfree(pline->filter[i].name);
if(pline->filter[i].cd_values && pline->filter[i].cd_values != pline->filter[i]._cd_values)
HDassert(pline->filter[i].cd_nelmts > H5Z_COMMON_CD_VALUES);
if(pline->filter[i].cd_values != pline->filter[i]._cd_values)
pline->filter[i].cd_values = H5MM_xfree(pline->filter[i].cd_values);
} /* end for */
/* Free filter array */
@ -500,8 +533,6 @@ H5O_pline_reset(void *mesg)
* Programmer: Quincey Koziol
* Saturday, March 11, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t

84
src/H5Z.c
View File

@ -698,15 +698,14 @@ H5Z_modify(const H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags,
size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/])
{
size_t idx; /* Index of filter in pipeline */
size_t i; /* Local index variable */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(H5Z_modify, FAIL)
assert(pline);
assert(filter>=0 && filter<=H5Z_FILTER_MAX);
assert(0==(flags & ~((unsigned)H5Z_FLAG_DEFMASK)));
assert(0==cd_nelmts || cd_values);
HDassert(pline);
HDassert(filter >= 0 && filter <= H5Z_FILTER_MAX);
HDassert(0 == (flags & ~((unsigned)H5Z_FLAG_DEFMASK)));
HDassert(0 == cd_nelmts || cd_values);
/* Locate the filter in the pipeline */
for(idx = 0; idx < pline->nused; idx++)
@ -722,14 +721,23 @@ H5Z_modify(const H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags,
pline->filter[idx].cd_nelmts = cd_nelmts;
/* Free any existing parameters */
if(pline->filter[idx].cd_values!=NULL)
if(pline->filter[idx].cd_values != NULL && pline->filter[idx].cd_values != pline->filter[idx]._cd_values)
H5MM_xfree(pline->filter[idx].cd_values);
/* Set parameters */
if(cd_nelmts > 0) {
size_t i; /* Local index variable */
/* Allocate memory or point at internal buffer */
if(cd_nelmts > H5Z_COMMON_CD_VALUES) {
pline->filter[idx].cd_values = H5MM_malloc(cd_nelmts * sizeof(unsigned));
if(NULL == pline->filter[idx].cd_values)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for filter parameters")
} /* end if */
else
pline->filter[idx].cd_values = pline->filter[idx]._cd_values;
/* Copy client data values */
for(i = 0; i < cd_nelmts; i++)
pline->filter[idx].cd_values[i] = cd_values[i];
} /* end if */
@ -759,15 +767,15 @@ herr_t
H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags,
size_t cd_nelmts, const unsigned int cd_values[/*cd_nelmts*/])
{
size_t idx, i;
size_t idx;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(H5Z_append, FAIL)
assert(pline);
assert(filter>=0 && filter<=H5Z_FILTER_MAX);
assert(0==(flags & ~((unsigned)H5Z_FLAG_DEFMASK)));
assert(0==cd_nelmts || cd_values);
HDassert(pline);
HDassert(filter >= 0 && filter <= H5Z_FILTER_MAX);
HDassert(0 == (flags & ~((unsigned)H5Z_FLAG_DEFMASK)));
HDassert(0 == cd_nelmts || cd_values);
/*
* Check filter limit. We do it here for early warnings although we may
@ -779,13 +787,14 @@ H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags,
/* Allocate additional space in the pipeline if it's full */
if(pline->nused >= pline->nalloc) {
H5O_pline_t x;
x.nalloc = MAX(H5Z_MAX_NFILTERS, 2 * pline->nalloc);
x.filter = H5MM_realloc(pline->filter, x.nalloc*sizeof(x.filter[0]));
if(NULL == x.filter)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for filter pipeline")
pline->nalloc = x.nalloc;
pline->filter = x.filter;
}
} /* end if */
/* Add the new filter to the pipeline */
idx = pline->nused;
@ -794,19 +803,29 @@ H5Z_append(H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags,
pline->filter[idx].name = NULL; /*we'll pick it up later*/
pline->filter[idx].cd_nelmts = cd_nelmts;
if(cd_nelmts > 0) {
size_t i; /* Local index variable */
/* Allocate memory or point at internal buffer */
if(cd_nelmts > H5Z_COMMON_CD_VALUES) {
pline->filter[idx].cd_values = H5MM_malloc(cd_nelmts * sizeof(unsigned));
if(NULL == pline->filter[idx].cd_values)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for filter")
} /* end if */
else
pline->filter[idx].cd_values = pline->filter[idx]._cd_values;
/* Copy client data values */
for(i = 0; i < cd_nelmts; i++)
pline->filter[idx].cd_values[i] = cd_values[i];
} else {
} /* end if */
else
pline->filter[idx].cd_values = NULL;
}
pline->nused++;
done:
FUNC_LEAVE_NOAPI(ret_value)
}
} /* end H5Z_append() */
/*-------------------------------------------------------------------------
@ -1134,8 +1153,8 @@ H5Z_delete(H5O_pline_t *pline, H5Z_filter_t filter)
FUNC_ENTER_NOAPI(H5Z_delete, FAIL)
/* Check args */
assert(pline);
assert(filter>=0 && filter<=H5Z_FILTER_MAX);
HDassert(pline);
HDassert(filter >= 0 && filter <= H5Z_FILTER_MAX);
/* if the pipeline has no filters, just return */
if(pline->nused==0)
@ -1149,27 +1168,40 @@ H5Z_delete(H5O_pline_t *pline, H5Z_filter_t filter)
/* Delete filter */
else {
size_t idx; /* Index of filter in pipeline */
unsigned found=0; /* Indicate filter was found in pipeline */
hbool_t found = FALSE; /* Indicate filter was found in pipeline */
/* Locate the filter in the pipeline */
for(idx = 0; idx < pline->nused; idx++)
if(pline->filter[idx].id == filter) {
found=1;
found = TRUE;
break;
}
} /* end if */
/* filter was not found in the pipeline */
if(!found)
HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, FAIL, "filter not in pipeline")
/* Free information for deleted filter */
H5MM_xfree(pline->filter[idx].name);
H5MM_xfree(pline->filter[idx].cd_values);
if(pline->filter[idx].name && pline->filter[idx].name != pline->filter[idx]._name)
HDassert((HDstrlen(pline->filter[idx].name) + 1) > H5Z_COMMON_NAME_LEN);
if(pline->filter[idx].name != pline->filter[idx]._name)
pline->filter[idx].name = H5MM_xfree(pline->filter[idx].name);
if(pline->filter[idx].cd_values && pline->filter[idx].cd_values != pline->filter[idx]._cd_values)
HDassert(pline->filter[idx].cd_nelmts > H5Z_COMMON_CD_VALUES);
if(pline->filter[idx].cd_values != pline->filter[idx]._cd_values)
pline->filter[idx].cd_values = H5MM_xfree(pline->filter[idx].cd_values);
/* Remove filter from pipeline array */
if((idx+1)<pline->nused)
HDmemcpy(&pline->filter[idx], &pline->filter[idx+1],
sizeof (H5Z_filter_info_t)*(pline->nused-(idx+1)));
if((idx + 1) < pline->nused) {
/* Copy filters down & fix up any client data value arrays using internal storage */
for(; (idx + 1) < pline->nused; idx++) {
pline->filter[idx] = pline->filter[idx + 1];
if(pline->filter[idx].name && (HDstrlen(pline->filter[idx].name) + 1) <= H5Z_COMMON_NAME_LEN)
pline->filter[idx].name = pline->filter[idx]._name;
if(pline->filter[idx].cd_nelmts <= H5Z_COMMON_CD_VALUES)
pline->filter[idx].cd_values = pline->filter[idx]._cd_values;
} /* end for */
} /* end if */
/* Decrement number of used filters */
pline->nused--;
@ -1180,7 +1212,7 @@ H5Z_delete(H5O_pline_t *pline, H5Z_filter_t filter)
done:
FUNC_LEAVE_NOAPI(ret_value)
}
} /* end H5Z_delete() */
/*-------------------------------------------------------------------------
* Function: H5Zget_filter_info

42
src/H5Zprivate.h
View File

@ -25,14 +25,9 @@
/* Private headers needed by this file */
#include "H5Tprivate.h" /* Datatypes */
/* Structure to store information about each filter's parameters */
typedef struct {
H5Z_filter_t id; /*filter identification number */
unsigned flags; /*defn and invocation flags */
char *name; /*optional filter name */
size_t cd_nelmts; /*number of elements in cd_values[] */
unsigned *cd_values; /*client data values */
} H5Z_filter_info_t;
/**************************/
/* Library Private Macros */
/**************************/
/* Special parameters for szip compression */
/* [These are aliases for the similar definitions in szlib.h, which we can't
@ -42,6 +37,37 @@ typedef struct {
#define H5_SZIP_MSB_OPTION_MASK 16
#define H5_SZIP_RAW_OPTION_MASK 128
/* Common # of 'client data values' for filters */
/* (avoids dynamic memory allocation in most cases) */
#define H5Z_COMMON_CD_VALUES 4
/* Common size of filter name */
/* (avoids dynamic memory allocation in most cases) */
#define H5Z_COMMON_NAME_LEN 12
/****************************/
/* Library Private Typedefs */
/****************************/
/* Structure to store information about each filter's parameters */
typedef struct {
H5Z_filter_t id; /*filter identification number */
unsigned flags; /*defn and invocation flags */
char _name[H5Z_COMMON_NAME_LEN]; /*internal filter name */
char *name; /*optional filter name */
size_t cd_nelmts; /*number of elements in cd_values[] */
unsigned _cd_values[H5Z_COMMON_CD_VALUES]; /*internal client data values */
unsigned *cd_values; /*client data values */
} H5Z_filter_info_t;
/*****************************/
/* Library-private Variables */
/*****************************/
/***************************************/
/* Library-private Function Prototypes */
/***************************************/
struct H5O_pline_t; /*forward decl*/
/* Internal API routines */

8
src/H5Zpublic.h
View File

@ -42,7 +42,13 @@ typedef int H5Z_filter_t;
/* General macros */
#define H5Z_FILTER_ALL 0 /* Symbol to remove all filters in H5Premove_filter */
#define H5Z_MAX_NFILTERS 32 /* Maximum number of filters allowed in a pipeline (should probably be allowed to be an unlimited amount) */
#define H5Z_MAX_NFILTERS 32 /* Maximum number of filters allowed in a pipeline */
/* (should probably be allowed to be an
* unlimited amount, but currently each
* filter uses a bit in a 32-bit field,
* so the format would have to be
* changed to accomodate that)
*/
/* Flags for filter definition (stored) */
#define H5Z_FLAG_DEFMASK 0x00ff /*definition flag mask */