[svn-r13814] Description:

Change back to clearing memory buffers by default, but give users option
to disable this, for performance reasons (--disable-clear-file-buffers).

    Also, add configure option (--enable-using-memchecker) to conveniently
control making the library "play nice" with memory checking tools like valgrind
and purify.

Tested on:
    Mac OS X/32 10.4.9 (amazon)
    FreeBSD/32 6.2 (duty)
    Linux/32 2.6 (kagiso)
This commit is contained in:
Quincey Koziol 2007-05-29 15:02:34 -05:00
parent 7b0fa3950e
commit bdcbada3f2
15 changed files with 130 additions and 62 deletions

View File

@ -2190,6 +2190,65 @@ case "X-$INSTRUMENT" in
;;
esac
dnl ----------------------------------------------------------------------
dnl Check if they would like to securely clear file buffers before they are
dnl written.
dnl
AC_SUBST([CLEARFILEBUF])
AC_MSG_CHECKING([whether to clear file buffers])
AC_ARG_ENABLE([clear-file-buffers],
[AC_HELP_STRING([--enable-clear-file-buffers],
[Securely clear file buffers before writing
to file. Default=yes.])],
[CLEARFILEBUF=$enableval])
case "X-$CLEARFILEBUF" in
*)
CLEARFILEBUF=yes
AC_MSG_RESULT([yes])
AC_DEFINE([CLEAR_MEMORY], [1],
[Define if the memory buffers being written to disk should be
cleared before writing.])
;;
X-no)
CLEARFILEBUF=no
AC_MSG_RESULT([no])
;;
esac
dnl ----------------------------------------------------------------------
dnl Check if they would like to use a memory checking tool (like valgrind's
dnl 'memcheck' tool, or Rational Purify, etc) and the library should be
dnl more scrupulous with it's memory operations. Enabling this also
dnl disables the library's free space manager code.
dnl
AC_SUBST([USINGMEMCHECKER])
AC_MSG_CHECKING([whether a memory checking tool will be used])
AC_ARG_ENABLE([using-memchecker],
[AC_HELP_STRING([--enable-using-memchecker],
[Enable this option if a memory allocation and/or
bounds checking tool will be used on the HDF5
library. Enabling this causes the library to be
more picky about it's memory operations and also
disables the library's free space manager code.
Default=no.])],
[USINGMEMCHECKER=$enableval])
case "X-$USINGMEMCHECKER" in
X-yes)
USINGMEMCHECKER=yes
AC_MSG_RESULT([yes])
AC_DEFINE([USING_MEMCHECKER], [1],
[Define if a memory checking tool will be used on the library,
to cause library to be very picky about memory operations and
also disable the internal free list manager code.])
;;
*)
USINGMEMCHECKER=no
AC_MSG_RESULT([no])
;;
esac
dnl Checkpoint the cache
AC_CACHE_SAVE
@ -3789,6 +3848,12 @@ IF_YES_NO "$STATIC_EXEC"
PRINT_N " Tracing"
IF_YES_NO "$TRACE_API"
PRINT_N "Clear file buffers before write"
IF_YES_NO "$CLEARFILEBUF"
PRINT_N " Using memory checker"
IF_YES_NO "$USINGMEMCHECKER"
PRINT_N " Optimization Instrumentation"
IF_YES_NO "$INSTRUMENT"

View File

@ -155,11 +155,11 @@ New Features
- Long double is assumed to be a supported C data type. It is a
stanadard C89 type. AKC - 2004/10/22
- The IA64 will use ecc as the C++ compiler by default.
- Added some initial support for making purify (or similar memory
checking products) happier by initializing buffers to zero and
disabling the internal free list code. To take advantage of this,
define 'H5_USING_PURIFY' in your CFLAGS when building the library.
QAK - 2004/07/23
- Added some initial support for making valgrind/Purify (or similar
memory checking products) happier by initializing buffers to zero
and disabling the internal free list code. To take advantage of
this, use the "--enable-using-memchecker" configure option when
building the library. QAK - 2004/07/23
- Fixed the long compile time of H5detect.c when v7.x Intel Compiler
is used with optimization NOT off. AKC - 2004/05/20
- Fixed configure setting of C++ for OSF1 platform. AKC - 2004/01/06

View File

@ -584,11 +584,11 @@ H5A_open_common(const H5G_loc_t *loc, H5A_t *attr)
HDassert(loc);
HDassert(attr);
#if defined(H5_USING_PURIFY) || !defined(NDEBUG)
#if defined(H5_USING_MEMCHECKER) || !defined(NDEBUG)
/* Clear object location */
if(H5O_loc_reset(&(attr->oloc)) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTOPENOBJ, FAIL, "unable to reset location")
#endif /* H5_USING_PURIFY */
#endif /* H5_USING_MEMCHECKER */
/* Free any previous group hier. path */
if(H5G_name_free(&(attr->path)) < 0)
@ -1989,7 +1989,7 @@ H5A_copy(H5A_t *_new_attr, const H5A_t *old_attr)
if(NULL == (new_attr->ds = H5S_copy(old_attr->ds, FALSE)))
HGOTO_ERROR(H5E_ATTR, H5E_CANTCOPY, NULL, "unable to copy attribute dataspace")
#if defined(H5_USING_PURIFY) || !defined(NDEBUG)
#if defined(H5_USING_MEMCHECKER) || !defined(NDEBUG)
/* Clear object location */
if(H5O_loc_reset(&(new_attr->oloc)) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTOPENOBJ, NULL, "unable to reset location")
@ -1997,7 +1997,7 @@ H5A_copy(H5A_t *_new_attr, const H5A_t *old_attr)
/* Clear path name */
if(H5G_name_reset(&(new_attr->path)) < 0)
HGOTO_ERROR(H5E_ATTR, H5E_CANTOPENOBJ, NULL, "unable to reset path")
#endif /* H5_USING_PURIFY */
#endif /* H5_USING_MEMCHECKER */
/* Copy the object location and group path */
if(H5O_loc_copy(&(new_attr->oloc), &(old_attr->oloc), H5_COPY_DEEP) < 0)

View File

@ -173,9 +173,9 @@ H5B2_shared_init (H5F_t *f, H5B2_t *bt2, const H5B2_class_t *type,
/* Allocate "page" for node I/O */
if((shared->page = H5FL_BLK_MALLOC(node_page, shared->node_size)) == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
#ifdef H5_USING_PURIFY
HDmemset(shared->page,0,shared->node_size);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(shared->page, 0, shared->node_size);
#endif /* H5_CLEAR_MEMORY */
/* Allocate array of node info structs */
if((shared->node_info = H5FL_SEQ_MALLOC(H5B2_node_info_t, (size_t)(shared->depth + 1))) == NULL)
@ -1946,9 +1946,9 @@ H5B2_create_leaf(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, H5B2_node_ptr_t *n
/* Allocate space for the native keys in memory */
if((leaf->leaf_native = (uint8_t *)H5FL_FAC_MALLOC(shared->node_info[0].nat_rec_fac)) == NULL)
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree leaf native keys")
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
HDmemset(leaf->leaf_native, 0, shared->type->nrec_size * shared->node_info[0].max_nrec);
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
/* Set number of records */
leaf->nrec = 0;
@ -2019,16 +2019,16 @@ H5B2_create_internal(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared,
/* Allocate space for the native keys in memory */
if((internal->int_native = (uint8_t *)H5FL_FAC_MALLOC(shared->node_info[depth].nat_rec_fac)) == NULL)
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal native keys")
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
HDmemset(internal->int_native, 0, shared->type->nrec_size * shared->node_info[depth].max_nrec);
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
/* Allocate space for the node pointers in memory */
if((internal->node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_MALLOC(shared->node_info[depth].node_ptr_fac)) == NULL)
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal node pointers")
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
HDmemset(internal->node_ptrs, 0, sizeof(H5B2_node_ptr_t) * (shared->node_info[depth].max_nrec + 1));
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
/* Set number of records & depth of the node */
internal->nrec = 0;

View File

@ -753,10 +753,10 @@ H5D_contig_writevv(const H5D_io_info_t *io_info,
/* Allocate room for the data sieve buffer */
if (NULL==(dset_contig->sieve_buf=H5FL_BLK_MALLOC(sieve_buf,dset_contig->sieve_buf_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
if(dset_contig->sieve_size > size)
HDmemset(dset_contig->sieve_buf + size, 0, (dset_contig->sieve_size - size));
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
/* Determine the new sieve buffer size & location */
dset_contig->sieve_loc=addr;

View File

@ -1526,9 +1526,9 @@ H5D_istore_shared_create (const H5F_t *f, H5O_layout_t *layout)
assert(shared->sizeof_rnode);
if(NULL==(shared->page=H5FL_BLK_MALLOC(chunk_page,shared->sizeof_rnode)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page")
#ifdef H5_USING_PURIFY
HDmemset(shared->page,0,shared->sizeof_rnode);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(shared->page, 0, shared->sizeof_rnode);
#endif /* H5_CLEAR_MEMORY */
if(NULL==(shared->nkey=H5FL_SEQ_MALLOC(size_t,(size_t)(2*H5F_KVALUE(f,H5B_ISTORE)+1))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page")
@ -1859,10 +1859,10 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
HDmemset(chunk, 0, chunk_size);
} /* end else */
} /* end if */
#ifdef H5_USING_PURIFY
else
HDmemset(chunk, 0, chunk_size);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
else
HDmemset(chunk, 0, chunk_size);
#endif /* H5_CLEAR_MEMORY */
#ifdef H5D_ISTORE_DEBUG
rdcc->ninits++;
#endif /* H5D_ISTORE_DEBUG */

View File

@ -3491,9 +3491,9 @@ H5FD_write(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t si
/* Reallocate the metadata accumulator buffer */
if((file->meta_accum=H5FL_BLK_REALLOC(meta_accum,file->meta_accum,file->accum_buf_size))==NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer")
#ifdef H5_USING_PURIFY
HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_size));
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(file->meta_accum + file->accum_size, 0, (file->accum_buf_size - (file->accum_size + size)));
#endif /* H5_CLEAR_MEMORY */
} /* end if */
/* Move the existing metadata to the proper location */
@ -3519,9 +3519,9 @@ HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_s
/* Reallocate the metadata accumulator buffer */
if((file->meta_accum=H5FL_BLK_REALLOC(meta_accum,file->meta_accum,file->accum_buf_size))==NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer")
#ifdef H5_USING_PURIFY
HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_size));
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(file->meta_accum + file->accum_size + size, 0, (file->accum_buf_size - (file->accum_size + size)));
#endif /* H5_CLEAR_MEMORY */
} /* end if */
/* Copy the new metadata to the end */
@ -3554,9 +3554,9 @@ HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_s
/* Reallocate the metadata accumulator buffer */
if((file->meta_accum=H5FL_BLK_REALLOC(meta_accum,file->meta_accum,file->accum_buf_size))==NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer")
#ifdef H5_USING_PURIFY
HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_size));
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(file->meta_accum + file->accum_size, 0, (file->accum_buf_size - file->accum_size));
#endif /* H5_CLEAR_MEMORY */
} /* end if */
/* Calculate the proper offset of the existing metadata */
@ -3588,9 +3588,9 @@ HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_s
/* Reallocate the metadata accumulator buffer */
if((file->meta_accum=H5FL_BLK_REALLOC(meta_accum,file->meta_accum,file->accum_buf_size))==NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer")
#ifdef H5_USING_PURIFY
HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_size));
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(file->meta_accum + file->accum_size, 0, (file->accum_buf_size - file->accum_size));
#endif /* H5_CLEAR_MEMORY */
} /* end if */
/* Copy the new metadata to the end */
@ -3625,9 +3625,12 @@ HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_s
/* Note the new buffer size */
file->accum_buf_size=size;
#ifdef H5_USING_PURIFY
HDmemset(file->meta_accum+file->accum_size,0,(file->accum_buf_size-file->accum_size));
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
{
size_t clear_size = MAX(file->accum_size, size);
HDmemset(file->meta_accum + clear_size, 0, (file->accum_buf_size - clear_size));
}
#endif /* H5_CLEAR_MEMORY */
} /* end if */
else {
/* Check if we should shrink the accumulator buffer */

View File

@ -826,9 +826,9 @@ H5FD_core_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, had
x = H5MM_malloc(new_eof);
else
x = H5MM_realloc(file->mem, new_eof);
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
HDmemset(x + file->eof, 0, (size_t)(new_eof - file->eof));
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
if (!x)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory block")
file->mem = x;

View File

@ -37,7 +37,7 @@
/* Macros for turning off free lists in the library */
/*#define H5_NO_FREE_LISTS*/
#if defined H5_NO_FREE_LISTS || defined H5_USING_PURIFY
#if defined H5_NO_FREE_LISTS || defined H5_USING_MEMCHECKER
#define H5_NO_REG_FREE_LISTS
#define H5_NO_ARR_FREE_LISTS
#define H5_NO_SEQ_FREE_LISTS

View File

@ -392,10 +392,10 @@ H5G_name_copy(H5G_name_t *dst, const H5G_name_t *src, H5_copy_depth_t depth)
/* Check arguments */
HDassert(src);
HDassert(dst);
#if defined(H5_USING_PURIFY) || !defined(NDEBUG)
#if defined(H5_USING_MEMCHECKER) || !defined(NDEBUG)
HDassert(dst->full_path_r == NULL);
HDassert(dst->user_path_r == NULL);
#endif /* H5_USING_PURIFY */
#endif /* H5_USING_MEMCHECKER */
HDassert(depth == H5_COPY_SHALLOW || depth == H5_COPY_DEEP);
/* Copy the top level information */

View File

@ -1604,9 +1604,9 @@ H5G_node_init(H5F_t *f)
assert(shared->sizeof_rnode);
if(NULL==(shared->page=H5FL_BLK_MALLOC(grp_page,shared->sizeof_rnode)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page")
#ifdef H5_USING_PURIFY
HDmemset(shared->page,0,shared->sizeof_rnode);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(shared->page, 0, shared->sizeof_rnode);
#endif /* H5_CLEAR_MEMORY */
if(NULL==(shared->nkey=H5FL_SEQ_MALLOC(size_t,(size_t)(2*H5F_KVALUE(f,H5B_SNODE)+1))))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page")

View File

@ -591,11 +591,11 @@ H5G_traverse_real(const H5G_loc_t *_loc, const char *name, unsigned target,
obj_loc.oloc = &obj_oloc;
obj_loc.path = &obj_path;
#if defined(H5_USING_PURIFY) || !defined(NDEBUG)
#if defined(H5_USING_MEMCHECKER) || !defined(NDEBUG)
/* Clear group location */
if(H5G_loc_reset(&grp_loc) < 0)
HGOTO_ERROR(H5E_SYM, H5E_CANTOPENOBJ, FAIL, "unable to reset location")
#endif /* H5_USING_PURIFY */
#endif /* H5_USING_MEMCHECKER */
/* Deep copy of the starting location to group location */
if(H5G_loc_copy(&grp_loc, &loc, H5_COPY_DEEP) < 0)

View File

@ -144,9 +144,9 @@ H5HF_man_dblock_create(hid_t dxpl_id, H5HF_hdr_t *hdr, H5HF_indirect_t *par_iblo
/* XXX: Change to using free-list factories */
if((dblock->blk = H5FL_BLK_MALLOC(direct_block, dblock->size)) == NULL)
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
#ifdef H5_USING_PURIFY
#ifdef H5_CLEAR_MEMORY
HDmemset(dblock->blk, 0, dblock->size);
#endif /* H5_USING_PURIFY */
#endif /* H5_CLEAR_MEMORY */
/* Allocate space for the direct block on disk */
if(HADDR_UNDEF == (dblock_addr = H5MF_alloc(hdr->f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)dblock->size)))

View File

@ -223,9 +223,9 @@ H5HG_create (H5F_t *f, hid_t dxpl_id, size_t size)
if (NULL==(heap->chunk = H5FL_BLK_MALLOC (heap_chunk,size)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, HADDR_UNDEF, \
"memory allocation failed");
#ifdef H5_USING_PURIFY
HDmemset(heap->chunk,0,size);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(heap->chunk, 0, size);
#endif /* H5_CLEAR_MEMORY */
heap->nalloc = H5HG_NOBJS (f, size);
heap->nused = 1; /* account for index 0, which is used for the free object */
if (NULL==(heap->obj = H5FL_SEQ_MALLOC (H5HG_obj_t,heap->nalloc)))
@ -845,9 +845,9 @@ H5HG_extend (H5F_t *f, H5HG_heap_t *heap, size_t size, unsigned * heap_flags_ptr
/* Re-allocate the heap information in memory */
if (NULL==(new_chunk = H5FL_BLK_REALLOC (heap_chunk, heap->chunk, heap->size+need)))
HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "new heap allocation failed");
#ifdef H5_USING_PURIFY
HDmemset(new_chunk+heap->size,0,need);
#endif /* H5_USING_PURIFY */
#ifdef H5_CLEAR_MEMORY
HDmemset(new_chunk + heap->size, 0, need);
#endif /* H5_CLEAR_MEMORY */
/* Adjust the size of the heap */
old_size=heap->size;

View File

@ -665,7 +665,7 @@ H5T_open(const H5G_loc_t *loc, hid_t dxpl_id)
if(NULL == (dt = H5FL_MALLOC(H5T_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "can't allocate space for datatype")
#if defined(H5_USING_PURIFY) || !defined(NDEBUG)
#if defined(H5_USING_MEMCHECKER) || !defined(NDEBUG)
/* Clear object location */
if(H5O_loc_reset(&(dt->oloc)) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTOPENOBJ, NULL, "unable to reset location")
@ -673,7 +673,7 @@ H5T_open(const H5G_loc_t *loc, hid_t dxpl_id)
/* Clear path name */
if(H5G_name_reset(&(dt->path)) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTOPENOBJ, NULL, "unable to reset path")
#endif /* H5_USING_PURIFY */
#endif /* H5_USING_MEMCHECKER */
/* Shallow copy (take ownership) of the object location object */
if(H5O_loc_copy(&dt->oloc, loc->oloc, H5_COPY_SHALLOW) < 0)