[svn-r16094] Description:

Add initial support for data block 'page init' bitmasks to super
blocks.

Tested on:
        Mac OS X/32 10.5.5 (amazon) in debug mode
        Mac OS X/32 10.5.5 (amazon) w/C++ & FORTRAN, w/threadsafe,
                                in production mode
        FreeBSD/32 6.3 (duty) in debug mode
        FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode
        Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
                                in debug mode
        Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x,
                                w/C++ & FORTRAN, in production mode
        Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
                                w/szip filter, in production mode
        Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
                                in production mode
        Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
        Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode
This commit is contained in:
Quincey Koziol 2008-11-17 22:42:13 -05:00
parent a28e8407cf
commit a19434caa0
6 changed files with 157 additions and 47 deletions

View File

@ -896,6 +896,7 @@ H5EA__cache_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
uint32_t stored_chksum; /* Stored metadata checksum value */
uint32_t computed_chksum; /* Computed metadata checksum value */
haddr_t arr_addr; /* Address of array header in the file */
size_t u; /* Local index variable */
/* Sanity check */
HDassert(f);
@ -951,15 +952,19 @@ H5EA__cache_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr,
/* Internal information */
/* Decode data block addresses */
if(sblock->ndblks > 0) {
size_t u; /* Local index variable */
/* Check for 'page init' bitmasks for this super block */
if(sblock->dblk_npages > 0) {
size_t tot_page_init_size = sblock->ndblks * sblock->dblk_page_init_size; /* Compute total size of 'page init' buffer */
/* Decode addresses of data blocks in super block */
for(u = 0; u < sblock->ndblks; u++)
H5F_addr_decode(f, &p, &sblock->dblk_addrs[u]);
/* Retrieve the 'page init' bitmasks */
HDmemcpy(sblock->page_init, p, tot_page_init_size);
p += tot_page_init_size;
} /* end if */
/* Decode data block addresses */
for(u = 0; u < sblock->ndblks; u++)
H5F_addr_decode(f, &p, &sblock->dblk_addrs[u]);
/* Sanity check */
/* (allow for checksum not decoded yet) */
HDassert((size_t)(p - buf) == (size - H5EA_SIZEOF_CHKSUM));
@ -1028,6 +1033,7 @@ H5EA__cache_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
uint8_t *p; /* Pointer into raw data buffer */
size_t size; /* Index block size on disk */
uint32_t metadata_chksum; /* Computed metadata checksum value */
size_t u; /* Local index variable */
/* Wrap the local buffer for serialized info */
if(NULL == (wb = H5WB_wrap(ser_buf, sizeof(ser_buf))))
@ -1061,15 +1067,19 @@ H5EA__cache_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr,
/* Internal information */
/* Encode data block addresses */
if(sblock->ndblks > 0) {
size_t u; /* Local index variable */
/* Check for 'page init' bitmasks for this super block */
if(sblock->dblk_npages > 0) {
size_t tot_page_init_size = sblock->ndblks * sblock->dblk_page_init_size; /* Compute total size of 'page init' buffer */
/* Encode addresses of data blocks in super block */
for(u = 0; u < sblock->ndblks; u++)
H5F_addr_encode(f, &p, sblock->dblk_addrs[u]);
/* Store the 'page init' bitmasks */
HDmemcpy(p, sblock->page_init, tot_page_init_size);
p += tot_page_init_size;
} /* end if */
/* Encode addresses of data blocks in super block */
for(u = 0; u < sblock->ndblks; u++)
H5F_addr_encode(f, &p, sblock->dblk_addrs[u]);
/* Compute metadata checksum */
metadata_chksum = H5_checksum_metadata(buf, (size_t)(p - buf), 0);

View File

@ -53,6 +53,10 @@
/* Max. # of bits for max. nelmts index */
#define H5EA_MAX_NELMTS_IDX_MAX 64
/* # of elements in a data block for a particular super block */
#define H5EA_SBLK_DBLK_NELMTS(s, m) \
(size_t)H5_EXP2(((s) + 1) / 2) * (m)
/******************/
/* Local Typedefs */
@ -125,6 +129,9 @@ H5EA__hdr_alloc(H5F_t *f, const H5EA_class_t *cls))
if(NULL == (hdr = H5FL_CALLOC(H5EA_hdr_t)))
H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array shared header")
/* Set non-zero internal fields */
hdr->addr = HADDR_UNDEF;
/* Set the internal parameters for the array */
hdr->f = f;
hdr->sizeof_addr = H5F_SIZEOF_ADDR(f);
@ -197,6 +204,7 @@ H5EA__hdr_init(H5EA_hdr_t *hdr))
/* Compute general information */
hdr->nsblks = 1 + (hdr->cparam.max_nelmts_bits - H5V_log2_of2(hdr->cparam.data_blk_min_elmts));
hdr->dblk_page_nelmts = (size_t)1 << hdr->cparam.max_dblk_page_nelmts_bits;
hdr->arr_off_size = H5EA_SIZEOF_OFFSET_BITS(hdr->cparam.max_nelmts_bits);
#ifdef QAK
HDfprintf(stderr, "%s: hdr->nsblks = %Zu\n", FUNC, hdr->nsblks);
@ -211,7 +219,7 @@ HDfprintf(stderr, "%s: hdr->nsblks = %Zu\n", FUNC, hdr->nsblks);
start_dblk = 0;
for(u = 0; u < hdr->nsblks; u++) {
hdr->sblk_info[u].ndblks = (size_t)H5_EXP2(u / 2);
hdr->sblk_info[u].dblk_nelmts = (size_t)H5_EXP2((u + 1) / 2) * hdr->cparam.data_blk_min_elmts;
hdr->sblk_info[u].dblk_nelmts = H5EA_SBLK_DBLK_NELMTS(u, hdr->cparam.data_blk_min_elmts);
hdr->sblk_info[u].start_idx = start_idx;
hdr->sblk_info[u].start_dblk = start_dblk;
#ifdef QAK
@ -359,7 +367,7 @@ haddr_t, HADDR_UNDEF, HADDR_UNDEF,
H5EA__hdr_create(H5F_t *f, hid_t dxpl_id, const H5EA_create_t *cparam))
/* Local variables */
H5EA_hdr_t *hdr; /* Extensible array header */
H5EA_hdr_t *hdr = NULL; /* Extensible array header */
#ifdef QAK
HDfprintf(stderr, "%s: Called\n", FUNC);
@ -370,6 +378,11 @@ HDfprintf(stderr, "%s: Called\n", FUNC);
HDassert(cparam);
#ifndef NDEBUG
{
unsigned sblk_idx; /* Super block index for first "actual" super block */
size_t dblk_nelmts; /* Number of data block elements */
size_t dblk_page_nelmts; /* Number of elements in a data block page */
/* Check for valid parameters */
if(cparam->raw_elmt_size == 0)
H5E_THROW(H5E_BADVALUE, "element size must be greater than zero")
@ -383,8 +396,19 @@ HDfprintf(stderr, "%s: Called\n", FUNC);
H5E_THROW(H5E_BADVALUE, "min # of data block pointers in super block must be power of two")
if(!POWER_OF_TWO(cparam->data_blk_min_elmts))
H5E_THROW(H5E_BADVALUE, "min # of elements per data block must be power of two")
dblk_page_nelmts = (size_t)1 << cparam->max_dblk_page_nelmts_bits;
if(dblk_page_nelmts < cparam->idx_blk_elmts)
H5E_THROW(H5E_BADVALUE, "# of elements per data block page must be greater than # of elements in index block")
/* Compute the number of elements in data blocks for first actual super block */
sblk_idx = H5EA_SBLK_FIRST_IDX(cparam->sup_blk_min_data_ptrs);
dblk_nelmts = H5EA_SBLK_DBLK_NELMTS(sblk_idx, cparam->data_blk_min_elmts);
if(dblk_page_nelmts < dblk_nelmts)
H5E_THROW(H5E_BADVALUE, "max. # of elements per data block page bits must be > # of elements in first data block from super block")
if(cparam->max_dblk_page_nelmts_bits > cparam->max_nelmts_bits)
H5E_THROW(H5E_BADVALUE, "max. # of elements per data block page bits must be <= max. # of elements bits")
}
#endif /* NDEBUG */
/* Allocate space for the shared information */
@ -413,6 +437,16 @@ HDfprintf(stderr, "%s: Called\n", FUNC);
ret_value = hdr->addr;
CATCH
if(!H5F_addr_defined(ret_value))
if(hdr) {
/* Release header's disk space */
if(H5F_addr_defined(hdr->addr) && H5MF_xfree(f, H5FD_MEM_EARRAY_HDR, dxpl_id, hdr->addr, (hsize_t)hdr->size) < 0)
H5E_THROW(H5E_CANTFREE, "unable to free extensible array header")
/* Destroy header */
if(H5EA__hdr_dest(hdr) < 0)
H5E_THROW(H5E_CANTFREE, "unable to destroy extensible array header")
} /* end if */
END_FUNC(PKG) /* end H5EA__hdr_create() */

View File

@ -123,7 +123,7 @@ H5EA__iblock_alloc(H5EA_hdr_t *hdr))
iblock->addr = HADDR_UNDEF;
/* Compute information */
iblock->nsblks = 2 * H5V_log2_of2((uint32_t)hdr->cparam.sup_blk_min_data_ptrs);
iblock->nsblks = H5EA_SBLK_FIRST_IDX(hdr->cparam.sup_blk_min_data_ptrs);
iblock->ndblk_addrs = 2 * ((size_t)hdr->cparam.sup_blk_min_data_ptrs - 1);
iblock->nsblk_addrs = hdr->nsblks - iblock->nsblks;
#ifdef QAK

View File

@ -407,6 +407,7 @@ func_init_failed: \
+ (s)->hdr->arr_off_size /* Offset of the block in the array */ \
\
/* Extensible Array Super Block specific fields */ \
+ ((s)->ndblks * (s)->dblk_page_init_size) /* Data block 'page init' bitmasks in super block (can be 0 if no pages) */ \
+ ((s)->ndblks * (s)->hdr->sizeof_addr) /* Data block addresses in super block */ \
)
@ -425,8 +426,10 @@ func_init_failed: \
)
/* Compute the # of bytes required to store an offset into a given buffer size */
#define H5EA_SIZEOF_OFFSET_BITS(b) (((b) + 7) / 8)
#define H5EA_SIZEOF_OFFSET_BITS(b) (((b) + 7) / 8)
/* Compute the first super block index that will hold a certain # of data block pointers */
#define H5EA_SBLK_FIRST_IDX(m) (2 * H5V_log2_of2((uint32_t)m))
/****************************/
/* Package Private Typedefs */
@ -477,6 +480,9 @@ typedef struct H5EA_hdr_t {
/* Super block information (not stored) */
size_t nsblks; /* Number of superblocks needed for array */
H5EA_sblk_info_t *sblk_info; /* Array of information for each super block */
/* Data block information (not stored) */
size_t dblk_page_nelmts; /* # of elements per data block page */
} H5EA_hdr_t;
/* The extensible array index block information */
@ -508,7 +514,8 @@ typedef struct H5EA_sblock_t {
/* Extensible array information (stored) */
hsize_t block_off; /* Offset of the block within the array's address space */
haddr_t *dblk_addrs; /* Buffer for addresses of data blocks in super block */
haddr_t *dblk_addrs; /* Addresses of data blocks in super block */
uint8_t *page_init; /* Bitmap of whether a data block page is initialized */
/* Internal array information (not stored) */
size_t rc; /* Reference count of objects using this block */
@ -520,6 +527,8 @@ typedef struct H5EA_sblock_t {
unsigned idx; /* Super block index within the extensible array */
size_t ndblks; /* # of data block addresses that are in super block */
size_t dblk_nelmts; /* # of elements for data blocks reachable through this super block */
size_t dblk_npages; /* # of pages in each data block */
size_t dblk_page_init_size; /* Size of 'page init' bitmask for each data block */
} H5EA_sblock_t;
/* The extensible array data block information */

View File

@ -87,6 +87,9 @@ H5FL_DEFINE_STATIC(H5EA_sblock_t);
/* Declare a free list to manage the haddr_t sequence information */
H5FL_SEQ_DEFINE_STATIC(haddr_t);
/* Declare a free list to manage blocks of 'page init' bitmasks */
H5FL_BLK_DEFINE(page_init);
/*-------------------------------------------------------------------------
@ -122,15 +125,35 @@ H5EA__sblock_alloc(H5EA_hdr_t *hdr, unsigned sblk_idx))
/* Compute/cache information */
sblock->idx = sblk_idx;
sblock->ndblks = hdr->sblk_info[sblk_idx].ndblks;
HDassert(sblock->ndblks);
sblock->dblk_nelmts = hdr->sblk_info[sblk_idx].dblk_nelmts;
#ifdef QAK
HDfprintf(stderr, "%s: sblock->ndblks = %Zu\n", FUNC, sblock->ndblks);
HDfprintf(stderr, "%s: hdr->dblk_page_nelmts = %Zu, sblock->ndblks = %Zu, sblock->dblk_nelmts = %Zu\n", FUNC, hdr->dblk_page_nelmts, sblock->ndblks, sblock->dblk_nelmts);
#endif /* QAK */
/* Allocate buffer for data block addresses in super block */
if(sblock->ndblks > 0)
if(NULL == (sblock->dblk_addrs = H5FL_SEQ_MALLOC(haddr_t, sblock->ndblks)))
H5E_THROW(H5E_CANTALLOC, "memory allocation failed for super block data block addresses")
if(NULL == (sblock->dblk_addrs = H5FL_SEQ_MALLOC(haddr_t, sblock->ndblks)))
H5E_THROW(H5E_CANTALLOC, "memory allocation failed for super block data block addresses")
/* Check if # of elements in data blocks requires paging */
if(sblock->dblk_nelmts > hdr->dblk_page_nelmts) {
/* Compute # of pages in each data block from this super block */
sblock->dblk_npages = sblock->dblk_nelmts / hdr->dblk_page_nelmts;
/* Sanity check that we have at least 2 pages in data block */
HDassert(sblock->dblk_npages > 1);
/* Sanity check for integer truncation */
HDassert((sblock->dblk_npages * hdr->dblk_page_nelmts) == sblock->dblk_nelmts);
/* Compute size of buffer for each data block's 'page init' bitmask */
sblock->dblk_page_init_size = ((sblock->dblk_npages) + 7) / 8;
HDassert(sblock->dblk_page_init_size > 0);
/* Allocate buffer for all 'page init' bitmasks in super block */
if(NULL == (sblock->page_init = H5FL_BLK_CALLOC(page_init, sblock->ndblks * sblock->dblk_page_init_size)))
H5E_THROW(H5E_CANTALLOC, "memory allocation failed for super block page init bitmask")
} /* end if */
/* Share common array information */
sblock->hdr = hdr;
@ -167,6 +190,7 @@ H5EA__sblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, unsigned sblk_idx))
/* Local variables */
H5EA_sblock_t *sblock = NULL; /* Extensible array super block */
haddr_t tmp_addr = HADDR_UNDEF; /* Address value to fill data block addresses with */
#ifdef QAK
HDfprintf(stderr, "%s: Called\n", FUNC);
@ -195,13 +219,8 @@ HDfprintf(stderr, "%s: sblock->block_off = %Hu\n", FUNC, sblock->block_off);
if(HADDR_UNDEF == (sblock->addr = H5MF_alloc(hdr->f, H5FD_MEM_EARRAY_SBLOCK, dxpl_id, (hsize_t)sblock->size)))
H5E_THROW(H5E_CANTALLOC, "file allocation failed for extensible array super block")
/* Reset data block addresses */
if(sblock->ndblks > 0) {
haddr_t tmp_addr = HADDR_UNDEF; /* Address value to fill data block addresses with */
/* Set all the data block addresses to "undefined" address value */
H5V_array_fill(sblock->dblk_addrs, &tmp_addr, sizeof(haddr_t), sblock->ndblks);
} /* end if */
/* Reset data block addresses to "undefined" address value */
H5V_array_fill(sblock->dblk_addrs, &tmp_addr, sizeof(haddr_t), sblock->ndblks);
/* Cache the new extensible array super block */
if(H5AC_set(hdr->f, dxpl_id, H5AC_EARRAY_SBLOCK, sblock->addr, sblock, H5AC__NO_FLAGS_SET) < 0)
@ -317,6 +336,7 @@ H5EA__sblock_delete(H5EA_hdr_t *hdr, hid_t dxpl_id, haddr_t sblk_addr,
/* Local variables */
H5EA_sblock_t *sblock = NULL; /* Pointer to super block */
size_t u; /* Local index variable */
#ifdef QAK
HDfprintf(stderr, "%s: Called\n", FUNC);
@ -330,21 +350,16 @@ HDfprintf(stderr, "%s: Called\n", FUNC);
if(NULL == (sblock = H5EA__sblock_protect(hdr, dxpl_id, sblk_addr, sblk_idx, H5AC_WRITE)))
H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array super block, address = %llu", (unsigned long_long)sblk_addr)
/* Check for super block having data block pointers */
if(sblock->ndblks > 0) {
size_t u; /* Local index variable */
/* Iterate over data blocks */
for(u = 0; u < sblock->ndblks; u++) {
/* Check for data block existing */
if(H5F_addr_defined(sblock->dblk_addrs[u])) {
/* Delete data block */
if(H5EA__dblock_delete(hdr, dxpl_id, sblock->dblk_addrs[u], sblock->dblk_nelmts) < 0)
H5E_THROW(H5E_CANTDELETE, "unable to delete extensible array data block")
sblock->dblk_addrs[u] = HADDR_UNDEF;
} /* end if */
} /* end for */
} /* end if */
/* Iterate over data blocks */
for(u = 0; u < sblock->ndblks; u++) {
/* Check for data block existing */
if(H5F_addr_defined(sblock->dblk_addrs[u])) {
/* Delete data block */
if(H5EA__dblock_delete(hdr, dxpl_id, sblock->dblk_addrs[u], sblock->dblk_nelmts) < 0)
H5E_THROW(H5E_CANTDELETE, "unable to delete extensible array data block")
sblock->dblk_addrs[u] = HADDR_UNDEF;
} /* end if */
} /* end for */
CATCH
/* Finished deleting super block in metadata cache */
@ -375,15 +390,21 @@ H5EA__sblock_dest(H5F_t *f, H5EA_sblock_t *sblock))
/* Sanity check */
HDassert(sblock);
HDassert(sblock->rc == 0);
#ifdef QAK
HDfprintf(stderr, "%s: sblock->hdr->dblk_page_nelmts = %Zu, sblock->ndblks = %Zu, sblock->dblk_nelmts = %Zu\n", FUNC, sblock->hdr->dblk_page_nelmts, sblock->ndblks, sblock->dblk_nelmts);
#endif /* QAK */
/* Set the shared array header's file context for this operation */
sblock->hdr->f = f;
/* Check if we've got data block addresses in the super block */
if(sblock->ndblks > 0) {
/* Free buffer for super block data block addresses */
HDassert(sblock->dblk_addrs);
/* Free buffer for super block data block addresses, if there are any */
if(sblock->dblk_addrs)
(void)H5FL_SEQ_FREE(haddr_t, sblock->dblk_addrs);
/* Free buffer for super block 'page init' bitmask, if there is one */
if(sblock->page_init) {
HDassert(sblock->dblk_npages > 0);
(void)H5FL_BLK_FREE(page_init, sblock->page_init);
} /* end if */
/* Decrement reference count on shared info */

View File

@ -565,6 +565,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -579,6 +580,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -592,6 +594,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -606,6 +609,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -618,6 +622,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -630,6 +635,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
@ -644,12 +650,41 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
} /* end if */
/* Set invalid max. # of elements per data block page bits */
if(test_cparam.idx_blk_elmts > 0) {
HDmemcpy(&test_cparam, cparam, sizeof(test_cparam));
test_cparam.max_dblk_page_nelmts_bits = H5V_log2_gen((uint64_t)test_cparam.idx_blk_elmts) - 1;
H5E_BEGIN_TRY {
ea = H5EA_create(f, H5P_DATASET_XFER_DEFAULT, &test_cparam);
} H5E_END_TRY;
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
} /* end if */
} /* end if */
HDmemcpy(&test_cparam, cparam, sizeof(test_cparam));
test_cparam.max_dblk_page_nelmts_bits = 4; /* corresponds to 16 elements in data block page, which is less than the 64 elements for the default settings */
H5E_BEGIN_TRY {
ea = H5EA_create(f, H5P_DATASET_XFER_DEFAULT, &test_cparam);
} H5E_END_TRY;
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR
} /* end if */
HDmemcpy(&test_cparam, cparam, sizeof(test_cparam));
test_cparam.max_dblk_page_nelmts_bits = test_cparam.max_nelmts_bits + 1;
H5E_BEGIN_TRY {
@ -658,6 +693,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t UNUSED *tpara
if(ea) {
/* Close opened extensible array */
H5EA_close(ea, H5P_DATASET_XFER_DEFAULT);
ea = NULL;
/* Indicate error */
TEST_ERROR