[svn-r24993] Purpose:

Adds write tracking to the core VFD, which can be configured
via the H5Pset/get_core_write_tracking() API call. When enabled
and writing to the backing store is enabled, this feature will
track writes and only write out the changed bytes on flush/close.

Tested on:
    32-bit LE linux (jam) w/ Fortran and C++
    64-bit BE linux (ostrich)
    64-bit Darwin 12.5.0 (kite)

All were tested using the core VFD w/ paging on (core_paged
in the VFD list). Make check-vfd was also tested on jam.
This commit is contained in:
Dana Robinson 2014-04-08 22:35:16 -05:00
parent 385b4b40ae
commit 5ae7ad342d
14 changed files with 526 additions and 88 deletions

View File

@ -25,19 +25,26 @@
/* Interface initialization */
#define H5_INTERFACE_INIT_FUNC H5FD_core_init_interface
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fprivate.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5FDcore.h" /* Core file driver */
#include "H5FLprivate.h" /* Free lists */
#include "H5Iprivate.h" /* IDs */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
#include "H5SLprivate.h" /* Skip lists */
/* The driver identification number, initialized at runtime */
static hid_t H5FD_CORE_g = 0;
/* The skip list node type. Represents a region in the file. */
typedef struct H5FD_core_region_t {
haddr_t start; /* Start address of the region */
haddr_t end; /* End address of the region */
} H5FD_core_region_t;
/* The description of a file belonging to this driver. The 'eoa' and 'eof'
* determine the amount of hdf5 address space in use and the high-water mark
* of the file (the current size of the underlying memory).
@ -50,6 +57,7 @@ typedef struct H5FD_core_t {
haddr_t eof; /* current allocated size */
size_t increment; /* multiples for mem allocation */
hbool_t backing_store; /* write to file name on flush */
size_t bstore_page_size; /* backing store page size */
int fd; /* backing store file descriptor */
/* Information for determining uniqueness of a file with a backing store */
#ifndef H5_HAVE_WIN32_API
@ -85,6 +93,7 @@ typedef struct H5FD_core_t {
#endif /* H5_HAVE_WIN32_API */
hbool_t dirty; /* changes not saved? */
H5FD_file_image_callbacks_t fi_callbacks; /* file image callbacks */
H5SL_t *dirty_list; /* dirty parts of the file */
} H5FD_core_t;
/* Driver-specific file access properties */
@ -96,6 +105,9 @@ typedef struct H5FD_core_fapl_t {
/* Allocate memory in multiples of this size by default */
#define H5FD_CORE_INCREMENT 8192
/* Define the block size for aggregation */
//#define H5FD_CORE_BLOCK_SIZE 524288 /* 512 K */
/* These macros check for overflow of various quantities. These macros
* assume that file_offset_t is signed and haddr_t and size_t are unsigned.
*
@ -117,6 +129,9 @@ typedef struct H5FD_core_fapl_t {
(size_t)((A)+(Z))<(size_t)(A))
/* Prototypes */
static herr_t H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end);
static herr_t H5FD_core_destroy_dirty_list(H5FD_core_t *file);
static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size);
static herr_t H5FD_core_term(void);
static void *H5FD_core_fapl_get(H5FD_t *_file);
static H5FD_t *H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id,
@ -170,6 +185,256 @@ static const H5FD_class_t H5FD_core_g = {
H5FD_FLMAP_DICHOTOMY /* fl_map */
};
/* Define a free list to manage the region type */
H5FL_DEFINE(H5FD_core_region_t);
/*-------------------------------------------------------------------------
* Function: H5FD_core_add_dirty_region
*
* Purpose: Add a new dirty region to the list for later flushing
* to the backing store.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end)
{
H5FD_core_region_t *b_item = NULL;
H5FD_core_region_t *a_item = NULL;
H5FD_core_region_t *item = NULL;
haddr_t b_addr = 0;
haddr_t a_addr = 0;
hbool_t create_new_node = TRUE;
herr_t ret_value = SUCCEED;
#ifdef DER
hbool_t was_adjusted = FALSE;
#endif
FUNC_ENTER_NOAPI_NOINIT
HDassert(file);
HDassert(file->dirty_list);
HDassert(start <= end);
#ifdef DER
fprintf(stderr, "Add region: (%llu, %llu)\n", start, end);
#endif
// /* Adjust the dirty region to the nearest block boundaries */
// if(start % H5FD_CORE_BLOCK_SIZE != 0) {
// start = (start / H5FD_CORE_BLOCK_SIZE) * H5FD_CORE_BLOCK_SIZE;
/* Adjust the dirty region to the nearest block boundaries */
if(start % file->bstore_page_size != 0) {
start = (start / file->bstore_page_size) * file->bstore_page_size;
#ifdef DER
was_adjusted = TRUE;
#endif
}
// if(end % H5FD_CORE_BLOCK_SIZE != (H5FD_CORE_BLOCK_SIZE - 1)) {
// end = (((end / H5FD_CORE_BLOCK_SIZE) + 1) * H5FD_CORE_BLOCK_SIZE) - 1;
if(end % file->bstore_page_size != (file->bstore_page_size - 1)) {
end = (((end / file->bstore_page_size) + 1) * file->bstore_page_size) - 1;
if(end > file->eof){
#ifdef DER
fprintf(stderr, "Adjusted to EOF\n");
#endif
end = file->eof - 1;
}
#ifdef DER
was_adjusted = TRUE;
#endif
}
#ifdef DER
if(was_adjusted)
fprintf(stderr, "Adjusted region: (%llu, %llu)\n", start, end);
#endif
/* Get the regions before and after the intended insertion point */
b_addr = start +1;
a_addr = end + 2;
b_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &b_addr);
a_item = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &a_addr);
/* Check to see if we need to extend the upper end of the NEW region */
if(a_item) {
if(start < a_item->start && end < a_item->end) {
/* Extend the end of the NEW region to match the existing AFTER region */
end = a_item->end;
}
}
/* Attempt to extend the PREV region */
if(b_item) {
if(start <= b_item->end + 1) {
/* Need to set this for the delete algorithm */
start = b_item->start;
/* We won't need to insert a new node since we can
* just update an existing one instead.
*/
create_new_node = FALSE;
}
}
/* Remove any old nodes that are no longer needed */
while(a_item && a_item->start > start) {
H5FD_core_region_t *less;
haddr_t key = a_item->start - 1;
/* Save the previous node before we trash this one */
less = (H5FD_core_region_t *)H5SL_less(file->dirty_list, &key);
/* Delete this node */
a_item = (H5FD_core_region_t *)H5SL_remove(file->dirty_list, &a_item->start);
a_item = H5FL_FREE(H5FD_core_region_t, a_item);
/* Set up to check the next node */
if(less)
a_item = less;
}
/* Insert the new node */
if(create_new_node) {
if(NULL == (item = (H5FD_core_region_t *)H5SL_search(file->dirty_list, &start))) {
/* Ok to insert. No pre-existing node with that key. */
item = (H5FD_core_region_t *)H5FL_CALLOC(H5FD_core_region_t);
item->start = start;
item->end = end;
if(H5SL_insert(file->dirty_list, item, &item->start) < 0)
HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, FAIL, "can't insert new dirty region: (%llu, %llu)\n", start, end)
}
else {
/* Store the new item endpoint if it's bigger */
item->end = (item->end < end) ? end : item->end;
}
}
else {
/* Update the size of the before region */
if(b_item->end < end)
b_item->end = end;
}
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_add_dirty_region() */
/*-------------------------------------------------------------------------
* Function: H5FD_core_destroy_dirty_list
*
* Purpose: Completely destroy the dirty list.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_core_destroy_dirty_list(H5FD_core_t *file)
{
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI_NOINIT
HDassert(file);
/* Destroy the list, including any remaining list elements */
if(file->dirty_list) {
H5FD_core_region_t *region = NULL;
#ifdef DER
{
size_t count = H5SL_count(file->dirty_list);
if(count != 0)
fprintf(stderr, "LIST NOT EMPTY AT DESTROY\n");
}
#endif
while(NULL != (region = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list)))
region = H5FL_FREE(H5FD_core_region_t, region);
if(H5SL_close(file->dirty_list) < 0)
HGOTO_ERROR(H5E_SLIST, H5E_CLOSEERROR, FAIL, "can't close core vfd dirty list")
file->dirty_list = NULL;
}
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_destroy_dirty_list() */
/*-------------------------------------------------------------------------
* Function: H5FD_core_write_to_bstore
*
* Purpose: Write data to the backing store.
*
* Return: SUCCEED/FAIL
*
*-------------------------------------------------------------------------
*/
static herr_t H5FD_core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size)
{
unsigned char *ptr = file->mem + addr; /* mutable pointer into the
* buffer (can't change mem)
*/
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
HDassert(file);
/* Write to backing store */
if((off_t)addr != HDlseek(file->fd, (off_t)addr, SEEK_SET))
HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store")
while (size > 0) {
h5_posix_io_t bytes_in = 0; /* # of bytes to write */
h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */
/* Trying to write more bytes than the return type can handle is
* undefined behavior in POSIX.
*/
if(size > H5_POSIX_MAX_IO_BYTES)
bytes_in = H5_POSIX_MAX_IO_BYTES;
else
bytes_in = (h5_posix_io_t)size;
#ifdef DER
fprintf(stderr, "\nNEW\n");
#endif
do {
bytes_wrote = HDwrite(file->fd, ptr, bytes_in);
#ifdef DER
fprintf(stderr, "bytes wrote: %lu\n", bytes_wrote);
#endif
} while(-1 == bytes_wrote && EINTR == errno);
if(-1 == bytes_wrote) { /* error */
int myerrno = errno;
time_t mytime = HDtime(NULL);
HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset);
} /* end if */
HDassert(bytes_wrote > 0);
HDassert((size_t)bytes_wrote <= size);
size -= (size_t)bytes_wrote;
ptr = (unsigned char *)ptr + bytes_wrote;
} /* end while */
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_write_to_bstore() */
/*-------------------------------------------------------------------------
* Function: H5FD_core_init_interface
@ -566,6 +831,40 @@ H5FD_core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
} /* end if */
} /* end if */
/* Set up write tracking if the backing store is on */
file->dirty_list = NULL;
if(fa->backing_store) {
hbool_t write_tracking_flag = FALSE; /* what the user asked for */
hbool_t use_write_tracking = FALSE; /* what we're actually doing */
/* Get the write tracking flag */
if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &write_tracking_flag) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking flag");
/* Get the page size */
if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &(file->bstore_page_size)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get core VFD write tracking page size");
/* default is to have write tracking OFF for create (hence the check to see
* if the user explicitly set a page size) and ON with the default page size
* on open (when not read-only).
*/
/* Only use write tracking if the file is open for writing */
use_write_tracking =
TRUE == write_tracking_flag /* user asked for write tracking */
&& !(o_flags & O_RDONLY) /* file is open for writing (i.e. not read-only) */
&& file->bstore_page_size != 0; /* page size is not zero */
/* initialize the dirty list */
if(use_write_tracking) {
if(NULL == (file->dirty_list = H5SL_create(H5SL_TYPE_HADDR, NULL)))
HGOTO_ERROR(H5E_SLIST, H5E_CANTCREATE, NULL, "can't create core vfd dirty region list");
#ifdef DER
fprintf(stderr, "\n");
#endif
} /* end if */
} /* end if */
/* Set return value */
ret_value = (H5FD_t *)file;
@ -604,7 +903,12 @@ H5FD_core_close(H5FD_t *_file)
/* Flush any changed buffers */
if(H5FD_core_flush(_file, (hid_t)-1, TRUE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file")
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush core vfd backing store")
/* Destroy the dirty region list */
if(file->dirty_list)
if(H5FD_core_destroy_dirty_list(file) != SUCCEED)
HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free core vfd dirty region list")
/* Release resources */
if(file->fd >= 0)
@ -1025,6 +1329,14 @@ H5FD_core_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, had
file->eof = new_eof;
} /* end if */
/* Add the buffer region to the dirty list if using that optimization */
if(file->dirty_list) {
haddr_t start = addr;
haddr_t end = addr + (haddr_t)size - 1;
if(H5FD_core_add_dirty_region(file, start, end) != SUCCEED)
HGOTO_ERROR(H5E_VFL, H5E_CANTINSERT, FAIL, "unable to add core VFD dirty region during write call - addresses: start=%llu end=%llu", start, end)
}
/* Write from BUF to memory */
HDmemcpy(file->mem + addr, buf, size);
@ -1059,45 +1371,41 @@ H5FD_core_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned UNUSED closing)
/* Write to backing store */
if (file->dirty && file->fd >= 0 && file->backing_store) {
haddr_t size = file->eof;
unsigned char *ptr = file->mem;
if(0 != HDlseek(file->fd, (off_t)0, SEEK_SET))
HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store")
while (size > 0) {
/* Use the dirty list, if available */
if(file->dirty_list) {
H5FD_core_region_t *item = NULL;
size_t size;
h5_posix_io_t bytes_in = 0; /* # of bytes to write */
h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */
#ifdef DER
fprintf(stderr, "FLUSHING. DIRTY LIST:\n");
#endif
while(NULL != (item = (H5FD_core_region_t *)H5SL_remove_first(file->dirty_list))) {
size = (size_t)((item->end - item->start) + 1);
#ifdef DER
fprintf(stderr, "(%llu, %llu : %lu)\n", item->start, item->end, size);
#endif
if(H5FD_core_write_to_bstore(file, item->start, size) != SUCCEED)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store")
item = H5FL_FREE(H5FD_core_region_t, item);
}
/* Trying to write more bytes than the return type can handle is
* undefined behavior in POSIX.
*/
if(size > H5_POSIX_MAX_IO_BYTES)
bytes_in = H5_POSIX_MAX_IO_BYTES;
else
bytes_in = (h5_posix_io_t)size;
#ifdef DER
fprintf(stderr, "EOF: %llu\n", file->eof);
fprintf(stderr, "EOA: %llu\n", file->eoa);
if(file->eoa > file->eof)
fprintf(stderr, "*** EOA BADNESS ***\n");
fprintf(stderr, "\n");
#endif
}
/* Otherwise, write the entire file out at once */
else {
if(H5FD_core_write_to_bstore(file, (haddr_t)0, (size_t)file->eof) != SUCCEED)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write to backing store")
do {
bytes_wrote = HDwrite(file->fd, ptr, bytes_in);
} while(-1 == bytes_wrote && EINTR == errno);
if(-1 == bytes_wrote) { /* error */
int myerrno = errno;
time_t mytime = HDtime(NULL);
HDoff_t myoffset = HDlseek(file->fd, (HDoff_t)0, SEEK_CUR);
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to backing store failed: time = %s, filename = '%s', file descriptor = %d, errno = %d, error message = '%s', ptr = %p, total write size = %llu, bytes this sub-write = %llu, bytes actually written = %llu, offset = %llu", HDctime(&mytime), file->name, file->fd, myerrno, HDstrerror(myerrno), ptr, (unsigned long long)size, (unsigned long long)bytes_in, (unsigned long long)bytes_wrote, (unsigned long long)myoffset);
} /* end if */
HDassert(bytes_wrote > 0);
HDassert((size_t)bytes_wrote <= size);
size -= (size_t)bytes_wrote;
ptr = (unsigned char *)ptr + bytes_wrote;
} /* end while */
file->dirty = FALSE;
}
@ -1221,6 +1529,10 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing)
if(-1 == HDftruncate(file->fd, (HDoff_t)new_eof))
HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly")
#endif /* H5_HAVE_WIN32_API */
#ifdef DER
fprintf(stderr, "OLD: Truncated to: %llu\n", file->eoa);
#endif
} /* end if */
/* Update the eof value */
@ -1231,3 +1543,4 @@ H5FD_core_truncate(H5FD_t *_file, hid_t UNUSED dxpl_id, hbool_t closing)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_core_truncate() */

View File

@ -462,6 +462,8 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t;
#define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */
#define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */
#define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */
#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */
/* ======================== File Mount properties ====================*/
#define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */

View File

@ -160,7 +160,16 @@
#define H5F_ACS_FILE_IMAGE_INFO_DEL H5P_file_image_info_del
#define H5F_ACS_FILE_IMAGE_INFO_COPY H5P_file_image_info_copy
#define H5F_ACS_FILE_IMAGE_INFO_CLOSE H5P_file_image_info_close
/* Definition of core VFD write tracking flag */
#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE sizeof(hbool_t)
#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF FALSE
#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_ENC H5P__encode_hbool_t
#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEC H5P__decode_hbool_t
/* Definition of core VFD write tracking page size */
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE sizeof(size_t)
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t
/******************/
/* Local Typedefs */
@ -246,8 +255,9 @@ static const H5FD_mem_t H5F_def_mem_type_g = H5F_ACS_MULTI_TYPE_DEF;
static const hbool_t H5F_def_latest_format_g = H5F_ACS_LATEST_FORMAT_DEF; /* Default setting for "use the latest version of the format" flag */
static const hbool_t H5F_def_want_posix_fd_g = H5F_ACS_WANT_POSIX_FD_DEF; /* Default setting for retrieving 'handle' from core VFD */
static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF; /* Default external file cache size */
static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */
static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */
static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */
static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */
/*-------------------------------------------------------------------------
@ -397,6 +407,18 @@ H5P_facc_reg_prop(H5P_genclass_t *pclass)
H5F_ACS_FILE_IMAGE_INFO_DEL, H5F_ACS_FILE_IMAGE_INFO_COPY, NULL, H5F_ACS_FILE_IMAGE_INFO_CLOSE) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
/* Register the core VFD backing store write tracking flag */
if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE, &H5F_def_core_write_tracking_flag_g ,
NULL, NULL, NULL, H5F_ACS_CORE_WRITE_TRACKING_FLAG_ENC, H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEC,
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
/* Register the size of the core VFD backing store page size */
if(H5P_register_real(pclass, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_SIZE, &H5F_def_core_write_tracking_page_size_g ,
NULL, NULL, NULL, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC,
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P_facc_reg_prop() */
@ -2980,3 +3002,76 @@ H5P__facc_multi_type_dec(const void **_pp, void *_value)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5P__facc_multi_type_dec() */
/*-------------------------------------------------------------------------
* Function: H5Pset_core_write_tracking
*
* Purpose: Enables/disables core VFD write tracking and page
* aggregation size.
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_core_write_tracking(hid_t plist_id, hbool_t is_enabled, size_t page_size)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* return value */
FUNC_ENTER_API(FAIL)
H5TRACE3("e", "ibz", plist_id, is_enabled, page_size);
/* Get the plist structure */
if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
/* Set values */
if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, &is_enabled) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking flag")
if(H5P_set(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, &page_size) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set core VFD write tracking page size")
done:
FUNC_LEAVE_API(ret_value)
}
/*-------------------------------------------------------------------------
* Function: H5Pget_core_write_tracking
*
* Purpose: Gets information about core VFD write tracking and page
* aggregation size.
*
* Return: Non-negative on success/Negative on failure
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_core_write_tracking(hid_t plist_id, hbool_t *is_enabled, size_t *page_size)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* return value */
FUNC_ENTER_API(FAIL)
H5TRACE3("e", "i*b*z", plist_id, is_enabled, page_size);
/* Get the plist structure */
if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
/* Get values */
if(is_enabled) {
if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME, is_enabled) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking flag")
} /* end if */
if(page_size) {
if(H5P_get(plist, H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME, page_size) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get core VFD write tracking page size")
} /* end if */
done:
FUNC_LEAVE_API(ret_value)
}

View File

@ -296,7 +296,6 @@ H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list
H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold);
H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold);
/* File access property list (FAPL) routines */
H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold,
hsize_t alignment);
@ -344,6 +343,9 @@ H5_DLL herr_t H5Pset_file_image_callbacks(hid_t fapl_id,
H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id,
H5FD_file_image_callbacks_t *callbacks_ptr);
H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size);
H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size);
/* Dataset creation property list (DCPL) routines */
H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout);
H5_DLL H5D_layout_t H5Pget_layout(hid_t plist_id);

View File

@ -103,7 +103,7 @@ LDADD=libh5test.la $(LIBHDF5)
ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \
ttsafe_acreate.c
VFD_LIST = sec2 stdio core split multi family
VFD_LIST = sec2 stdio core core_paged split multi family
if DIRECT_VFD_CONDITIONAL
VFD_LIST += direct
endif

View File

@ -1155,7 +1155,8 @@ LDADD = libh5test.la $(LIBHDF5)
ttsafe_SOURCES = ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \
ttsafe_acreate.c
VFD_LIST = sec2 stdio core split multi family $(am__append_4)
VFD_LIST = sec2 stdio core core_paged split multi family \
$(am__append_4)
# Sources for testhdf5 executable
testhdf5_SOURCES = testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \

View File

@ -381,6 +381,8 @@ main(void)
FAIL_STACK_ERROR
if((H5Pset_mdc_config(fapl, &my_cache_config)) < 0)
FAIL_STACK_ERROR
if((H5Pset_core_write_tracking(fapl, TRUE, 1024 * 1024)) < 0)
FAIL_STACK_ERROR
/* Test encoding & decoding property list */
if(test_encode_decode(fapl) < 0)

View File

@ -327,6 +327,8 @@ main(void)
assert(ret > 0);
if((ret = H5Pset_mdc_config(fapl1, &my_cache_config)) < 0)
assert(ret > 0);
if((ret = H5Pset_core_write_tracking(fapl1, TRUE, (size_t)(1024 * 1024))) < 0)
assert(ret > 0);
if((ret = encode_plist(fapl1, little_endian, "testfiles/plist_files/fapl_le", "testfiles/plist_files/fapl_be")) < 0)
assert(ret > 0);

View File

@ -511,64 +511,71 @@ h5_fileaccess(void)
/* First use the environment variable, then the constant */
val = HDgetenv("HDF5_DRIVER");
#ifdef HDF5_DRIVER
if (!val) val = HDF5_DRIVER;
if (!val)
val = HDF5_DRIVER;
#endif
if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
if (!val || !*val) return fapl; /*use default*/
if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0)
return -1;
if (!val || !*val)
return fapl; /*use default*/
HDstrncpy(s, val, sizeof s);
s[sizeof(s)-1] = '\0';
if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
if (!HDstrcmp(name, "sec2")) {
/* Unix read() and write() system calls */
if (H5Pset_fapl_sec2(fapl)<0) return -1;
/* Unix read() and write() system calls */
if (H5Pset_fapl_sec2(fapl)<0) return -1;
} else if (!HDstrcmp(name, "stdio")) {
/* Standard C fread() and fwrite() system calls */
if (H5Pset_fapl_stdio(fapl)<0) return -1;
/* Standard C fread() and fwrite() system calls */
if (H5Pset_fapl_stdio(fapl)<0) return -1;
} else if (!HDstrcmp(name, "core")) {
/* In-core temporary file with 1MB increment */
if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
} else if (!HDstrcmp(name, "split")) {
/* Split meta data and raw data each using default driver */
if (H5Pset_fapl_split(fapl,
/* In-memory driver settings (backing store on, 1 MB increment) */
if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
} else if (!HDstrcmp(name, "core_paged")) {
/* In-memory driver with write tracking and paging on */
if (H5Pset_fapl_core(fapl, (size_t)1, TRUE)<0) return -1;
if (H5Pset_core_write_tracking(fapl, TRUE, (size_t)4096)<0) return -1;
} else if (!HDstrcmp(name, "split")) {
/* Split meta data and raw data each using default driver */
if (H5Pset_fapl_split(fapl,
"-m.h5", H5P_DEFAULT,
"-r.h5", H5P_DEFAULT)<0)
return -1;
return -1;
} else if (!HDstrcmp(name, "multi")) {
/* Multi-file driver, general case of the split driver */
H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
hid_t memb_fapl[H5FD_MEM_NTYPES];
const char *memb_name[H5FD_MEM_NTYPES];
char sv[H5FD_MEM_NTYPES][1024];
haddr_t memb_addr[H5FD_MEM_NTYPES];
/* Multi-file driver, general case of the split driver */
H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
hid_t memb_fapl[H5FD_MEM_NTYPES];
const char *memb_name[H5FD_MEM_NTYPES];
char sv[H5FD_MEM_NTYPES][1024];
haddr_t memb_addr[H5FD_MEM_NTYPES];
H5FD_mem_t mt;
HDmemset(memb_map, 0, sizeof memb_map);
HDmemset(memb_fapl, 0, sizeof memb_fapl);
HDmemset(memb_name, 0, sizeof memb_name);
HDmemset(memb_addr, 0, sizeof memb_addr);
HDmemset(memb_map, 0, sizeof memb_map);
HDmemset(memb_fapl, 0, sizeof memb_fapl);
HDmemset(memb_name, 0, sizeof memb_name);
HDmemset(memb_addr, 0, sizeof memb_addr);
HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES);
for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) {
memb_fapl[mt] = H5P_DEFAULT;
sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
memb_name[mt] = sv[mt];
memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
} /* end for */
HDassert(HDstrlen(multi_letters)==H5FD_MEM_NTYPES);
for(mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, mt)) {
memb_fapl[mt] = H5P_DEFAULT;
sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
memb_name[mt] = sv[mt];
memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
} /* end for */
if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name,
memb_addr, FALSE)<0) {
return -1;
}
if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name,
memb_addr, FALSE)<0) {
return -1;
}
} else if (!HDstrcmp(name, "family")) {
hsize_t fam_size = 100*1024*1024; /*100 MB*/
/* Family of files, each 1MB and using the default driver */
if ((val=HDstrtok(NULL, " \t\n\r")))
fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
/* Family of files, each 1MB and using the default driver */
if ((val=HDstrtok(NULL, " \t\n\r")))
fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
return -1;
} else if (!HDstrcmp(name, "log")) {
unsigned log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC;
@ -578,20 +585,21 @@ h5_fileaccess(void)
log_flags = (unsigned)HDstrtol(val, NULL, 0);
if (H5Pset_fapl_log(fapl, NULL, log_flags, (size_t)0) < 0)
return -1;
return -1;
} else if (!HDstrcmp(name, "direct")) {
#ifdef H5_HAVE_DIRECT
/* Linux direct read() and write() system calls. Set memory boundary, file block size,
* and copy buffer size to the default values. */
if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0) return -1;
/* Linux direct read() and write() system calls. Set memory boundary, file block size,
* and copy buffer size to the default values. */
if (H5Pset_fapl_direct(fapl, 1024, 4096, 8*4096)<0)
return -1;
#endif
} else if(!HDstrcmp(name, "latest")) {
/* use the latest format */
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
/* use the latest format */
if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
return -1;
} else {
/* Unknown driver */
return -1;
/* Unknown driver */
return -1;
}
return fapl;

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -28,6 +28,7 @@
#define FAMILY_SIZE2 (5*KB)
#define MULTI_SIZE 128
#define CORE_INCREMENT (4*KB)
#define CORE_PAGE_SIZE (1024 * 1024)
#define DSET1_NAME "dset1"
#define DSET1_DIM1 1024
#define DSET1_DIM2 32
@ -376,6 +377,8 @@ test_core(void)
char filename[1024];
void *fhandle=NULL;
hsize_t file_size;
hbool_t use_write_tracking;
size_t write_tracking_page_size;
int *points = NULL, *check = NULL, *p1, *p2;
hid_t dset1=-1, space1=-1;
hsize_t dims1[2];
@ -387,6 +390,8 @@ test_core(void)
fapl = h5_fileaccess();
if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
TEST_ERROR;
if(H5Pset_core_write_tracking(fapl, TRUE, CORE_PAGE_SIZE) < 0)
TEST_ERROR;
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
@ -400,6 +405,14 @@ test_core(void)
if(H5FD_CORE != H5Pget_driver(access_fapl))
TEST_ERROR;
/* Check that the backing store write tracking info was saved */
if(H5Pget_core_write_tracking(fapl, &use_write_tracking, &write_tracking_page_size) < 0)
TEST_ERROR;
if(TRUE != use_write_tracking)
TEST_ERROR;
if(CORE_PAGE_SIZE != write_tracking_page_size)
TEST_ERROR;
/* ...and close the property list */
if (H5Pclose(access_fapl) < 0)
TEST_ERROR;