mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-03-07 16:37:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r2600] Purpose:

Browse Source 
Implemented new feature
Description:
    Added data sieve buffering code to raw I/O data path.  This is enabled for
    all the VFL drivers except the mpio & core drivers.  Also added two new
    API functions to control the sieve buffer size: H5Pset_sieve_buf_size() and
    H5Pget_sieve_buf_size().
Platforms tested:
    Solaris 2.6 (i.e. baldric)
This commit is contained in:
Quincey Koziol 2000-09-26 17:50:18 -05:00
parent 415e326afe
commit 183c8245af
15 changed files with 580 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/H5F.c
View File

@ -264,6 +264,7 @@ H5F_init_interface(void)
H5F_access_dflt.alignment = 1; /*no alignment*/
H5F_access_dflt.gc_ref = 0; /*don't garbage-collect references*/
H5F_access_dflt.meta_block_size = 2048; /* set metadata block allocations to 2KB */
H5F_access_dflt.sieve_buf_size = 64*1024; /* set sieve buffer allocation to 64KB */
H5F_access_dflt.driver_id = H5FD_SEC2; /*default driver*/
H5F_access_dflt.driver_info = NULL; /*driver file access properties*/
@ -522,6 +523,7 @@ H5Fget_access_plist(hid_t file_id)
_fapl.alignment = f->shared->alignment;
_fapl.gc_ref = f->shared->gc_ref;
_fapl.meta_block_size = f->shared->lf->def_meta_block_size;
_fapl.sieve_buf_size = f->shared->sieve_buf_size;
_fapl.driver_id = f->shared->lf->driver_id;
_fapl.driver_info = NULL; /*just for now */
@ -761,6 +763,7 @@ H5F_new(H5F_file_t *shared, hid_t fcpl_id, hid_t fapl_id)
f->shared->threshold = fapl->threshold;
f->shared->alignment = fapl->alignment;
f->shared->gc_ref = fapl->gc_ref;
f->shared->sieve_buf_size = fapl->sieve_buf_size;
#ifdef H5_HAVE_PARALLEL
/*
@ -854,6 +857,12 @@ H5F_dest(H5F_t *f)
}
f->shared->cwfs = H5MM_xfree (f->shared->cwfs);
/* Free the data sieve buffer, if it's been allocated */
if(f->shared->sieve_buf) {
assert(f->shared->sieve_dirty==0); /* The buffer had better be flushed... */
f->shared->sieve_buf = H5MM_xfree (f->shared->sieve_buf);
} /* end if */
/* Destroy file creation properties */
H5P_close(f->shared->fcpl);
@ -1670,6 +1679,18 @@ H5F_flush(H5F_t *f, H5F_scope_t scope, hbool_t invalidate,
}
}
/* flush the data sieve buffer, if we have a dirty one */
if(!alloc_only && f->shared->sieve_buf && f->shared->sieve_dirty) {
/* Write dirty data sieve buffer to file */
if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, H5P_DEFAULT, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
} /* end if */
/* flush the entire raw data cache */
if (!alloc_only && H5F_istore_flush (f, invalidate)<0) {
HRETURN_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,

36
src/H5FDdpss.c
View File

@ -114,6 +114,7 @@ typedef struct H5FD_dpss_t {
static H5FD_t *H5FD_dpss_open (const char *name, unsigned flags,
hid_t UNUSED fapl_id, haddr_t maxaddr);
static herr_t H5FD_dpss_close (H5FD_t *_file);
static herr_t H5FD_dpss_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_dpss_get_eoa (H5FD_t *_file);
static herr_t H5FD_dpss_set_eoa (H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_dpss_get_eof (H5FD_t *_file);
@ -139,7 +140,7 @@ static const H5FD_class_t H5FD_dpss_g = {
H5FD_dpss_open, /* open */
H5FD_dpss_close, /* close */
NULL, /* cmp */
NULL, /*query */
H5FD_dpss_query, /*query */
NULL, /* alloc */
NULL, /* free */
H5FD_dpss_get_eoa, /* get_eoa */
@ -392,6 +393,39 @@ H5FD_dpss_close (H5FD_t *_file)
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5FD_dpss_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Tuesday, September 26, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_dpss_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
{
herr_t ret_value=SUCCEED;
FUNC_ENTER(H5FD_dpss_query, FAIL);
/* Set the VFL feature flags that this driver supports */
if(flags) {
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5FD_dpss_get_eoa

1
src/H5FDfamily.c
View File

@ -637,6 +637,7 @@ H5FD_family_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);

37
src/H5FDgass.c
View File

@ -100,6 +100,7 @@ typedef struct H5FD_gass_t {
static H5FD_t *H5FD_gass_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
static herr_t H5FD_gass_close(H5FD_t *_file);
static herr_t H5FD_gass_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_gass_get_eoa(H5FD_t *_file);
static herr_t H5FD_gass_set_eoa(H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_gass_get_eof(H5FD_t *_file);
@ -129,8 +130,8 @@ static const H5FD_class_t H5FD_gass_g = {
NULL, /*dxpl_free */
H5FD_gass_open, /*open */
H5FD_gass_close, /*close */
NULL, /*query */
NULL, /*cmp */
H5FD_gass_query, /*query */
NULL, /*alloc */
NULL, /*free */
H5FD_gass_get_eoa, /*get_eoa */
@ -432,6 +433,40 @@ H5FD_gass_close (H5FD_t *_file)
}
/*-------------------------------------------------------------------------
* Function: H5FD_gass_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Tuesday, September 26, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_gass_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
{
herr_t ret_value=SUCCEED;
FUNC_ENTER(H5FD_gass_query, FAIL);
/* Set the VFL feature flags that this driver supports */
if(flags) {
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5FD_gass_get_eoa
*

1
src/H5FDlog.c
View File

@ -646,6 +646,7 @@ H5FD_log_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);

32
src/H5FDmulti.c
View File

@ -115,6 +115,7 @@ static H5FD_t *H5FD_multi_open(const char *name, unsigned flags,
hid_t fapl_id, haddr_t maxaddr);
static herr_t H5FD_multi_close(H5FD_t *_file);
static int H5FD_multi_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
static herr_t H5FD_multi_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_multi_get_eoa(H5FD_t *_file);
static herr_t H5FD_multi_set_eoa(H5FD_t *_file, haddr_t eoa);
static haddr_t H5FD_multi_get_eof(H5FD_t *_file);
@ -144,7 +145,7 @@ static const H5FD_class_t H5FD_multi_g = {
H5FD_multi_open, /*open */
H5FD_multi_close, /*close */
H5FD_multi_cmp, /*cmp */
NULL, /*query */
H5FD_multi_query, /*query */
H5FD_multi_alloc, /*alloc */
H5FD_multi_free, /*free */
H5FD_multi_get_eoa, /*get_eoa */
@ -1298,6 +1299,35 @@ H5FD_multi_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
return H5FDcmp(f1->memb[mt], f2->memb[mt]);
}
/*-------------------------------------------------------------------------
* Function: H5FD_multi_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Tuesday, September 26, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */)
{
/* Set the VFL feature flags that this driver supports */
if(flags) {
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
return(0);
}
/*-------------------------------------------------------------------------
* Function: H5FD_multi_get_eoa

10
src/H5FDpublic.h
View File

@ -75,7 +75,7 @@ typedef enum H5FD_mem_t {
/* Define VFL driver features that can be enabled on a per-driver basis */
/* These are returned queried with the 'query' function pointer in H5FD_class_t */
/* These are returned with the 'query' function pointer in H5FD_class_t */
/*
* Defining the H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that
* the library will attempt to allocate a larger block for metadata and
@ -89,6 +89,14 @@ typedef enum H5FD_mem_t {
* 'write' routine.
*/
#define H5FD_FEAT_ACCUMULATE_METADATA 0x00000002
/*
* Defining the H5FD_FEAT_DATA_SIEVE for a VFL driver means that
* the library will attempt to cache raw data as it is read from/written to
* a file in a "data seive" buffer. See Rajeev Thakur's papers:
* http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz
* http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz
*/
#define H5FD_FEAT_DATA_SIEVE 0x00000004
/* Forward declaration */

1
src/H5FDsec2.c
View File

@ -416,6 +416,7 @@ H5FD_sec2_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);

36
src/H5FDsrb.c
View File

@ -56,6 +56,7 @@ static hid_t H5FD_SRB_g = 0;
static H5FD_t *H5FD_srb_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
static herr_t H5FD_srb_close(H5FD_t *_file);
static herr_t H5FD_srb_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_srb_get_eoa(H5FD_t *_file);
static herr_t H5FD_srb_set_eoa(H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_srb_get_eof(H5FD_t *_file);
@ -99,7 +100,7 @@ static const H5FD_class_t H5FD_srb_g = {
H5FD_srb_open, /*open */
H5FD_srb_close, /*close */
NULL, /*cmp */
NULL, /*query */
H5FD_srb_query, /*query */
NULL, /*alloc */
NULL, /*free */
H5FD_srb_get_eoa, /*get_eoa */
@ -361,6 +362,39 @@ H5FD_srb_close(H5FD_t *_file)
FUNC_LEAVE(SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5FD_srb_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Tuesday, September 26, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_srb_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
{
herr_t ret_value=SUCCEED;
FUNC_ENTER(H5FD_srb_query, FAIL);
/* Set the VFL feature flags that this driver supports */
if(flags) {
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5FD_srb_get_eoa
*

1
src/H5FDstdio.c
View File

@ -437,6 +437,7 @@ H5FD_stdio_query(const H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
return(0);

36
src/H5FDstream.c
View File

@ -108,6 +108,7 @@ static H5FD_t *H5FD_stream_open (const char *name, unsigned flags,
hid_t fapl_id, haddr_t maxaddr);
static herr_t H5FD_stream_flush (H5FD_t *_stream);
static herr_t H5FD_stream_close (H5FD_t *_stream);
static herr_t H5FD_stream_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_stream_get_eoa (H5FD_t *_stream);
static herr_t H5FD_stream_set_eoa (H5FD_t *_stream, haddr_t addr);
static haddr_t H5FD_stream_get_eof (H5FD_t *_stream);
@ -134,8 +135,8 @@ static const H5FD_class_t H5FD_stream_g =
NULL, /* dxpl_free */
H5FD_stream_open, /* open */
H5FD_stream_close, /* close */
NULL, /* query */
NULL, /* cmp */
H5FD_stream_query, /*query */
NULL, /* alloc */
NULL, /* free */
H5FD_stream_get_eoa, /* get_eoa */
@ -751,6 +752,39 @@ static herr_t H5FD_stream_close (H5FD_t *_stream)
FUNC_LEAVE (0);
}
/*-------------------------------------------------------------------------
* Function: H5FD_stream_query
*
* Purpose: Set the flags that this VFL driver is capable of supporting.
* (listed in H5FDpublic.h)
*
* Return: Success: non-negative
*
* Failure: negative
*
* Programmer: Quincey Koziol
* Tuesday, September 26, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
static herr_t
H5FD_stream_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
{
herr_t ret_value=SUCCEED;
FUNC_ENTER(H5FD_stream_query, FAIL);
/* Set the VFL feature flags that this driver supports */
if(flags) {
*flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
}
/*-------------------------------------------------------------------------
* Function: H5FD_stream_get_eoa

298
src/H5Farray.c
View File

@ -15,8 +15,10 @@
#include <H5Dprivate.h>
#include <H5Eprivate.h>
#include <H5Fprivate.h>
#include <H5FDprivate.h> /*file driver */
#include <H5Iprivate.h>
#include <H5MFprivate.h>
#include <H5MMprivate.h> /*memory management */
#include <H5Oprivate.h>
#include <H5Pprivate.h>
#include <H5Vprivate.h>
@ -138,6 +140,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
size_t nelmts, z; /*number of elements */
intn ndims; /*stride dimensionality */
haddr_t addr; /*address in file */
haddr_t eof; /*end of file address */
intn i, j; /*counters */
hbool_t carray; /*carry for subtraction */
#ifdef H5_HAVE_PARALLEL
@ -286,19 +289,147 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
z<nelmts;
z++, xfer_parms->gather_reads--) {
#else
#ifdef QAK
printf("%s: nelmts=%d, addr=%lu, elmt_size=%lu\n",FUNC,(int)nelmts,(unsigned long)addr,(unsigned long)elmt_size);
printf("%s: sieve_buf=%p, sieve_loc=%lu, sieve_size=%lu, sieve_buf_size=%lu, sieve_dirty=%u\n",FUNC,f->shared->sieve_buf,(unsigned long)f->shared->sieve_loc,(unsigned long)f->shared->sieve_size,(unsigned long)f->shared->sieve_buf_size,(unsigned)f->shared->sieve_dirty);
printf("%s: feature_flags=%lx\n",FUNC,(unsigned long)f->shared->lf->feature_flags);
#endif /* QAK */
for (z=0; z<nelmts; z++) {
#endif
/* Read from file */
if (efl && efl->nused>0) {
if (H5O_efl_read(f, efl, addr, elmt_size, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"external data read failed");
}
} else if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
/* Read directly from file if the dataset is in an external file */
/* Note: We can't use data sieve buffers for datasets in external files
* because the 'addr' of all external files is set to 0 (above) and
* all datasets in external files would alias to the same set of
* file offsets, totally mixing up the data sieve buffer information. -QAK
*/
if (efl && efl->nused>0) {
if (H5O_efl_read(f, efl, addr, elmt_size, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"external data read failed");
}
} else {
/* Check if data sieving is enabled */
if(f->shared->lf->feature_flags&H5FD_FEAT_DATA_SIEVE) {
/* Try reading from the data sieve buffer */
if(f->shared->sieve_buf) {
/* If entire read is within the sieve buffer, read it from the buffer */
if((addr>=f->shared->sieve_loc && addr<(f->shared->sieve_loc+f->shared->sieve_size))
&& ((addr+elmt_size-1)>=f->shared->sieve_loc && (addr+elmt_size-1)<(f->shared->sieve_loc+f->shared->sieve_size))) {
/* Grab the data out of the buffer */
HDmemcpy(buf,f->shared->sieve_buf+(addr-f->shared->sieve_loc),elmt_size);
} /* end if */
/* Entire request is not within this data sieve buffer */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(elmt_size>f->shared->sieve_buf_size) {
/* Check for any overlap with the current sieve buffer */
if((f->shared->sieve_loc>=addr && f->shared->sieve_loc<(addr+elmt_size))
|| ((f->shared->sieve_loc+f->shared->sieve_size-1)>=addr && (f->shared->sieve_loc+f->shared->sieve_size-1)<(addr+elmt_size))) {
/* Flush the sieve buffer, if it's dirty */
if(f->shared->sieve_dirty) {
/* Write to file */
if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
} /* end if */
} /* end if */
/* Read directly into the user's buffer */
if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
} /* end if */
/* Element size fits within the buffer size */
else {
/* Flush the sieve buffer if it's dirty */
if(f->shared->sieve_dirty) {
/* Write to file */
if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
} /* end if */
/* Determine the new sieve buffer size & location */
f->shared->sieve_loc=addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
"unable to determine file size");
}
f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
/* Read the new sieve buffer */
if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
HDmemcpy(buf,f->shared->sieve_buf,elmt_size);
} /* end else */
} /* end else */
} /* end if */
/* No data sieve buffer yet, go allocate one */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(elmt_size>f->shared->sieve_buf_size) {
if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
} /* end if */
else {
/* Allocate room for the data sieve buffer */
if (NULL==(f->shared->sieve_buf=H5MM_malloc(f->shared->sieve_buf_size))) {
HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
"memory allocation failed");
}
/* Determine the new sieve buffer size & location */
f->shared->sieve_loc=addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
"unable to determine file size");
}
f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
/* Read the new sieve buffer */
if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
HDmemcpy(buf,f->shared->sieve_buf,elmt_size);
} /* end else */
} /* end else */
} /* end if */
else {
if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
} /* end else */
} /* end else */
/* Decrement indices and advance pointers */
for (j=ndims-1, carray=TRUE; j>=0 && carray; --j) {
@ -318,8 +449,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
*/
if (efl && efl->nused>0) {
HRETURN_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL,
"chunking and external files are mutually "
"exclusive");
"chunking and external files are mutually exclusive");
}
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
@ -395,6 +525,7 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
size_t nelmts, z; /*number of elements */
intn ndims; /*dimensionality */
haddr_t addr; /*address in file */
haddr_t eof; /*end of file address */
intn i, j; /*counters */
hbool_t carray; /*carry for subtraction */
#ifdef H5_HAVE_PARALLEL
@ -535,14 +666,141 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
/* Write to file */
if (efl && efl->nused>0) {
if (H5O_efl_write(f, efl, addr, elmt_size, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"external data write failed");
}
} else if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
if (H5O_efl_write(f, efl, addr, elmt_size, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"external data write failed");
}
} else {
/* Check if data sieving is enabled */
if(f->shared->lf->feature_flags&H5FD_FEAT_DATA_SIEVE) {
/* Try writing to the data sieve buffer */
if(f->shared->sieve_buf) {
/* If entire write is within the sieve buffer, write it to the buffer */
if((addr>=f->shared->sieve_loc && addr<(f->shared->sieve_loc+f->shared->sieve_size))
&& ((addr+elmt_size-1)>=f->shared->sieve_loc && (addr+elmt_size-1)<(f->shared->sieve_loc+f->shared->sieve_size))) {
/* Grab the data out of the buffer */
HDmemcpy(f->shared->sieve_buf+(addr-f->shared->sieve_loc),buf,elmt_size);
/* Set sieve buffer dirty flag */
f->shared->sieve_dirty=1;
} /* end if */
/* Entire request is not within this data sieve buffer */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(elmt_size>f->shared->sieve_buf_size) {
/* Check for any overlap with the current sieve buffer */
if((f->shared->sieve_loc>=addr && f->shared->sieve_loc<(addr+elmt_size))
|| ((f->shared->sieve_loc+f->shared->sieve_size-1)>=addr && (f->shared->sieve_loc+f->shared->sieve_size-1)<(addr+elmt_size))) {
/* Flush the sieve buffer, if it's dirty */
if(f->shared->sieve_dirty) {
/* Write to file */
if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
} /* end if */
/* Force the sieve buffer to be re-read the next time */
f->shared->sieve_loc=HADDR_UNDEF;
f->shared->sieve_size=0;
} /* end if */
/* Write directly from the user's buffer */
if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
} /* end if */
/* Element size fits within the buffer size */
else {
/* Flush the sieve buffer if it's dirty */
if(f->shared->sieve_dirty) {
/* Write to file */
if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
/* Reset sieve buffer dirty flag */
f->shared->sieve_dirty=0;
} /* end if */
/* Determine the new sieve buffer size & location */
f->shared->sieve_loc=addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
"unable to determine file size");
}
f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
/* Read the new sieve buffer */
if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
/* Grab the data out of the buffer (must be first piece of data in buffer) */
HDmemcpy(f->shared->sieve_buf,buf,elmt_size);
/* Set sieve buffer dirty flag */
f->shared->sieve_dirty=1;
} /* end else */
} /* end else */
} /* end if */
/* No data sieve buffer yet, go allocate one */
else {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(elmt_size>f->shared->sieve_buf_size) {
if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
} /* end if */
else {
/* Allocate room for the data sieve buffer */
if (NULL==(f->shared->sieve_buf=H5MM_malloc(f->shared->sieve_buf_size))) {
HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
"memory allocation failed");
}
/* Determine the new sieve buffer size & location */
f->shared->sieve_loc=addr;
/* Make certain we don't read off the end of the file */
if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
"unable to determine file size");
}
f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
/* Read the new sieve buffer */
if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
/* Grab the data out of the buffer (must be first piece of data in buffer) */
HDmemcpy(f->shared->sieve_buf,buf,elmt_size);
/* Set sieve buffer dirty flag */
f->shared->sieve_dirty=1;
} /* end else */
} /* end else */
} /* end if */
else {
if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
} /* end else */
} /* end else */
/* Decrement indices and advance pointers */
for (j=ndims-1, carray=TRUE; j>=0 && carray; --j) {

11
src/H5Fprivate.h
View File

@ -254,7 +254,8 @@ typedef struct H5F_access_t {
double rdcc_w0; /* Preempt read chunks first? [0.0..1.0]*/
hsize_t threshold; /* Threshold for alignment */
hsize_t alignment; /* Alignment */
size_t meta_block_size; /* Minimum metadata allocation block size (when aggregating metadata allocations */
size_t meta_block_size; /* Minimum metadata allocation block size (when aggregating metadata allocations) */
hsize_t sieve_buf_size; /* Maximum sieve buffer size (when data sieving is allowed by file driver) */
uintn gc_ref; /* Garbage-collect references? */
hid_t driver_id; /* File driver ID */
void *driver_info; /* File driver specific information */
@ -328,6 +329,14 @@ typedef struct H5F_file_t {
struct H5G_t *root_grp; /* Open root group */
intn ncwfs; /* Num entries on cwfs list */
struct H5HG_heap_t **cwfs; /* Global heap cache */
/* Data Sieve Buffering fields */
unsigned char *sieve_buf; /* Buffer to hold data sieve buffer */
haddr_t sieve_loc; /* File location (offset) of the data sieve buffer */
hsize_t sieve_size; /* Size of the data sieve buffer used (in bytes) */
hsize_t sieve_buf_size; /* Size of the data sieve buffer allocated (in bytes) */
unsigned sieve_dirty; /* Flag to indicate that the data sieve buffer is dirty */
H5F_rdcc_t rdcc; /* Raw data chunk cache */
} H5F_file_t;

84
src/H5P.c
View File

@ -2990,6 +2990,90 @@ H5Pget_meta_block_size(hid_t fapl_id, hsize_t *size/*out*/)
FUNC_LEAVE (SUCCEED);
}
/*-------------------------------------------------------------------------
* Function: H5Pset_sieve_buf_size
*
* Purpose: Sets the maximum size of the data seive buffer used for file
* drivers which are capable of using data sieving. The data sieve
* buffer is used when performing I/O on datasets in the file. Using a
* buffer which is large anough to hold several pieces of the dataset
* being read in for hyperslab selections boosts performance by quite a
* bit.
*
* The default value is set to 64KB, indicating that file I/O for raw data
* reads and writes will occur in at least 64KB blocks.
* Setting the value to 0 with this API function will turn off the
* data sieving, even if the VFL driver attempts to use that strategy.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* Thursday, September 21, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pset_sieve_buf_size(hid_t fapl_id, hsize_t size)
{
H5F_access_t *fapl = NULL;
FUNC_ENTER (H5Pset_sieve_buf_size, FAIL);
H5TRACE2("e","ih",fapl_id,size);
/* Check args */
if (H5P_FILE_ACCESS != H5P_get_class (fapl_id) ||
NULL == (fapl = H5I_object (fapl_id))) {
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
"not a file access property list");
}
/* Set values */
fapl->sieve_buf_size = size;
FUNC_LEAVE (SUCCEED);
} /* end H5Pset_sieve_buf_size() */
/*-------------------------------------------------------------------------
* Function: H5Pget_sieve_buf_size
*
* Purpose: Returns the current settings for the data sieve buffer size
* property from a file access property list.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
* Thursday, September 21, 2000
*
* Modifications:
*
*-------------------------------------------------------------------------
*/
herr_t
H5Pget_sieve_buf_size(hid_t fapl_id, hsize_t *size/*out*/)
{
H5F_access_t *fapl = NULL;
FUNC_ENTER (H5Pget_sieve_buf_size, FAIL);
H5TRACE2("e","ix",fapl_id,size);
/* Check args */
if (H5P_FILE_ACCESS != H5P_get_class (fapl_id) ||
NULL == (fapl = H5I_object (fapl_id))) {
HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
"not a file access property list");
}
/* Get values */
if (size)
*size = fapl->sieve_buf_size;
FUNC_LEAVE (SUCCEED);
} /* end H5Pget_sieve_buf_size() */
/*--------------------------------------------------------------------------
NAME

2
src/H5Ppublic.h
View File

@ -181,6 +181,8 @@ __DLL__ herr_t H5Pget_vlen_mem_manager(hid_t plist_id,
void **free_info);
__DLL__ herr_t H5Pset_meta_block_size(hid_t fapl_id, hsize_t size);
__DLL__ herr_t H5Pget_meta_block_size(hid_t fapl_id, hsize_t *size/*out*/);
__DLL__ herr_t H5Pset_sieve_buf_size(hid_t fapl_id, hsize_t size);
__DLL__ herr_t H5Pget_sieve_buf_size(hid_t fapl_id, hsize_t *size/*out*/);
#ifdef __cplusplus
}