mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-30 15:32:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Begin converting dataset code to use shared file pointer instead of top file

Browse Source 
pointer.
This commit is contained in:
Quincey Koziol 2019-08-20 18:21:53 -05:00
parent 25f982abbd
commit 67d7d28c03
5 changed files with 126 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
src/H5Dcontig.c
View File

@ -58,7 +58,7 @@
/* Callback info for sieve buffer readvv operation */
typedef struct H5D_contig_readvv_sieve_ud_t {
H5F_t *file; /* File for dataset */
H5F_file_t *f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */
const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */
unsigned char *rbuf; /* Pointer to buffer to fill */
@ -66,14 +66,14 @@ typedef struct H5D_contig_readvv_sieve_ud_t {
/* Callback info for [plain] readvv operation */
typedef struct H5D_contig_readvv_ud_t {
H5F_t *file; /* File for dataset */
H5F_file_t *f_sh; /* Shared file for dataset */
haddr_t dset_addr; /* Address of dataset */
unsigned char *rbuf; /* Pointer to buffer to fill */
} H5D_contig_readvv_ud_t;
/* Callback info for sieve buffer writevv operation */
typedef struct H5D_contig_writevv_sieve_ud_t {
H5F_t *file; /* File for dataset */
H5F_file_t *f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig; /* Cached information about contiguous data */
const H5D_contig_storage_t *store_contig; /* Contiguous storage info for this I/O operation */
const unsigned char *wbuf; /* Pointer to buffer to write */
@ -81,7 +81,7 @@ typedef struct H5D_contig_writevv_sieve_ud_t {
/* Callback info for [plain] writevv operation */
typedef struct H5D_contig_writevv_ud_t {
H5F_t *file; /* File for dataset */
H5F_file_t *f_sh; /* Shared file for dataset */
haddr_t dset_addr; /* Address of dataset */
const unsigned char *wbuf; /* Pointer to buffer to write */
} H5D_contig_writevv_ud_t;
@ -718,7 +718,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
void *_udata)
{
H5D_contig_readvv_sieve_ud_t *udata = (H5D_contig_readvv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_t *file = udata->file; /* File for dataset */
H5F_file_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */
const H5D_contig_storage_t *store_contig = udata->store_contig; /* Contiguous storage info for this I/O operation */
unsigned char *buf; /* Pointer to buffer to fill */
@ -750,7 +750,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
if(NULL == dset_contig->sieve_buf) {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(len > dset_contig->sieve_buf_size) {
if(H5F_block_read(file, H5FD_MEM_DRAW, addr, len, buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
} /* end if */
else {
@ -762,7 +762,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
dset_contig->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW)))
if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size")
/* Set up the buffer parameters */
@ -773,7 +773,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
/* Read the new sieve buffer */
if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
@ -804,7 +804,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Flush the sieve buffer, if it's dirty */
if(dset_contig->sieve_dirty) {
/* Write to file */
if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
/* Reset sieve buffer dirty flag */
@ -813,7 +813,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
} /* end if */
/* Read directly into the user's buffer */
if(H5F_block_read(file, H5FD_MEM_DRAW, addr, len, buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
} /* end if */
/* Element size fits within the buffer size */
@ -821,7 +821,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Flush the sieve buffer if it's dirty */
if(dset_contig->sieve_dirty) {
/* Write to file */
if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
/* Reset sieve buffer dirty flag */
@ -832,7 +832,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
dset_contig->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW)))
if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size")
/* Only need this when resizing sieve buffer */
@ -847,7 +847,7 @@ H5D__contig_readvv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
H5_CHECKED_ASSIGN(dset_contig->sieve_size, size_t, min, hsize_t);
/* Read the new sieve buffer */
if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
/* Grab the data out of the buffer (must be first piece of data in buffer ) */
@ -885,7 +885,7 @@ H5D__contig_readvv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata
FUNC_ENTER_STATIC
/* Write data */
if(H5F_block_read(udata->file, H5FD_MEM_DRAW, (udata->dset_addr + dst_off),
if(H5F_shared_block_read(udata->f_sh, H5FD_MEM_DRAW, (udata->dset_addr + dst_off),
len, (udata->rbuf + src_off)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
@ -931,11 +931,11 @@ H5D__contig_readvv(const H5D_io_info_t *io_info,
HDassert(mem_off_arr);
/* Check if data sieving is enabled */
if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_DATA_SIEVE)) {
if(H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_DATA_SIEVE)) {
H5D_contig_readvv_sieve_ud_t udata; /* User data for H5VM_opvv() operator */
/* Set up user data for H5VM_opvv() */
udata.file = io_info->dset->oloc.file;
udata.f_sh = io_info->f_sh;
udata.dset_contig = &(io_info->dset->shared->cache.contig);
udata.store_contig = &(io_info->store->contig);
udata.rbuf = (unsigned char *)io_info->u.rbuf;
@ -950,7 +950,7 @@ H5D__contig_readvv(const H5D_io_info_t *io_info,
H5D_contig_readvv_ud_t udata; /* User data for H5VM_opvv() operator */
/* Set up user data for H5VM_opvv() */
udata.file = io_info->dset->oloc.file;
udata.f_sh = io_info->f_sh;
udata.dset_addr = io_info->store->contig.dset_addr;
udata.rbuf = (unsigned char *)io_info->u.rbuf;
@ -983,7 +983,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
void *_udata)
{
H5D_contig_writevv_sieve_ud_t *udata = (H5D_contig_writevv_sieve_ud_t *)_udata; /* User data for H5VM_opvv() operator */
H5F_t *file = udata->file; /* File for dataset */
H5F_file_t *f_sh = udata->f_sh; /* Shared file for dataset */
H5D_rdcdc_t *dset_contig = udata->dset_contig; /* Cached information about contiguous data */
const H5D_contig_storage_t *store_contig = udata->store_contig; /* Contiguous storage info for this I/O operation */
const unsigned char *buf; /* Pointer to buffer to fill */
@ -1015,7 +1015,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
if(NULL == dset_contig->sieve_buf) {
/* Check if we can actually hold the I/O request in the sieve buffer */
if(len > dset_contig->sieve_buf_size) {
if(H5F_block_write(file, H5FD_MEM_DRAW, addr, len, buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
} /* end if */
else {
@ -1031,7 +1031,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
dset_contig->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW)))
if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size")
/* Set up the buffer parameters */
@ -1044,7 +1044,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Check if there is any point in reading the data from the file */
if(dset_contig->sieve_size > len) {
/* Read the new sieve buffer */
if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
} /* end if */
@ -1084,7 +1084,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Flush the sieve buffer, if it's dirty */
if(dset_contig->sieve_dirty) {
/* Write to file */
if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
/* Reset sieve buffer dirty flag */
@ -1097,7 +1097,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
} /* end if */
/* Write directly from the user's buffer */
if(H5F_block_write(file, H5FD_MEM_DRAW, addr, len, buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, addr, len, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
} /* end if */
/* Element size fits within the buffer size */
@ -1132,7 +1132,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Flush the sieve buffer if it's dirty */
if(dset_contig->sieve_dirty) {
/* Write to file */
if(H5F_block_write(file, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_write(f_sh, H5FD_MEM_DRAW, sieve_start, sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
/* Reset sieve buffer dirty flag */
@ -1143,7 +1143,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
dset_contig->sieve_loc = addr;
/* Make certain we don't read off the end of the file */
if(HADDR_UNDEF == (rel_eoa = H5F_get_eoa(file, H5FD_MEM_DRAW)))
if(HADDR_UNDEF == (rel_eoa = H5F_shared_get_eoa(f_sh, H5FD_MEM_DRAW)))
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to determine file size")
/* Only need this when resizing sieve buffer */
@ -1160,7 +1160,7 @@ H5D__contig_writevv_sieve_cb(hsize_t dst_off, hsize_t src_off, size_t len,
/* Check if there is any point in reading the data from the file */
if(dset_contig->sieve_size > len) {
/* Read the new sieve buffer */
if(H5F_block_read(file, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
if(H5F_shared_block_read(f_sh, H5FD_MEM_DRAW, dset_contig->sieve_loc, dset_contig->sieve_size, dset_contig->sieve_buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "block read failed")
} /* end if */
@ -1200,7 +1200,7 @@ H5D__contig_writevv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udat
FUNC_ENTER_STATIC
/* Write data */
if(H5F_block_write(udata->file, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), len, (udata->wbuf + src_off)) < 0)
if(H5F_shared_block_write(udata->f_sh, H5FD_MEM_DRAW, (udata->dset_addr + dst_off), len, (udata->wbuf + src_off)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "block write failed")
done:
@ -1245,11 +1245,11 @@ H5D__contig_writevv(const H5D_io_info_t *io_info,
HDassert(mem_off_arr);
/* Check if data sieving is enabled */
if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_DATA_SIEVE)) {
if(H5F_SHARED_HAS_FEATURE(io_info->f_sh, H5FD_FEAT_DATA_SIEVE)) {
H5D_contig_writevv_sieve_ud_t udata; /* User data for H5VM_opvv() operator */
/* Set up user data for H5VM_opvv() */
udata.file = io_info->dset->oloc.file;
udata.f_sh = io_info->f_sh;
udata.dset_contig = &(io_info->dset->shared->cache.contig);
udata.store_contig = &(io_info->store->contig);
udata.wbuf = (const unsigned char *)io_info->u.wbuf;
@ -1264,7 +1264,7 @@ H5D__contig_writevv(const H5D_io_info_t *io_info,
H5D_contig_writevv_ud_t udata; /* User data for H5VM_opvv() operator */
/* Set up user data for H5VM_opvv() */
udata.file = io_info->dset->oloc.file;
udata.f_sh = io_info->f_sh;
udata.dset_addr = io_info->store->contig.dset_addr;
udata.wbuf = (const unsigned char *)io_info->u.wbuf;

1
src/H5Dio.c
View File

@ -857,6 +857,7 @@ H5D__ioinfo_init(H5D_t *dset, const H5D_type_info_t *type_info,
/* Set up "normal" I/O fields */
io_info->dset = dset;
io_info->f_sh = H5F_SHARED(dset->oloc.file);
io_info->store = store;
/* Set I/O operations to initial values */

4
src/H5Dpkg.h
View File

@ -47,11 +47,13 @@
/* [Simple] Macro to construct a H5D_io_info_t from it's components */
#define H5D_BUILD_IO_INFO_WRT(io_info, ds, str, buf) \
(io_info)->dset = ds; \
(io_info)->f_sh = H5F_SHARED((ds)->oloc.file); \
(io_info)->store = str; \
(io_info)->op_type = H5D_IO_OP_WRITE; \
(io_info)->u.wbuf = buf
#define H5D_BUILD_IO_INFO_RD(io_info, ds, str, buf) \
(io_info)->dset = ds; \
(io_info)->f_sh = H5F_SHARED((ds)->oloc.file); \
(io_info)->store = str; \
(io_info)->op_type = H5D_IO_OP_READ; \
(io_info)->u.rbuf = buf
@ -207,6 +209,8 @@ typedef enum H5D_io_op_type_t {
typedef struct H5D_io_info_t {
const H5D_t *dset; /* Pointer to dataset being operated on */
/* QAK: Delete the f_sh field when oloc has a shared file pointer? */
H5F_file_t *f_sh; /* Pointer to shared file struct that dataset is within */
#ifdef H5_HAVE_PARALLEL
MPI_Comm comm; /* MPI communicator for file */
hbool_t using_mpi_vfd; /* Whether the file is using an MPI-based VFD */

89
src/H5Fio.c
View File

@ -75,6 +75,50 @@
/*******************/
/*-------------------------------------------------------------------------
* Function: H5F_shared_block_read
*
* Purpose: Reads some data from a file/server/etc into a buffer.
* The data is contiguous. The address is relative to the base
* address for the file.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* matzke@llnl.gov
* Jul 10 1997
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_shared_block_read(H5F_file_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/)
{
H5FD_mem_t map_type; /* Mapped memory type */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(f_sh);
HDassert(buf);
HDassert(H5F_addr_defined(addr));
/* Check for attempting I/O on 'temporary' file address */
if(H5F_addr_le(f_sh->tmp_addr, (addr + size)))
HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space")
/* Treat global heap as raw data */
map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type;
/* Pass through page buffer layer */
if(H5PB_read(f_sh, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_shared_block_read() */
/*-------------------------------------------------------------------------
* Function: H5F_block_read
@ -120,6 +164,51 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_block_read() */
/*-------------------------------------------------------------------------
* Function: H5F_shared_block_write
*
* Purpose: Writes some data from memory to a file/server/etc. The
* data is contiguous. The address is relative to the base
* address.
*
* Return: Non-negative on success/Negative on failure
*
* Programmer: Robb Matzke
* matzke@llnl.gov
* Jul 10 1997
*
*-------------------------------------------------------------------------
*/
herr_t
H5F_shared_block_write(H5F_file_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf)
{
H5FD_mem_t map_type; /* Mapped memory type */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
/* Sanity checks */
HDassert(f_sh);
HDassert(H5F_SHARED_INTENT(f_sh) & H5F_ACC_RDWR);
HDassert(buf);
HDassert(H5F_addr_defined(addr));
/* Check for attempting I/O on 'temporary' file address */
if(H5F_addr_le(f_sh->tmp_addr, (addr + size)))
HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space")
/* Treat global heap as raw data */
map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type;
/* Pass through page buffer layer */
if(H5PB_write(f_sh, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_shared_block_write() */
/*-------------------------------------------------------------------------
* Function: H5F_block_write

2
src/H5Fprivate.h
View File

@ -804,7 +804,9 @@ H5_DLL herr_t H5F_traverse_mount(struct H5O_loc_t *oloc/*in,out*/);
H5_DLL herr_t H5F_flush_mounts(H5F_t *f);
/* Functions that operate on blocks of bytes wrt super block */
H5_DLL herr_t H5F_shared_block_read(H5F_file_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/);
H5_DLL herr_t H5F_block_read(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/);
H5_DLL herr_t H5F_shared_block_write(H5F_file_t *f_sh, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5F_block_write(H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf);
/* Functions that flush or evict */