mirror of
https://github.com/HDFGroup/hdf5.git
synced 2024-11-27 02:10:55 +08:00
[svn-r5680] Purpose:
New feature.
Description:
There is some discussion among the SAF team as to whether it is better
to use MPI derived types for raw data transfers (thus needing a
MPI_File_set_view() call), or whether it is better to use a sequence of
low-level MPI types (i.e. MPI_BYTE) for the raw data transfer.
Solution:
Added an in internal flag to determine whether derived types are preferred
(the default), or whether they should be avoided. An environment variable
("HDF5_MPI_PREFER_DERIVED_TYPES") can be set by users to control whether MPI
types should be used or not. Set the environment variable to "0" (i.e.:
'setenv HDF5_MPI_PREFER_DERIVED_TYPES 0' to avoid using MPI derived types.
Platforms tested:
IRIX64 6.5 (modi4) w/parallel
This commit is contained in:
Quincey Koziol
parent
8aa24fc44a
commit
b1514ea26d
@ -185,6 +185,13 @@ Documentation
|
||||
New Features
|
||||
============
|
||||
|
||||
* Added environment variable flag to control whether creating MPI derived
|
||||
typed is preferred or not. This can affect performance, depending on
|
||||
which way the MPI-I/O library is optimized for. The default is set to
|
||||
prefer MPI derived types for collective raw data transfers, setting the
|
||||
HDF5_MPI_PREFER_DERIVED_TYPES environment variable to "0" (i.e.:
|
||||
"setenv HDF5_MPI_PREFER_DERIVED_TYPES 0") changes the preference to avoid
|
||||
using then whenever possible. QAK - 2002/06/19
|
||||
* Changed MPI I/O routines to avoid creating MPI derived types (and thus
|
||||
needing to set the file view) for contiguous selections within datasets,
|
||||
which should result in some performance improvement for those types of
|
||||
|
||||
@ -2280,7 +2280,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "collective access for MPIO driver only");
|
||||
|
||||
/* Set the "parallel I/O possible" flag, for H5S_find() */
|
||||
if (H5_mpi_opt_types_g && IS_H5FD_MPIO(dataset->ent.file)) {
|
||||
if (H5S_mpi_opt_types_g && IS_H5FD_MPIO(dataset->ent.file)) {
|
||||
/* Only collective write should call this since it eventually
|
||||
* calls MPI_File_set_view which is a collective call.
|
||||
* See H5S_mpio_spaces_xfer() for details.
|
||||
@ -2715,7 +2715,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
|
||||
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "collective access for MPIO driver only");
|
||||
|
||||
/* Set the "parallel I/O possible" flag, for H5S_find() */
|
||||
if (H5_mpi_opt_types_g && IS_H5FD_MPIO(dataset->ent.file)) {
|
||||
if (H5S_mpi_opt_types_g && IS_H5FD_MPIO(dataset->ent.file)) {
|
||||
/* Only collective write should call this since it eventually
|
||||
* calls MPI_File_set_view which is a collective call.
|
||||
* See H5S_mpio_spaces_xfer() for details.
|
||||
|
||||
16
src/H5S.c
16
src/H5S.c
@ -41,8 +41,9 @@ static size_t H5S_aconv_g = 0; /*entries allocated*/
|
||||
static size_t H5S_nconv_g = 0; /*entries used*/
|
||||
|
||||
#ifdef H5_HAVE_PARALLEL
|
||||
/* Global var whose value can be set from environment variable also */
|
||||
hbool_t H5_mpi_opt_types_g = TRUE;
|
||||
/* Global vars whose value can be set from environment variable also */
|
||||
hbool_t H5S_mpi_opt_types_g = TRUE;
|
||||
hbool_t H5S_mpi_prefer_derived_types_g = TRUE;
|
||||
#endif /* H5_HAVE_PARALLEL */
|
||||
|
||||
/* Declare a free list to manage the H5S_simple_t struct */
|
||||
@ -94,9 +95,14 @@ H5S_init_interface(void)
|
||||
{
|
||||
/* Allow MPI buf-and-file-type optimizations? */
|
||||
const char *s = HDgetenv ("HDF5_MPI_OPT_TYPES");
|
||||
if (s && HDisdigit(*s)) {
|
||||
H5_mpi_opt_types_g = (int)HDstrtol (s, NULL, 0);
|
||||
}
|
||||
if (s && HDisdigit(*s))
|
||||
H5S_mpi_opt_types_g = (int)HDstrtol (s, NULL, 0);
|
||||
}
|
||||
{
|
||||
/* Prefer MPI derived types for collective data transfers? */
|
||||
const char *s = HDgetenv ("HDF5_MPI_PREFER_DERIVED_TYPES");
|
||||
if (s && HDisdigit(*s))
|
||||
H5S_mpi_prefer_derived_types_g = (int)HDstrtol (s, NULL, 0);
|
||||
}
|
||||
#endif /* H5_HAVE_PARALLEL */
|
||||
|
||||
|
||||
108
src/H5Smpio.c
108
src/H5Smpio.c
@ -40,7 +40,7 @@
|
||||
static int interface_initialize_g = 0;
|
||||
|
||||
static herr_t
|
||||
H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_all_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -48,7 +48,7 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
|
||||
hbool_t *use_view,
|
||||
hbool_t *is_derived_type );
|
||||
static herr_t
|
||||
H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_hyper_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -56,7 +56,7 @@ H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
|
||||
hbool_t *use_view,
|
||||
hbool_t *is_derived_type );
|
||||
static herr_t
|
||||
H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_hyper_contig_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -64,7 +64,7 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
hbool_t *use_view,
|
||||
hbool_t *is_derived_type );
|
||||
static herr_t
|
||||
H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -74,7 +74,7 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
static herr_t
|
||||
H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
const H5S_t *file_space, const H5S_t *mem_space,
|
||||
hid_t dxpl_id, void *buf/*out*/, const hbool_t do_write);
|
||||
hid_t dxpl_id, void *buf/*out*/, hbool_t do_write);
|
||||
|
||||
/*-------------------------------------------------------------------------
|
||||
* Function: H5S_mpio_all_type
|
||||
@ -97,10 +97,14 @@ H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
* Quincey Koziol, June 18, 2002
|
||||
* Added 'extra_offset' and 'use_view' parameters
|
||||
*
|
||||
* Quincey Koziol, June 19, 2002
|
||||
* Added 'prefer_derived_types' flag to choose whether MPI derived types
|
||||
* should be created or not.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
static herr_t
|
||||
H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_all_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -121,12 +125,25 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
|
||||
for (u=0; u<space->extent.u.simple.rank; ++u)
|
||||
total_bytes *= space->extent.u.simple.size[u];
|
||||
|
||||
/* fill in the return values */
|
||||
*new_type = MPI_BYTE;
|
||||
H5_ASSIGN_OVERFLOW(*count, total_bytes, hsize_t, size_t);
|
||||
*extra_offset = 0;
|
||||
*use_view = 0;
|
||||
*is_derived_type = 0;
|
||||
/* Check if we should prefer creating a derived type */
|
||||
if(prefer_derived_types) {
|
||||
/* fill in the return values */
|
||||
H5_CHECK_OVERFLOW(total_bytes, hsize_t, int);
|
||||
if (MPI_Type_contiguous( (int)total_bytes, MPI_BYTE, new_type ))
|
||||
HRETURN_ERROR(H5E_DATASPACE, H5E_MPI, FAIL,"couldn't create MPI contiguous type");
|
||||
*count = 1;
|
||||
*extra_offset = 0;
|
||||
*use_view = 1;
|
||||
*is_derived_type = 1;
|
||||
} /* end if */
|
||||
else {
|
||||
/* fill in the return values */
|
||||
*new_type = MPI_BYTE;
|
||||
H5_ASSIGN_OVERFLOW(*count, total_bytes, hsize_t, size_t);
|
||||
*extra_offset = 0;
|
||||
*use_view = 0;
|
||||
*is_derived_type = 0;
|
||||
} /* end else */
|
||||
|
||||
#ifdef H5Smpi_DEBUG
|
||||
HDfprintf(stdout, "Leave %s total_bytes=%Hu\n", FUNC, total_bytes );
|
||||
@ -162,10 +179,14 @@ H5S_mpio_all_type( const H5S_t *space, const size_t elmt_size,
|
||||
* Added 'extra_offset' and 'use_view' parameters. Also accomodate
|
||||
* selection offset in MPI type built.
|
||||
*
|
||||
* Quincey Koziol, June 19, 2002
|
||||
* Added 'prefer_derived_types' flag to choose whether MPI derived types
|
||||
* should be created or not. (Ignored for this routine)
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
static herr_t
|
||||
H5S_mpio_hyper_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_hyper_type( const H5S_t *space, size_t elmt_size, hbool_t UNUSED prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -491,10 +512,14 @@ done:
|
||||
*
|
||||
* Modifications:
|
||||
*
|
||||
* Quincey Koziol, June 19, 2002
|
||||
* Added 'prefer_derived_types' flag to choose whether MPI derived types
|
||||
* should be created or not.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
static herr_t
|
||||
H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_hyper_contig_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -508,7 +533,7 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
hsize_t acc; /* Accumulator */
|
||||
hsize_t slab[H5O_LAYOUT_NDIMS]; /* Hyperslab size */
|
||||
hssize_t offset[H5O_LAYOUT_NDIMS]; /* Offset in selection */
|
||||
int ndims; /* Number of dimensions of dataset */
|
||||
int ndims; /* Number of dimensions of dataset */
|
||||
int i; /* Local index */
|
||||
|
||||
FUNC_ENTER_NOINIT(H5S_mpio_hyper_contig_type);
|
||||
@ -540,12 +565,25 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
for(i=0,byte_offset=0; i<ndims; i++)
|
||||
byte_offset+=offset[i]*slab[i];
|
||||
|
||||
/* fill in the return values */
|
||||
*new_type = MPI_BYTE;
|
||||
H5_ASSIGN_OVERFLOW(*count, total_bytes, hsize_t, size_t);
|
||||
*extra_offset = byte_offset;
|
||||
*use_view = 0;
|
||||
*is_derived_type = 0;
|
||||
/* Check if we should prefer creating a derived type */
|
||||
if(prefer_derived_types) {
|
||||
/* fill in the return values */
|
||||
H5_CHECK_OVERFLOW(total_bytes, hsize_t, int);
|
||||
if (MPI_Type_contiguous( (int)total_bytes, MPI_BYTE, new_type ))
|
||||
HRETURN_ERROR(H5E_DATASPACE, H5E_MPI, FAIL,"couldn't create MPI contiguous type");
|
||||
*count = 1;
|
||||
*extra_offset = byte_offset;
|
||||
*use_view = 1;
|
||||
*is_derived_type = 1;
|
||||
} /* end if */
|
||||
else {
|
||||
/* fill in the return values */
|
||||
*new_type = MPI_BYTE;
|
||||
H5_ASSIGN_OVERFLOW(*count, total_bytes, hsize_t, size_t);
|
||||
*extra_offset = byte_offset;
|
||||
*use_view = 0;
|
||||
*is_derived_type = 0;
|
||||
} /* end else */
|
||||
|
||||
#ifdef H5Smpi_DEBUG
|
||||
HDfprintf(stdout, "Leave %s total_bytes=%Hu\n", FUNC, total_bytes );
|
||||
@ -576,10 +614,14 @@ H5S_mpio_hyper_contig_type( const H5S_t *space, const size_t elmt_size,
|
||||
* Quincey Koziol, June 18, 2002
|
||||
* Added 'extra_offset' and 'use_view' parameters
|
||||
*
|
||||
* Quincey Koziol, June 19, 2002
|
||||
* Added 'prefer_derived_types' flag to choose whether MPI derived types
|
||||
* should be created or not.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
static herr_t
|
||||
H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
H5S_mpio_space_type( const H5S_t *space, size_t elmt_size, hbool_t prefer_derived_types,
|
||||
/* out: */
|
||||
MPI_Datatype *new_type,
|
||||
size_t *count,
|
||||
@ -606,7 +648,7 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
switch(space->select.type) {
|
||||
case H5S_SEL_NONE:
|
||||
case H5S_SEL_ALL:
|
||||
err = H5S_mpio_all_type( space, elmt_size,
|
||||
err = H5S_mpio_all_type( space, elmt_size, prefer_derived_types,
|
||||
/* out: */ new_type, count, extra_offset, use_view, is_derived_type );
|
||||
if (err<0)
|
||||
HRETURN_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't convert \"all\" selection to MPI type");
|
||||
@ -619,11 +661,11 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
|
||||
case H5S_SEL_HYPERSLABS:
|
||||
if(H5S_select_contiguous(space)) {
|
||||
err = H5S_mpio_hyper_contig_type( space, elmt_size,
|
||||
err = H5S_mpio_hyper_contig_type( space, elmt_size, prefer_derived_types,
|
||||
/* out: */ new_type, count, extra_offset, use_view, is_derived_type );
|
||||
} /* end if */
|
||||
else {
|
||||
err = H5S_mpio_hyper_type( space, elmt_size,
|
||||
err = H5S_mpio_hyper_type( space, elmt_size, prefer_derived_types,
|
||||
/* out: */ new_type, count, extra_offset, use_view, is_derived_type );
|
||||
} /* end else */
|
||||
if (err<0)
|
||||
@ -691,13 +733,18 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size,
|
||||
* Removed 'dc_plist' parameter, since it was not used. Also, switch to
|
||||
* getting the 'use_view' and 'extra_offset' settings for each selection.
|
||||
*
|
||||
* Quincey Koziol, June 19, 2002
|
||||
* Use 'prefer_derived_types' flag (from HDF5_MPI_PREFER_DERIVED_TYPES
|
||||
* environment variable) to choose whether MPI derived types should be
|
||||
* preferred or not.
|
||||
*
|
||||
*-------------------------------------------------------------------------
|
||||
*/
|
||||
static herr_t
|
||||
H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
const H5S_t *file_space, const H5S_t *mem_space,
|
||||
hid_t dxpl_id, void *_buf /*out*/,
|
||||
const hbool_t do_write )
|
||||
hbool_t do_write )
|
||||
{
|
||||
haddr_t addr; /* Address of dataset (or selection) within file */
|
||||
size_t mpi_buf_count, mpi_file_count; /* Number of "objects" to transfer */
|
||||
@ -708,6 +755,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
hbool_t mbt_is_derived=0, /* Whether the buffer (memory) type is derived and needs to be free'd */
|
||||
mft_is_derived=0; /* Whether the file type is derived and needs to be free'd */
|
||||
hbool_t plist_is_setup=0; /* Whether the dxpl has been customized */
|
||||
hbool_t prefer_derived_types=0;/* Whether to prefer MPI derived types or not */
|
||||
uint8_t *buf=(uint8_t *)_buf; /* Alias for pointer arithmetic */
|
||||
int err; /* Error detection value */
|
||||
herr_t ret_value = SUCCEED; /* Return value */
|
||||
@ -725,8 +773,12 @@ H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
assert(H5I_GENPROP_LST==H5I_get_type(dxpl_id));
|
||||
assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER));
|
||||
|
||||
/* Get the preference for MPI derived types */
|
||||
/* (Set via the "HDF5_MPI_PREFER_DERIVED_TYPES" environment variable for now) */
|
||||
prefer_derived_types= H5S_mpi_prefer_derived_types_g;
|
||||
|
||||
/* create the MPI buffer type */
|
||||
err = H5S_mpio_space_type( mem_space, elmt_size,
|
||||
err = H5S_mpio_space_type( mem_space, elmt_size, prefer_derived_types,
|
||||
/* out: */
|
||||
&mpi_buf_type,
|
||||
&mpi_buf_count,
|
||||
@ -737,7 +789,7 @@ H5S_mpio_spaces_xfer(H5F_t *f, const H5O_layout_t *layout, size_t elmt_size,
|
||||
HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"couldn't create MPI buf type");
|
||||
|
||||
/* create the MPI file type */
|
||||
err = H5S_mpio_space_type( file_space, elmt_size,
|
||||
err = H5S_mpio_space_type( file_space, elmt_size, prefer_derived_types,
|
||||
/* out: */
|
||||
&mpi_file_type,
|
||||
&mpi_file_count,
|
||||
|
||||
@ -249,8 +249,10 @@ __DLL__ htri_t H5S_mpio_opt_possible(const H5S_t *mem_space,
|
||||
const H5S_t *file_space, const unsigned flags);
|
||||
|
||||
#ifndef _H5S_IN_H5S_C
|
||||
/* Global var whose value comes from environment variable */
|
||||
__DLLVAR__ hbool_t H5_mpi_opt_types_g;
|
||||
/* Global vars whose value comes from environment variable */
|
||||
/* (Defined in H5S.c) */
|
||||
__DLLVAR__ hbool_t H5S_mpi_opt_types_g;
|
||||
__DLLVAR__ hbool_t H5S_mpi_prefer_derived_types_g;
|
||||
#endif /* _H5S_IN_H5S_C */
|
||||
|
||||
#endif /* H5_HAVE_PARALLEL */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user