mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-11-27 02:10:55 +08:00
Code Issues Packages Projects Releases Wiki Activity
8c56c5096e
hdf5/perform/pio_engine.c
Larry Knox f32690b53c [svn-r24997] Added compiler versions for mpich to settings file and configure summary. 
For make installcheck, compile and run installed examples using the installed scripts.  They were being compiled but not run.
Add Fortran2003 examples to the run-ex-fortran script when fortran2003 is enabled.
Set flag to -O3 in production mode for Intel compilers other than those with specified other settings.
Gentoo patches:
    Remove unnecessary setting of LD_LIBRARY_PATH from configure.
    Prevent potential array subscript out of bounds error in perform/pio_engine.c and sio_engine.c.

Tested with h5committest on jam, koala, ostrich and platypus, on emu and kite, and parallel tests on jam, koala and platypus with mpich built with gnu 4.8.2 and with intel compilers.
2014-04-08 23:55:49 -05:00

2679 lines
102 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the files COPYING and Copyright.html. COPYING can be found at the root *
* of the source code distribution tree; Copyright.html can be found at the *
* root level of an installed copy of the electronic HDF5 document set and *
* is linked from the top-level documents page. It can also be found at *
* http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
* access to either file, you may request a copy from help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Author: Albert Cheng of NCSA, Oct 24, 2001.
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <stdlib.h>
#include <fcntl.h>
#ifdef H5_HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <errno.h>
#include "hdf5.h"
#ifdef H5_HAVE_PARALLEL
#include <mpi.h>
#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */
# include <mpio.h>
#endif /* !MPI_FILE_NULL */
#include "pio_perf.h"
#include "pio_timer.h"
/* Macro definitions */
#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 6
# define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate(fd, name, type, space, dcpl)
# define H5DOPEN(fd, name) H5Dopen(fd, name)
#else
# define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate2(fd, name, type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)
# define H5DOPEN(fd, name) H5Dopen2(fd, name, H5P_DEFAULT)
#endif
/* sizes of various items. these sizes won't change during program execution */
/* The following three must have the same type */
#define ELMT_SIZE (sizeof(unsigned char)) /* we're doing bytes */
#define ELMT_MPI_TYPE MPI_BYTE
#define ELMT_H5_TYPE H5T_NATIVE_UCHAR
#define GOTOERROR(errcode) { ret_code = errcode; goto done; }
#define GOTODONE { goto done; }
#define ERRMSG(mesg) { \
fprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \
fprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", \
mesg, (int)__LINE__, __FILE__); \
}
#define MSG(mesg) { \
fprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \
fprintf(stderr, "(%s) at line %4d in %s\n", \
mesg, (int)__LINE__, __FILE__); \
}
/* verify: if val is false (0), print mesg. */
#define VRFY(val, mesg) do { \
if (!val) { \
ERRMSG(mesg); \
GOTOERROR(FAIL); \
} \
} while(0)
/* POSIX I/O macros */
#define POSIXCREATE(fn) HDopen(fn, O_CREAT|O_TRUNC|O_RDWR, 0600)
#define POSIXOPEN(fn, F) HDopen(fn, F, 0600)
#define POSIXCLOSE(F) HDclose(F)
#define POSIXSEEK(F,L) HDlseek(F, L, SEEK_SET)
#define POSIXWRITE(F,B,S) HDwrite(F,B,S)
#define POSIXREAD(F,B,S) HDread(F,B,S)
enum {
PIO_CREATE = 1,
PIO_WRITE = 2,
PIO_READ = 4
};
/* Global variables */
static int clean_file_g = -1; /*whether to cleanup temporary test */
/*files. -1 is not defined; */
/*0 is no cleanup; 1 is do cleanup */
/*
* In a parallel machine, the filesystem suitable for compiling is
* unlikely a parallel file system that is suitable for parallel I/O.
* There is no standard pathname for the parallel file system. /tmp
* is about the best guess.
*/
#ifndef HDF5_PARAPREFIX
# define HDF5_PARAPREFIX ""
#endif /* !HDF5_PARAPREFIX */
#ifndef MIN
# define MIN(a,b) ((a) < (b) ? (a) : (b))
#endif /* !MIN */
/* the different types of file descriptors we can expect */
typedef union _file_descr {
int posixfd; /* POSIX file handle*/
MPI_File mpifd; /* MPI file */
hid_t h5fd; /* HDF5 file */
} file_descr;
/* local functions */
static char *pio_create_filename(iotype iot, const char *base_name,
char *fullname, size_t size);
static herr_t do_write(results *res, file_descr *fd, parameters *parms,
long ndsets, off_t nelmts, size_t buf_size, void *buffer);
static herr_t do_read(results *res, file_descr *fd, parameters *parms,
long ndsets, off_t nelmts, size_t buf_size, void *buffer /*out*/);
static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/,
int flags);
static herr_t do_fclose(iotype iot, file_descr *fd);
static void do_cleanupfile(iotype iot, char *fname);
/*
* Function: do_pio
* Purpose: PIO Engine where Parallel IO are executed.
* Return: results
* Programmer: Albert Cheng, Bill Wendling 2001/12/12
* Modifications:
* Added 2D testing (Christian Chilan, 10. August 2005)
*/
results
do_pio(parameters param)
{
/* return codes */
herr_t ret_code = 0; /*return code */
results res;
file_descr fd;
iotype iot;
char fname[FILENAME_MAX];
long nf;
long ndsets;
off_t nbytes; /*number of bytes per dataset */
off_t snbytes; /*general dataset size */
/*for 1D, it is the actual dataset size */
/*for 2D, it is the size of a side of the dataset square */
char *buffer = NULL; /*data buffer pointer */
size_t buf_size; /*general buffer size in bytes */
/*for 1D, it is the actual buffer size */
/*for 2D, it is the length of the buffer rectangle */
size_t blk_size; /*data block size in bytes */
size_t bsize; /*actual buffer size */
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
/* Sanity check parameters */
/* IO type */
iot = param.io_type;
switch (iot) {
case MPIO:
fd.mpifd = MPI_FILE_NULL;
res.timers = pio_time_new(MPI_TIMER);
break;
case POSIXIO:
fd.posixfd = -1;
res.timers = pio_time_new(MPI_TIMER);
break;
case PHDF5:
fd.h5fd = -1;
res.timers = pio_time_new(MPI_TIMER);
break;
default:
/* unknown request */
fprintf(stderr, "Unknown IO type request (%d)\n", iot);
GOTOERROR(FAIL);
}
ndsets = param.num_dsets; /* number of datasets per file */
nbytes = param.num_bytes; /* number of bytes per dataset */
buf_size = param.buf_size;
blk_size = param.blk_size;
if (!param.dim2d){
snbytes = nbytes; /* General dataset size */
bsize = buf_size; /* Actual buffer size */
}
else {
snbytes = (off_t)sqrt(nbytes); /* General dataset size */
bsize = buf_size * blk_size; /* Actual buffer size */
}
if (param.num_files < 0 ) {
fprintf(stderr,
"number of files must be >= 0 (%ld)\n",
param.num_files);
GOTOERROR(FAIL);
}
if (ndsets < 0 ) {
fprintf(stderr,
"number of datasets per file must be >= 0 (%ld)\n",
ndsets);
GOTOERROR(FAIL);
}
if (param.num_procs <= 0 ) {
fprintf(stderr,
"maximum number of process to use must be > 0 (%d)\n",
param.num_procs);
GOTOERROR(FAIL);
}
/* Validate transfer buffer size & block size*/
if(blk_size<=0) {
HDfprintf(stderr,
"Transfer block size (%zu) must be > 0\n", blk_size);
GOTOERROR(FAIL);
}
if(buf_size<=0) {
HDfprintf(stderr,
"Transfer buffer size (%zu) must be > 0\n", buf_size);
GOTOERROR(FAIL);
}
if ((buf_size % blk_size) != 0){
HDfprintf(stderr,
"Transfer buffer size (%zu) must be a multiple of the "
"interleaved I/O block size (%zu)\n",
buf_size, blk_size);
GOTOERROR(FAIL);
}
if((snbytes%pio_mpi_nprocs_g)!=0) {
HDfprintf(stderr,
"Dataset size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the "
"number of processes (%d)\n",
(long long)snbytes, pio_mpi_nprocs_g);
GOTOERROR(FAIL);
}
if (!param.dim2d){
if(((snbytes/pio_mpi_nprocs_g)%buf_size)!=0) {
HDfprintf(stderr,
"Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the "
"trasfer buffer size (%zu)\n",
(long long)(snbytes/pio_mpi_nprocs_g), buf_size);
GOTOERROR(FAIL);
}
}
else {
if((snbytes%buf_size)!=0) {
HDfprintf(stderr,
"Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the "
"trasfer buffer size (%zu)\n",
(long long)snbytes, buf_size);
GOTOERROR(FAIL);
}
}
/* Allocate transfer buffer */
if ((buffer = malloc(bsize)) == NULL){
HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n",
bsize);
GOTOERROR(FAIL);
}
if (pio_debug_level >= 4) {
int myrank;
MPI_Comm_rank(pio_comm_g, &myrank);
/* output all of the times for all iterations */
if (myrank == 0)
fprintf(output, "Timer details:\n");
}
for (nf = 1; nf <= param.num_files; nf++) {
/*
* Write performance measurement
*/
/* Open file for write */
char base_name[256];
sprintf(base_name, "#pio_tmp_%lu", nf);
pio_create_filename(iot, base_name, fname, sizeof(fname));
if (pio_debug_level > 0)
HDfprintf(output, "rank %d: data filename=%s\n",
pio_mpi_rank_g, fname);
/* Need barrier to make sure everyone starts at the same time */
MPI_Barrier(pio_comm_g);
set_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART);
hrc = do_fopen(&param, fname, &fd, PIO_CREATE | PIO_WRITE);
VRFY((hrc == SUCCESS), "do_fopen failed");
set_time(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART);
hrc = do_write(&res, &fd, &param, ndsets, nbytes, buf_size, buffer);
hrc == SUCCESS;
set_time(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP);
VRFY((hrc == SUCCESS), "do_write failed");
/* Close file for write */
hrc = do_fclose(iot, &fd);
set_time(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP);
VRFY((hrc == SUCCESS), "do_fclose failed");
if (!param.h5_write_only) {
/*
* Read performance measurement
*/
/* Need barrier to make sure everyone is done writing and has
* closed the file. Also to make sure everyone starts reading
* at the same time.
*/
MPI_Barrier(pio_comm_g);
/* Open file for read */
set_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART);
hrc = do_fopen(&param, fname, &fd, PIO_READ);
VRFY((hrc == SUCCESS), "do_fopen failed");
set_time(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTART);
hrc = do_read(&res, &fd, &param, ndsets, nbytes, buf_size, buffer);
set_time(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP);
VRFY((hrc == SUCCESS), "do_read failed");
/* Close file for read */
hrc = do_fclose(iot, &fd);
set_time(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP);
VRFY((hrc == SUCCESS), "do_fclose failed");
}
/* Need barrier to make sure everyone is done with the file */
/* before it may be removed by do_cleanupfile */
MPI_Barrier(pio_comm_g);
do_cleanupfile(iot, fname);
}
done:
/* clean up */
/* release HDF5 objects */
/* close any opened files */
/* no remove(fname) because that should have happened normally. */
switch (iot) {
case POSIXIO:
if (fd.posixfd != -1)
hrc = do_fclose(iot, &fd);
break;
case MPIO:
if (fd.mpifd != MPI_FILE_NULL)
hrc = do_fclose(iot, &fd);
break;
case PHDF5:
if (fd.h5fd != -1)
hrc = do_fclose(iot, &fd);
break;
}
/* release generic resources */
if(buffer)
free(buffer);
res.ret_code = ret_code;
return res;
}
/*
* Function: pio_create_filename
* Purpose: Create a new filename to write to. Determine the correct
* suffix to append to the filename by the type of I/O we're
* doing. Also, place in the /tmp/{$USER,$LOGIN} directory if
* USER or LOGIN are specified in the environment.
* Return: Pointer to filename or NULL
* Programmer: Bill Wendling, 21. November 2001
* Modifications:
*/
static char *
pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size)
{
const char *prefix, *suffix = "";
char *ptr, last = '\0';
size_t i, j;
if (!base_name || !fullname || size < 1)
return NULL;
HDmemset(fullname, 0, size);
switch (iot) {
case POSIXIO:
suffix = ".posix";
break;
case MPIO:
suffix = ".mpio";
break;
case PHDF5:
suffix = ".h5";
break;
}
/* First use the environment variable and then try the constant */
prefix = HDgetenv("HDF5_PARAPREFIX");
#ifdef HDF5_PARAPREFIX
if (!prefix)
prefix = HDF5_PARAPREFIX;
#endif /* HDF5_PARAPREFIX */
/* Prepend the prefix value to the base name */
if (prefix && *prefix) {
/* If the prefix specifies the HDF5_PARAPREFIX directory, then
* default to using the "/tmp/$USER" or "/tmp/$LOGIN"
* directory instead. */
register char *user, *login, *subdir;
user = HDgetenv("USER");
login = HDgetenv("LOGIN");
subdir = (user ? user : login);
if (subdir) {
for (i = 0; i < size-1 && prefix[i]; i++)
fullname[i] = prefix[i];
fullname[i++] = '/';
for (j = 0; i < size && subdir[j]; i++, j++)
fullname[i] = subdir[j];
}
else {
/* We didn't append the prefix yet */
HDstrncpy(fullname, prefix, size);
fullname[size - 1] = '\0';
}
if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) {
/* Append the base_name with a slash first. Multiple slashes are
* handled below. */
h5_stat_t buf;
if (HDstat(fullname, &buf) < 0)
/* The directory doesn't exist just yet */
if (HDmkdir(fullname, (mode_t) 0755) < 0 && errno != EEXIST) {
/* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory.
* Default to PREFIX's original prefix value. */
HDstrcpy(fullname, prefix);
}
HDstrcat(fullname, "/");
HDstrcat(fullname, base_name);
}
else {
/* Buffer is too small */
return NULL;
}
}
else if (HDstrlen(base_name) >= size) {
/* Buffer is too small */
return NULL;
}
else {
HDstrcpy(fullname, base_name);
}
/* Append a suffix */
if (suffix) {
if (HDstrlen(fullname) + HDstrlen(suffix) >= size)
return NULL;
HDstrcat(fullname, suffix);
}
/* Remove any double slashes in the filename */
for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) {
if (*ptr != '/' || last != '/')
fullname[j++] = *ptr;
last = *ptr;
}
return fullname;
}
/*
* Function: do_write
* Purpose: Write the required amount of data to the file.
* Return: SUCCESS or FAIL
* Programmer: Albert Cheng, Bill Wendling, 2001/12/13
* Modifications:
* Added 2D testing (Christian Chilan, 10. August 2005)
*/
static herr_t
do_write(results *res, file_descr *fd, parameters *parms, long ndsets,
off_t nbytes, size_t buf_size, void *buffer)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
long ndset;
size_t blk_size; /* The block size to subdivide the xfer buffer into */
off_t nbytes_xfer; /* Total number of bytes transferred so far */
size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */
size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */
char dname[64];
off_t dset_offset=0; /*dataset offset in a file */
off_t bytes_begin[2]; /*first elmt this process transfer */
off_t bytes_count; /*number of elmts this process transfer */
off_t snbytes=0; /*size of a side of the dataset square */
unsigned char *buf_p; /* Current buffer pointer */
/* POSIX variables */
off_t file_offset; /* File offset of the next transfer */
off_t file_offset_advance; /* File offset advance after each I/O operation */
off_t posix_file_offset; /* Base file offset of the next transfer */
/* MPI variables */
MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/
MPI_Offset mpi_offset; /* Offset in MPI file */
MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */
MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */
MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */
MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */
MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */
MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */
MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */
MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */
MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */
MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */
MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */
MPI_Status mpi_status;
int mrc; /* MPI return code */
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
hsize_t h5dims[2]; /*dataset dim sizes */
hid_t h5dset_space_id = -1; /*dataset space ID */
hid_t h5mem_space_id = -1; /*memory dataspace ID */
hid_t h5ds_id = -1; /*dataset handle */
hsize_t h5block[2]; /*dataspace selection */
hsize_t h5stride[2];
hsize_t h5count[2];
hsize_t h5start[2];
hssize_t h5offset[2]; /* Selection offset within dataspace */
hid_t h5dcpl = -1; /* Dataset creation property list */
hid_t h5dxpl = -1; /* Dataset transfer property list */
/* Get the parameters from the parameter block */
blk_size=parms->blk_size;
/* There are two kinds of transfer patterns, contiguous and interleaved.
* Let 0,1,2,...,n be data accessed by process 0,1,2,...,n
* where n is rank of the last process.
* In contiguous pattern, data are accessed as
* 000...111...222...nnn...
* In interleaved pattern, data are accessed as
* 012...n012...n...
* These are all in the scope of one dataset.
*/
/* 1D dataspace */
if (!parms->dim2d){
/* Contiguous Pattern: */
if (!parms->interleaved) {
bytes_begin[0] = (off_t)(((double)nbytes*pio_mpi_rank_g)/pio_mpi_nprocs_g);
} /* end if */
/* Interleaved Pattern: */
else {
bytes_begin[0] = (off_t)(blk_size*pio_mpi_rank_g);
} /* end else */
/* Prepare buffer for verifying data */
if (parms->verify)
memset(buffer,pio_mpi_rank_g+1,buf_size);
}/* end if */
/* 2D dataspace */
else {
/* nbytes is always the number of bytes per dataset (1D or 2D). If the
dataspace is 2D, snbytes is the size of a side of the dataset square.
*/
snbytes = (off_t)sqrt(nbytes);
/* Contiguous Pattern: */
if (!parms->interleaved) {
bytes_begin[0] = (off_t)((double)snbytes*pio_mpi_rank_g / pio_mpi_nprocs_g);
bytes_begin[1] = 0;
} /* end if */
/* Interleaved Pattern: */
else {
bytes_begin[0] = 0;
if(!parms->h5_use_chunks || parms->io_type==PHDF5)
bytes_begin[1] = (off_t)(blk_size*pio_mpi_rank_g);
else
bytes_begin[1] = (off_t)(blk_size*blk_size*pio_mpi_rank_g);
} /* end else */
/* Prepare buffer for verifying data */
if (parms->verify)
memset(buffer,pio_mpi_rank_g+1,buf_size*blk_size);
} /* end else */
/* Calculate the total number of bytes (bytes_count) to be
* transferred by this process. It may be different for different
* transfer pattern due to rounding to integral values.
*/
/*
* Calculate the beginning bytes of this process and the next.
* bytes_count is the difference between these two beginnings.
* This way, it eliminates any rounding errors.
* (This is tricky, don't mess with the formula, rounding errors
* can easily get introduced) */
bytes_count = (off_t)(((double)nbytes*(pio_mpi_rank_g+1)) / pio_mpi_nprocs_g)
- (off_t)(((double)nbytes*pio_mpi_rank_g) / pio_mpi_nprocs_g);
/* debug */
if (pio_debug_level >= 4) {
HDprint_rank(output);
if (!parms->dim2d) {
HDfprintf(output, "Debug(do_write): "
"buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH "d\n",
buf_size, (long long)bytes_begin[0],
(long long)bytes_count);
} else {
HDfprintf(output, "Debug(do_write): "
"linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n",
buf_size*blk_size, (long long)bytes_begin[0],
(long long)bytes_begin[1], (long long)bytes_count);
}
}
/* I/O Access specific setup */
switch (parms->io_type) {
case POSIXIO:
/* No extra setup */
break;
case MPIO: /* MPI-I/O setup */
/* 1D dataspace */
if (!parms->dim2d){
/* Build block's derived type */
mrc = MPI_Type_contiguous((int)blk_size,
MPI_BYTE, &mpi_blk_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Build file's derived type */
mrc = MPI_Type_vector((int)(buf_size/blk_size), (int)1,
(int)pio_mpi_nprocs_g, mpi_blk_type, &mpi_file_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit file type */
mrc = MPI_Type_commit( &mpi_file_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Commit buffer type */
mrc = MPI_Type_commit( &mpi_blk_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
} /* end if */
/* 2D dataspace */
else {
/* Build partial buffer derived type for contiguous access */
mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE,
&mpi_partial_buffer_cont);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit partial buffer derived type */
mrc = MPI_Type_commit(&mpi_partial_buffer_cont);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build contiguous file's derived type */
mrc = MPI_Type_vector((int)blk_size, (int)1, (int)(snbytes/buf_size),
mpi_partial_buffer_cont, &mpi_cont_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit contiguous file type */
mrc = MPI_Type_commit(&mpi_cont_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build partial buffer derived type for interleaved access */
mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE,
&mpi_partial_buffer_inter);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit partial buffer derived type */
mrc = MPI_Type_commit(&mpi_partial_buffer_inter);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build interleaved file's derived type */
mrc = MPI_Type_vector((int)buf_size, (int)1, (int)(snbytes/blk_size),
mpi_partial_buffer_inter, &mpi_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit interleaved file type */
mrc = MPI_Type_commit(&mpi_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build full buffer derived type */
mrc = MPI_Type_contiguous((int)(blk_size*buf_size), MPI_BYTE,
&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit full buffer derived type */
mrc = MPI_Type_commit(&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build full chunk derived type */
mrc = MPI_Type_contiguous((int)(blk_size*blk_size), MPI_BYTE,
&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit full chunk derived type */
mrc = MPI_Type_commit(&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build chunk interleaved file's derived type */
mrc = MPI_Type_vector((int)(buf_size/blk_size), (int)1, (int)(snbytes/blk_size),
mpi_full_chunk, &mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit chunk interleaved file type */
mrc = MPI_Type_commit(&mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
} /* end else */
break;
case PHDF5: /* HDF5 setup */
/* 1D dataspace */
if (!parms->dim2d){
if(nbytes>0) {
/* define a contiguous dataset of nbytes native bytes */
h5dims[0] = nbytes;
h5dset_space_id = H5Screate_simple(1, h5dims, NULL);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
/* Set up the file dset space id to select the pattern to access */
if (!parms->interleaved){
/* Contiguous pattern */
h5start[0] = bytes_begin[0];
h5stride[0] = h5block[0] = blk_size;
h5count[0] = buf_size/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5start[0] = bytes_begin[0];
h5stride[0] = blk_size*pio_mpi_nprocs_g;
h5block[0] = blk_size;
h5count[0] = buf_size/blk_size;
} /* end else */
hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET,
h5start, h5stride, h5count, h5block);
VRFY((hrc >= 0), "H5Sselect_hyperslab");
} /* end if */
else {
h5dset_space_id = H5Screate(H5S_SCALAR);
VRFY((h5dset_space_id >= 0), "H5Screate");
} /* end else */
/* Create the memory dataspace that corresponds to the xfer buffer */
if(buf_size>0) {
h5dims[0] = buf_size;
h5mem_space_id = H5Screate_simple(1, h5dims, NULL);
VRFY((h5mem_space_id >= 0), "H5Screate_simple");
} /* end if */
else {
h5mem_space_id = H5Screate(H5S_SCALAR);
VRFY((h5mem_space_id >= 0), "H5Screate");
} /* end else */
} /* end if */
/* 2D dataspace */
else {
if(nbytes>0) {
/* define a contiguous dataset of nbytes native bytes */
h5dims[0] = snbytes;
h5dims[1] = snbytes;
h5dset_space_id = H5Screate_simple(2, h5dims, NULL);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
/* Set up the file dset space id to select the pattern to access */
if (!parms->interleaved){
/* Contiguous pattern */
h5start[0] = bytes_begin[0];
h5start[1] = bytes_begin[1];
h5stride[0] = 1;
h5stride[1] = h5block[0] = h5block[1] = blk_size;
h5count[0] = 1;
h5count[1] = buf_size/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5start[0] = bytes_begin[0];
h5start[1] = bytes_begin[1];
h5stride[0] = blk_size;
h5stride[1] = blk_size*pio_mpi_nprocs_g;
h5block[0] = h5block[1] = blk_size;
h5count[0] = buf_size/blk_size;
h5count[1] = 1;
} /* end else */
hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET,
h5start, h5stride, h5count, h5block);
VRFY((hrc >= 0), "H5Sselect_hyperslab");
} /* end if */
else {
h5dset_space_id = H5Screate(H5S_SCALAR);
VRFY((h5dset_space_id >= 0), "H5Screate");
} /* end else */
/* Create the memory dataspace that corresponds to the xfer buffer */
if(buf_size>0) {
if (!parms->interleaved){
h5dims[0] = blk_size;
h5dims[1] = buf_size;
}else{
h5dims[0] = buf_size;
h5dims[1] = blk_size;
}
h5mem_space_id = H5Screate_simple(2, h5dims, NULL);
VRFY((h5mem_space_id >= 0), "H5Screate_simple");
} /* end if */
else {
h5mem_space_id = H5Screate(H5S_SCALAR);
VRFY((h5mem_space_id >= 0), "H5Screate");
} /* end else */
} /* end else */
/* Create the dataset transfer property list */
h5dxpl = H5Pcreate(H5P_DATASET_XFER);
if (h5dxpl < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
/* Change to collective I/O, if asked */
if(parms->collective) {
hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE);
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
} /* end if */
} /* end if */
break;
} /* end switch */
for (ndset = 1; ndset <= ndsets; ++ndset) {
/* Calculate dataset offset within a file */
/* create dataset */
switch (parms->io_type) {
case POSIXIO:
case MPIO:
/* both posix and mpi io just need dataset offset in file*/
dset_offset = (ndset - 1) * nbytes;
break;
case PHDF5:
h5dcpl = H5Pcreate(H5P_DATASET_CREATE);
if (h5dcpl < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
/* 1D dataspace */
if (!parms->dim2d){
/* Make the dataset chunked if asked */
if(parms->h5_use_chunks) {
/* Set the chunk size to be the same as the buffer size */
h5dims[0] = blk_size;
hrc = H5Pset_chunk(h5dcpl, 1, h5dims);
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
} /* end if */
} /* end if */
}/* end if */
else{
/* 2D dataspace */
if(parms->h5_use_chunks) {
/* Set the chunk size to be the same as the block size */
h5dims[0] = blk_size;
h5dims[1] = blk_size;
hrc = H5Pset_chunk(h5dcpl, 2, h5dims);
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
} /* end if */
} /* end if */
}/* end else */
sprintf(dname, "Dataset_%ld", ndset);
h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE,
h5dset_space_id, h5dcpl);
if (h5ds_id < 0) {
fprintf(stderr, "HDF5 Dataset Create failed\n");
GOTOERROR(FAIL);
}
hrc = H5Pclose(h5dcpl);
/* verifying the close of the dcpl */
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Close failed\n");
GOTOERROR(FAIL);
}
break;
}
/* The task is to transfer bytes_count bytes, starting at
* bytes_begin position, using transfer buffer of buf_size bytes.
* If interleaved, select buf_size at a time, in round robin
* fashion, according to number of process. Otherwise, select
* all bytes_count in contiguous.
*/
nbytes_xfer = 0 ;
/* 1D dataspace */
if (!parms->dim2d){
/* Set base file offset for all I/O patterns and POSIX access */
posix_file_offset = dset_offset + bytes_begin[0];
/* Set base file offset for all I/O patterns and MPI access */
mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]);
} /* end if */
else {
/* Set base file offset for all I/O patterns and POSIX access */
posix_file_offset=dset_offset + bytes_begin[0]*snbytes+
bytes_begin[1];
/* Set base file offset for all I/O patterns and MPI access */
mpi_file_offset=(MPI_Offset)(dset_offset + bytes_begin[0]*snbytes+
bytes_begin[1]);
} /* end else */
/* Start "raw data" write timer */
set_time(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART);
while (nbytes_xfer < bytes_count){
/* Write */
/* Calculate offset of write within a dataset/file */
switch (parms->io_type) {
case POSIXIO:
/* 1D dataspace */
if (!parms->dim2d){
/* Contiguous pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset = posix_file_offset + (off_t)nbytes_xfer;
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are written */
rc = ((ssize_t)buf_size ==
POSIXWRITE(fd->posixfd, buffer, buf_size));
VRFY((rc != 0), "POSIXWRITE");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Set the base of user's buffer */
buf_p=(unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size;
/* Loop over the buffers to write */
while(nbytes_toxfer>0) {
/* Skip offset over blocks of other processes */
file_offset = posix_file_offset +
(off_t)(nbytes_xfer*pio_mpi_nprocs_g);
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are written */
rc = ((ssize_t)blk_size ==
POSIXWRITE(fd->posixfd, buf_p, blk_size));
VRFY((rc != 0), "POSIXWRITE");
/* Advance location in buffer */
buf_p+=blk_size;
/* Advance global offset in dataset */
nbytes_xfer+=blk_size;
/* Decrement number of bytes left this time */
nbytes_toxfer-=blk_size;
} /* end while */
} /* end else */
} /* end if */
/* 2D dataspace */
else {
/* Contiguous storage */
if (!parms->h5_use_chunks) {
/* Contiguous access pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)(((nbytes_xfer/blk_size)
/snbytes)*(blk_size*snbytes)+((nbytes_xfer/blk_size)%snbytes));
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = buf_size;
/* Global offset advance after each I/O operation */
file_offset_advance = (off_t)snbytes;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)((((nbytes_xfer/buf_size)
*pio_mpi_nprocs_g)/snbytes)*(buf_size*snbytes)
+((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)%snbytes);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size;
/* Global offset advance after each I/O operation */
file_offset_advance = (off_t)snbytes;
} /* end else */
} /* end if */
/* Chunked storage */
else {
/*Contiguous access pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)nbytes_xfer;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * buf_size;
/* Global offset advance after each I/O operation */
file_offset_advance = 0;
} /* end if */
/*Interleaved access pattern */
else {
/* Compute file offset */
/* Before simplification */
/* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size
*snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size))
*pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */
file_offset=posix_file_offset+(off_t)(((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes*blk_size))*(buf_size*snbytes)+((nbytes_xfer/(buf_size/blk_size))
*pio_mpi_nprocs_g)%(snbytes*blk_size));
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * blk_size;
/* Global offset advance after each I/O operation */
/* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */
file_offset_advance = (off_t)(snbytes*blk_size);
} /* end else */
} /* end else */
/* Common code for file access */
/* Set the base of user's buffer */
buf_p = (unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size*blk_size;
/* Loop over portions of the buffer to write */
while(nbytes_toxfer>0){
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are written */
rc = ((ssize_t)nbytes_xfer_advance ==
POSIXWRITE(fd->posixfd, buf_p, nbytes_xfer_advance));
VRFY((rc != 0), "POSIXWRITE");
/* Advance location in buffer */
buf_p+=nbytes_xfer_advance;
/* Advance global offset in dataset */
nbytes_xfer+=nbytes_xfer_advance;
/* Decrement number of bytes left this time */
nbytes_toxfer-=nbytes_xfer_advance;
/* Partially advance file offset */
file_offset+=file_offset_advance;
} /* end while */
} /* end else */
break;
case MPIO:
/* 1D dataspace */
if (!parms->dim2d){
/* Independent file access */
if(!parms->collective) {
/* Contiguous pattern */
if (!parms->interleaved){
/* Compute offset in file */
mpi_offset = mpi_file_offset +
nbytes_xfer;
/* Perform independent write */
mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buffer,
(int)(buf_size/blk_size), mpi_blk_type,
&mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Set the base of user's buffer */
buf_p=(unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size;
/* Loop over the buffers to write */
while(nbytes_toxfer>0) {
/* Skip offset over blocks of other processes */
mpi_offset = mpi_file_offset +
(nbytes_xfer*pio_mpi_nprocs_g);
/* Perform independent write */
mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p,
(int)1, mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance location in buffer */
buf_p+=blk_size;
/* Advance global offset in dataset */
nbytes_xfer+=blk_size;
/* Decrement number of bytes left this time */
nbytes_toxfer-=blk_size;
} /* end while */
} /* end else */
} /* end if */
/* Collective file access */
else {
/* Contiguous access pattern */
if (!parms->interleaved){
/* Compute offset in file */
mpi_offset = mpi_file_offset +
nbytes_xfer;
/* Perform independent write */
mrc = MPI_File_write_at_all(fd->mpifd, mpi_offset, buffer,
(int)(buf_size/blk_size), mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute offset in file */
mpi_offset = mpi_file_offset +
(nbytes_xfer*pio_mpi_nprocs_g);
/* Set the file view */
mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type,
mpi_file_type, (char*)"native", h5_io_info_g);
VRFY((mrc==MPI_SUCCESS), "MPIO_VIEW");
/* Perform write */
mrc = MPI_File_write_at_all(fd->mpifd, 0, buffer,
(int)(buf_size/blk_size), mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end else */
} /* end else */
} /* end if */
/* 2D dataspace */
else {
/* Contiguous storage */
if (!parms->h5_use_chunks) {
/* Contiguous access pattern */
if (!parms->interleaved) {
/* Compute offset in file */
mpi_offset=mpi_file_offset+((nbytes_xfer/blk_size)/snbytes)*
(blk_size*snbytes)+((nbytes_xfer/blk_size)%snbytes);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = buf_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = snbytes;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_cont_type;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute offset in file */
mpi_offset=mpi_file_offset+(((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)/snbytes)*
(buf_size*snbytes)+((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)%snbytes;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = snbytes;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_inter_type;
} /* end else */
} /* end if */
/* Chunked storage */
else {
/*Contiguous access pattern */
if (!parms->interleaved) {
/* Compute offset in file */
mpi_offset=mpi_file_offset+nbytes_xfer;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * buf_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = 0;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_full_buffer;
} /* end if */
/*Interleaved access pattern */
else {
/* Compute offset in file */
/* Before simplification */
/* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*
(buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+
((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes
/blk_size*(blk_size*blk_size)); */
mpi_offset=mpi_file_offset+((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes*blk_size))*(buf_size*snbytes)
+((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes*blk_size);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * blk_size;
/* Global offset advance after each I/O operation */
/* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */
mpi_offset_advance = (MPI_Offset)(snbytes*blk_size);
/* MPI type to be used for collective access */
mpi_collective_type = mpi_chunk_inter_type;
} /* end else */
} /* end else */
/* Common code for independent file access */
if (!parms->collective) {
/* Set the base of user's buffer */
buf_p = (unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size * blk_size;
/* Loop over portions of the buffer to write */
while(nbytes_toxfer>0){
/* Perform independent write */
mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p,
(int)nbytes_xfer_advance, MPI_BYTE, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance location in buffer */
buf_p+=nbytes_xfer_advance;
/* Advance global offset in dataset */
nbytes_xfer+=nbytes_xfer_advance;
/* Decrement number of bytes left this time */
nbytes_toxfer-=nbytes_xfer_advance;
/* Partially advance global offset in dataset */
mpi_offset+=mpi_offset_advance;
} /* end while */
} /* end if */
/* Common code for collective file access */
else {
/* Set the file view */
mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE,
mpi_collective_type, (char *)"native", h5_io_info_g);
VRFY((mrc==MPI_SUCCESS), "MPIO_VIEW");
/* Perform write */
MPI_File_write_at_all(fd->mpifd, 0, buffer,(int)(buf_size*blk_size),
MPI_BYTE, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_WRITE");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size*blk_size;
} /* end else */
} /* end else */
break;
case PHDF5:
/* 1D dataspace */
if (!parms->dim2d){
/* Set up the file dset space id to move the selection to process */
if (!parms->interleaved){
/* Contiguous pattern */
h5offset[0] = nbytes_xfer;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5offset[0] = (nbytes_xfer*pio_mpi_nprocs_g);
} /* end else */
hrc = H5Soffset_simple(h5dset_space_id, h5offset);
VRFY((hrc >= 0), "H5Soffset_simple");
/* Write the buffer out */
hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id,
h5dset_space_id, h5dxpl, buffer);
VRFY((hrc >= 0), "H5Dwrite");
/* Increment number of bytes transferred */
nbytes_xfer += buf_size;
} /* end if */
/* 2D dataspace */
else {
/* Set up the file dset space id to move the selection to process */
if (!parms->interleaved){
/* Contiguous pattern */
h5offset[0] = (nbytes_xfer/(snbytes*blk_size))*blk_size;
h5offset[1] = (nbytes_xfer%(snbytes*blk_size))/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5offset[0] = ((nbytes_xfer*pio_mpi_nprocs_g)/(snbytes*buf_size))*buf_size;
h5offset[1] = ((nbytes_xfer*pio_mpi_nprocs_g)%(snbytes*buf_size))/buf_size;
} /* end else */
hrc = H5Soffset_simple(h5dset_space_id, h5offset);
VRFY((hrc >= 0), "H5Soffset_simple");
/* Write the buffer out */
hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id,
h5dset_space_id, h5dxpl, buffer);
VRFY((hrc >= 0), "H5Dwrite");
/* Increment number of bytes transferred */
nbytes_xfer += buf_size*blk_size;
} /* end else */
break;
} /* switch (parms->io_type) */
} /* end while */
/* Stop "raw data" write timer */
set_time(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP);
/* Calculate write time */
/* Close dataset. Only HDF5 needs to do an explicit close. */
if (parms->io_type == PHDF5) {
hrc = H5Dclose(h5ds_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Close failed\n");
GOTOERROR(FAIL);
}
h5ds_id = -1;
} /* end if */
} /* end for */
done:
/* release MPI-I/O objects */
if (parms->io_type == MPIO) {
/* 1D dataspace */
if (!parms->dim2d){
/* Free file type */
mrc = MPI_Type_free( &mpi_file_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free buffer type */
mrc = MPI_Type_free( &mpi_blk_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
} /* end if */
/* 2D dataspace */
else {
/* Free partial buffer type for contiguous access */
mrc = MPI_Type_free( &mpi_partial_buffer_cont );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free contiguous file type */
mrc = MPI_Type_free( &mpi_cont_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free partial buffer type for interleaved access */
mrc = MPI_Type_free( &mpi_partial_buffer_inter );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free interleaved file type */
mrc = MPI_Type_free( &mpi_inter_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free full buffer type */
mrc = MPI_Type_free(&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free full chunk type */
mrc = MPI_Type_free(&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free chunk interleaved file type */
mrc = MPI_Type_free(&mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
} /* end else */
} /* end if */
/* release HDF5 objects */
if (h5dset_space_id != -1) {
hrc = H5Sclose(h5dset_space_id);
if (hrc < 0){
fprintf(stderr, "HDF5 Dataset Space Close failed\n");
ret_code = FAIL;
} else {
h5dset_space_id = -1;
}
}
if (h5mem_space_id != -1) {
hrc = H5Sclose(h5mem_space_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Memory Space Close failed\n");
ret_code = FAIL;
} else {
h5mem_space_id = -1;
}
}
if (h5dxpl != -1) {
hrc = H5Pclose(h5dxpl);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n");
ret_code = FAIL;
} else {
h5dxpl = -1;
}
}
return ret_code;
}
/*
* Function: do_read
* Purpose: read the required amount of data from the file.
* Return: SUCCESS or FAIL
* Programmer: Albert Cheng 2001/12/13
* Modifications:
* Added 2D testing (Christian Chilan, 10. August 2005)
*/
static herr_t
do_read(results *res, file_descr *fd, parameters *parms, long ndsets,
off_t nbytes, size_t buf_size, void *buffer /*out*/)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
long ndset;
size_t blk_size; /* The block size to subdivide the xfer buffer into */
size_t bsize; /* Size of the actual buffer */
off_t nbytes_xfer; /* Total number of bytes transferred so far */
size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */
size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */
char dname[64];
off_t dset_offset=0; /*dataset offset in a file */
off_t bytes_begin[2]; /*first elmt this process transfer */
off_t bytes_count; /*number of elmts this process transfer */
off_t snbytes=0; /*size of a side of the dataset square */
unsigned char *buf_p; /* Current buffer pointer */
/* POSIX variables */
off_t file_offset; /* File offset of the next transfer */
off_t file_offset_advance; /* File offset advance after each I/O operation */
off_t posix_file_offset; /* Base file offset of the next transfer */
/* MPI variables */
MPI_Offset mpi_file_offset;/* Base file offset of the next transfer*/
MPI_Offset mpi_offset; /* Offset in MPI file */
MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */
MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */
MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */
MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */
MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */
MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */
MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */
MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */
MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */
MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */
MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */
MPI_Status mpi_status;
int mrc; /* MPI return code */
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
hsize_t h5dims[2]; /*dataset dim sizes */
hid_t h5dset_space_id = -1; /*dataset space ID */
hid_t h5mem_space_id = -1; /*memory dataspace ID */
hid_t h5ds_id = -1; /*dataset handle */
hsize_t h5block[2]; /*dataspace selection */
hsize_t h5stride[2];
hsize_t h5count[2];
hsize_t h5start[2];
hssize_t h5offset[2]; /* Selection offset within dataspace */
hid_t h5dxpl = -1; /* Dataset transfer property list */
/* Get the parameters from the parameter block */
blk_size=parms->blk_size;
/* There are two kinds of transfer patterns, contiguous and interleaved.
* Let 0,1,2,...,n be data accessed by process 0,1,2,...,n
* where n is rank of the last process.
* In contiguous pattern, data are accessed as
* 000...111...222...nnn...
* In interleaved pattern, data are accessed as
* 012...n012...n...
* These are all in the scope of one dataset.
*/
/* 1D dataspace */
if (!parms->dim2d){
bsize = buf_size;
/* Contiguous Pattern: */
if (!parms->interleaved) {
bytes_begin[0] = (off_t)(((double)nbytes*pio_mpi_rank_g)/pio_mpi_nprocs_g);
} /* end if */
/* Interleaved Pattern: */
else {
bytes_begin[0] = (off_t)(blk_size*pio_mpi_rank_g);
} /* end else */
}/* end if */
/* 2D dataspace */
else {
/* nbytes is always the number of bytes per dataset (1D or 2D). If the
dataspace is 2D, snbytes is the size of a side of the 'dataset square'.
*/
snbytes = (off_t)sqrt(nbytes);
bsize = buf_size * blk_size;
/* Contiguous Pattern: */
if (!parms->interleaved) {
bytes_begin[0] = (off_t)((double)snbytes*pio_mpi_rank_g / pio_mpi_nprocs_g);
bytes_begin[1] = 0;
} /* end if */
/* Interleaved Pattern: */
else {
bytes_begin[0] = 0;
if (!parms->h5_use_chunks || parms->io_type==PHDF5)
bytes_begin[1] = (off_t)(blk_size*pio_mpi_rank_g);
else
bytes_begin[1] = (off_t)(blk_size*blk_size*pio_mpi_rank_g);
} /* end else */
} /* end else */
/* Calculate the total number of bytes (bytes_count) to be
* transferred by this process. It may be different for different
* transfer pattern due to rounding to integral values.
*/
/*
* Calculate the beginning bytes of this process and the next.
* bytes_count is the difference between these two beginnings.
* This way, it eliminates any rounding errors.
* (This is tricky, don't mess with the formula, rounding errors
* can easily get introduced) */
bytes_count = (off_t)(((double)nbytes*(pio_mpi_rank_g+1)) / pio_mpi_nprocs_g)
- (off_t)(((double)nbytes*pio_mpi_rank_g) / pio_mpi_nprocs_g);
/* debug */
if (pio_debug_level >= 4) {
HDprint_rank(output);
if (!parms->dim2d) {
HDfprintf(output, "Debug(do_write): "
"buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH "d\n",
buf_size, (long long)bytes_begin[0],
(long long)bytes_count);
} else {
HDfprintf(output, "Debug(do_write): "
"linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n",
buf_size*blk_size, (long long)bytes_begin[0],
(long long)bytes_begin[1], (long long)bytes_count);
}
}
/* I/O Access specific setup */
switch (parms->io_type) {
case POSIXIO:
/* No extra setup */
break;
case MPIO: /* MPI-I/O setup */
/* 1D dataspace */
if (!parms->dim2d){
/* Build block's derived type */
mrc = MPI_Type_contiguous((int)blk_size,
MPI_BYTE, &mpi_blk_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Build file's derived type */
mrc = MPI_Type_vector((int)(buf_size/blk_size), (int)1,
(int)pio_mpi_nprocs_g, mpi_blk_type, &mpi_file_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit file type */
mrc = MPI_Type_commit( &mpi_file_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Commit buffer type */
mrc = MPI_Type_commit( &mpi_blk_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
} /* end if */
/* 2D dataspace */
else {
/* Build partial buffer derived type for contiguous access */
mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE,
&mpi_partial_buffer_cont);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit partial buffer derived type */
mrc = MPI_Type_commit(&mpi_partial_buffer_cont);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build contiguous file's derived type */
mrc = MPI_Type_vector((int)blk_size, (int)1, (int)(snbytes/buf_size),
mpi_partial_buffer_cont, &mpi_cont_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit contiguous file type */
mrc = MPI_Type_commit(&mpi_cont_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build partial buffer derived type for interleaved access */
mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE,
&mpi_partial_buffer_inter);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit partial buffer derived type */
mrc = MPI_Type_commit(&mpi_partial_buffer_inter);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build interleaved file's derived type */
mrc = MPI_Type_vector((int)buf_size, (int)1, (int)(snbytes/blk_size),
mpi_partial_buffer_inter, &mpi_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit interleaved file type */
mrc = MPI_Type_commit(&mpi_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build full buffer derived type */
mrc = MPI_Type_contiguous((int)(blk_size*buf_size), MPI_BYTE,
&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit full buffer derived type */
mrc = MPI_Type_commit(&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build full chunk derived type */
mrc = MPI_Type_contiguous((int)(blk_size*blk_size), MPI_BYTE,
&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit full chunk derived type */
mrc = MPI_Type_commit(&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
/* Build chunk interleaved file's derived type */
mrc = MPI_Type_vector((int)(buf_size/blk_size), (int)1, (int)(snbytes/blk_size),
mpi_full_chunk, &mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_CREATE");
/* Commit chunk interleaved file type */
mrc = MPI_Type_commit(&mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_COMMIT");
} /* end else */
break;
case PHDF5: /* HDF5 setup */
/* 1D dataspace */
if (!parms->dim2d){
if(nbytes>0) {
/* define a contiguous dataset of nbytes native bytes */
h5dims[0] = nbytes;
h5dset_space_id = H5Screate_simple(1, h5dims, NULL);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
/* Set up the file dset space id to select the pattern to access */
if (!parms->interleaved){
/* Contiguous pattern */
h5start[0] = bytes_begin[0];
h5stride[0] = h5block[0] = blk_size;
h5count[0] = buf_size/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5start[0] = bytes_begin[0];
h5stride[0] = blk_size*pio_mpi_nprocs_g;
h5block[0] = blk_size;
h5count[0] = buf_size/blk_size;
} /* end else */
hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET,
h5start, h5stride, h5count, h5block);
VRFY((hrc >= 0), "H5Sselect_hyperslab");
} /* end if */
else {
h5dset_space_id = H5Screate(H5S_SCALAR);
VRFY((h5dset_space_id >= 0), "H5Screate");
} /* end else */
/* Create the memory dataspace that corresponds to the xfer buffer */
if(buf_size>0) {
h5dims[0] = buf_size;
h5mem_space_id = H5Screate_simple(1, h5dims, NULL);
VRFY((h5mem_space_id >= 0), "H5Screate_simple");
} /* end if */
else {
h5mem_space_id = H5Screate(H5S_SCALAR);
VRFY((h5mem_space_id >= 0), "H5Screate");
} /* end else */
} /* end if */
/* 2D dataspace */
else {
if(nbytes>0) {
/* define a contiguous dataset of nbytes native bytes */
h5dims[0] = snbytes;
h5dims[1] = snbytes;
h5dset_space_id = H5Screate_simple(2, h5dims, NULL);
VRFY((h5dset_space_id >= 0), "H5Screate_simple");
/* Set up the file dset space id to select the pattern to access */
if (!parms->interleaved){
/* Contiguous pattern */
h5start[0] = bytes_begin[0];
h5start[1] = bytes_begin[1];
h5stride[0] = 1;
h5stride[1] = h5block[0] = h5block[1] = blk_size;
h5count[0] = 1;
h5count[1] = buf_size/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5start[0] = bytes_begin[0];
h5start[1] = bytes_begin[1];
h5stride[0] = blk_size;
h5stride[1] = blk_size*pio_mpi_nprocs_g;
h5block[0] = h5block[1] = blk_size;
h5count[0] = buf_size/blk_size;
h5count[1] = 1;
} /* end else */
hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET,
h5start, h5stride, h5count, h5block);
VRFY((hrc >= 0), "H5Sselect_hyperslab");
} /* end if */
else {
h5dset_space_id = H5Screate(H5S_SCALAR);
VRFY((h5dset_space_id >= 0), "H5Screate");
} /* end else */
/* Create the memory dataspace that corresponds to the xfer buffer */
if(buf_size>0) {
if (!parms->interleaved){
h5dims[0] = blk_size;
h5dims[1] = buf_size;
}else{
h5dims[0] = buf_size;
h5dims[1] = blk_size;
}
h5mem_space_id = H5Screate_simple(2, h5dims, NULL);
VRFY((h5mem_space_id >= 0), "H5Screate_simple");
} /* end if */
else {
h5mem_space_id = H5Screate(H5S_SCALAR);
VRFY((h5mem_space_id >= 0), "H5Screate");
} /* end else */
} /* end else */
/* Create the dataset transfer property list */
h5dxpl = H5Pcreate(H5P_DATASET_XFER);
if (h5dxpl < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
/* Change to collective I/O, if asked */
if(parms->collective) {
hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE);
if (hrc < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
} /* end if */
} /* end if */
break;
} /* end switch */
for (ndset = 1; ndset <= ndsets; ++ndset) {
/* Calculate dataset offset within a file */
/* create dataset */
switch (parms->io_type) {
case POSIXIO:
case MPIO:
/* both posix and mpi io just need dataset offset in file*/
dset_offset = (ndset - 1) * nbytes;
break;
case PHDF5:
sprintf(dname, "Dataset_%ld", ndset);
h5ds_id = H5DOPEN(fd->h5fd, dname);
if (h5ds_id < 0) {
fprintf(stderr, "HDF5 Dataset open failed\n");
GOTOERROR(FAIL);
}
break;
}
/* The task is to transfer bytes_count bytes, starting at
* bytes_begin position, using transfer buffer of buf_size bytes.
* If interleaved, select buf_size at a time, in round robin
* fashion, according to number of process. Otherwise, select
* all bytes_count in contiguous.
*/
nbytes_xfer = 0 ;
/* 1D dataspace */
if (!parms->dim2d){
/* Set base file offset for all I/O patterns and POSIX access */
posix_file_offset = dset_offset + bytes_begin[0];
/* Set base file offset for all I/O patterns and MPI access */
mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]);
} /* end if */
else {
/* Set base file offset for all I/O patterns and POSIX access */
posix_file_offset=dset_offset + bytes_begin[0]*snbytes+
bytes_begin[1];
/* Set base file offset for all I/O patterns and MPI access */
mpi_file_offset=(MPI_Offset)(dset_offset + bytes_begin[0]*snbytes+
bytes_begin[1]);
} /* end else */
/* Start "raw data" read timer */
set_time(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART);
while (nbytes_xfer < bytes_count){
/* Read */
/* Calculate offset of read within a dataset/file */
switch (parms->io_type) {
case POSIXIO:
/* 1D dataspace */
if (!parms->dim2d){
/* Contiguous pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset = posix_file_offset + (off_t)nbytes_xfer;
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are read */
rc = ((ssize_t)buf_size ==
POSIXREAD(fd->posixfd, buffer, buf_size));
VRFY((rc != 0), "POSIXREAD");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Set the base of user's buffer */
buf_p=(unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size;
/* Loop over the buffers to read */
while(nbytes_toxfer>0) {
/* Skip offset over blocks of other processes */
file_offset = posix_file_offset +
(off_t)(nbytes_xfer*pio_mpi_nprocs_g);
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are read */
rc = ((ssize_t)blk_size ==
POSIXREAD(fd->posixfd, buf_p, blk_size));
VRFY((rc != 0), "POSIXREAD");
/* Advance location in buffer */
buf_p+=blk_size;
/* Advance global offset in dataset */
nbytes_xfer+=blk_size;
/* Decrement number of bytes left this time */
nbytes_toxfer-=blk_size;
} /* end while */
} /* end else */
} /* end if */
/* 2D dataspace */
else {
/* Contiguous storage */
if (!parms->h5_use_chunks) {
/* Contiguous access pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)(((nbytes_xfer/blk_size)
/snbytes)*(blk_size*snbytes)+((nbytes_xfer/blk_size)%snbytes));
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = buf_size;
/* Global offset advance after each I/O operation */
file_offset_advance = (off_t)snbytes;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)((((nbytes_xfer/buf_size)
*pio_mpi_nprocs_g)/snbytes)*(buf_size*snbytes)
+((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)%snbytes);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size;
/* Global offset advance after each I/O operation */
file_offset_advance = (off_t)snbytes;
} /* end else */
} /* end if */
/* Chunked storage */
else {
/*Contiguous access pattern */
if (!parms->interleaved) {
/* Compute file offset */
file_offset=posix_file_offset+(off_t)nbytes_xfer;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * buf_size;
/* Global offset advance after each I/O operation */
file_offset_advance = 0;
} /* end if */
/*Interleaved access pattern */
else {
/* Compute file offset */
/* Before simplification */
/* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size
*snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size))
*pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */
file_offset=posix_file_offset+(off_t)(((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes*blk_size))*(buf_size*snbytes)+((nbytes_xfer/(buf_size/blk_size))
*pio_mpi_nprocs_g)%(snbytes*blk_size));
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * blk_size;
/* Global offset advance after each I/O operation */
/* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */
file_offset_advance = (off_t)(snbytes*blk_size);
} /* end else */
} /* end else */
/* Common code for file access */
/* Set the base of user's buffer */
buf_p = (unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size*blk_size;
/* Loop over portions of the buffer to read */
while(nbytes_toxfer>0){
/* only care if seek returns error */
rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0;
VRFY((rc==0), "POSIXSEEK");
/* check if all bytes are read */
rc = ((ssize_t)nbytes_xfer_advance ==
POSIXREAD(fd->posixfd, buf_p, nbytes_xfer_advance));
VRFY((rc != 0), "POSIXREAD");
/* Advance location in buffer */
buf_p+=nbytes_xfer_advance;
/* Advance global offset in dataset */
nbytes_xfer+=nbytes_xfer_advance;
/* Decrement number of bytes left this time */
nbytes_toxfer-=nbytes_xfer_advance;
/* Partially advance file offset */
file_offset+=file_offset_advance;
} /* end while */
} /* end else */
break;
case MPIO:
/* 1D dataspace */
if (!parms->dim2d){
/* Independent file access */
if(!parms->collective) {
/* Contiguous pattern */
if (!parms->interleaved){
/* Compute offset in file */
mpi_offset = mpi_file_offset +
nbytes_xfer;
/* Perform independent read */
mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer,
(int)(buf_size/blk_size), mpi_blk_type,
&mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Set the base of user's buffer */
buf_p=(unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size;
/* Loop over the buffers to read */
while(nbytes_toxfer>0) {
/* Skip offset over blocks of other processes */
mpi_offset = mpi_file_offset +
(nbytes_xfer*pio_mpi_nprocs_g);
/* Perform independent read */
mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p,
(int)1, mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance location in buffer */
buf_p+=blk_size;
/* Advance global offset in dataset */
nbytes_xfer+=blk_size;
/* Decrement number of bytes left this time */
nbytes_toxfer-=blk_size;
} /* end while */
} /* end else */
} /* end if */
/* Collective file access */
else {
/* Contiguous access pattern */
if (!parms->interleaved){
/* Compute offset in file */
mpi_offset = mpi_file_offset +
nbytes_xfer;
/* Perform collective read */
mrc = MPI_File_read_at_all(fd->mpifd, mpi_offset, buffer,
(int)(buf_size/blk_size), mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute offset in file */
mpi_offset = mpi_file_offset +
(nbytes_xfer*pio_mpi_nprocs_g);
/* Set the file view */
mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type,
mpi_file_type, (char*)"native", h5_io_info_g);
VRFY((mrc==MPI_SUCCESS), "MPIO_VIEW");
/* Perform collective read */
mrc = MPI_File_read_at_all(fd->mpifd, 0, buffer,
(int)(buf_size/blk_size), mpi_blk_type, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size;
} /* end else */
} /* end else */
} /* end if */
/* 2D dataspace */
else {
/* Contiguous storage */
if (!parms->h5_use_chunks) {
/* Contiguous access pattern */
if (!parms->interleaved) {
/* Compute offset in file */
mpi_offset=mpi_file_offset+((nbytes_xfer/blk_size)/snbytes)*
(blk_size*snbytes)+((nbytes_xfer/blk_size)%snbytes);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = buf_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = snbytes;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_cont_type;
} /* end if */
/* Interleaved access pattern */
else {
/* Compute offset in file */
mpi_offset=mpi_file_offset+(((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)/snbytes)*
(buf_size*snbytes)+((nbytes_xfer/buf_size)*pio_mpi_nprocs_g)%snbytes;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = snbytes;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_inter_type;
} /* end else */
} /* end if */
/* Chunked storage */
else {
/*Contiguous access pattern */
if (!parms->interleaved) {
/* Compute offset in file */
mpi_offset=mpi_file_offset+nbytes_xfer;
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * buf_size;
/* Global offset advance after each I/O operation */
mpi_offset_advance = 0;
/* MPI type to be used for collective access */
mpi_collective_type = mpi_full_buffer;
} /* end if */
/*Interleaved access pattern */
else {
/* Compute offset in file */
/* Before simplification */
/* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*
(buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+
((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes
/blk_size*(blk_size*blk_size)); */
mpi_offset=mpi_file_offset+((nbytes_xfer/(buf_size/blk_size)
*pio_mpi_nprocs_g)/(snbytes*blk_size))*(buf_size*snbytes)
+((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes*blk_size);
/* Number of bytes to be transferred per I/O operation */
nbytes_xfer_advance = blk_size * blk_size;
/* Global offset advance after each I/O operation */
/* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */
mpi_offset_advance = (MPI_Offset)(snbytes*blk_size);
/* MPI type to be used for collective access */
mpi_collective_type = mpi_chunk_inter_type;
} /* end else */
} /* end else */
/* Common code for independent file access */
if (!parms->collective) {
/* Set the base of user's buffer */
buf_p = (unsigned char *)buffer;
/* Set the number of bytes to transfer this time */
nbytes_toxfer = buf_size * blk_size;
/* Loop over portions of the buffer to read */
while(nbytes_toxfer>0){
/* Perform independent read */
mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p,
(int)nbytes_xfer_advance, MPI_BYTE, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance location in buffer */
buf_p+=nbytes_xfer_advance;
/* Advance global offset in dataset */
nbytes_xfer+=nbytes_xfer_advance;
/* Decrement number of bytes left this time */
nbytes_toxfer-=nbytes_xfer_advance;
/* Partially advance global offset in dataset */
mpi_offset+=mpi_offset_advance;
} /* end while */
} /* end if */
/* Common code for collective file access */
else {
/* Set the file view */
mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE,
mpi_collective_type, (char *)"native", h5_io_info_g);
VRFY((mrc==MPI_SUCCESS), "MPIO_VIEW");
/* Perform read */
MPI_File_read_at_all(fd->mpifd, 0, buffer,(int)(buf_size*blk_size),
MPI_BYTE, &mpi_status);
VRFY((mrc==MPI_SUCCESS), "MPIO_READ");
/* Advance global offset in dataset */
nbytes_xfer+=buf_size*blk_size;
} /* end else */
} /* end else */
break;
case PHDF5:
/* 1D dataspace */
if (!parms->dim2d){
/* Set up the file dset space id to move the selection to process */
if (!parms->interleaved){
/* Contiguous pattern */
h5offset[0] = nbytes_xfer;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5offset[0] = (nbytes_xfer*pio_mpi_nprocs_g);
} /* end else */
hrc = H5Soffset_simple(h5dset_space_id, h5offset);
VRFY((hrc >= 0), "H5Soffset_simple");
/* Read the buffer in */
hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id,
h5dset_space_id, h5dxpl, buffer);
VRFY((hrc >= 0), "H5Dread");
/* Increment number of bytes transferred */
nbytes_xfer += buf_size;
} /* end if */
/* 2D dataspace */
else {
/* Set up the file dset space id to move the selection to process */
if (!parms->interleaved){
/* Contiguous pattern */
h5offset[0] = (nbytes_xfer/(snbytes*blk_size))*blk_size;
h5offset[1] = (nbytes_xfer%(snbytes*blk_size))/blk_size;
} /* end if */
else {
/* Interleaved access pattern */
/* Skip offset over blocks of other processes */
h5offset[0] = ((nbytes_xfer*pio_mpi_nprocs_g)/(snbytes*buf_size))*buf_size;
h5offset[1] = ((nbytes_xfer*pio_mpi_nprocs_g)%(snbytes*buf_size))/buf_size;
} /* end else */
hrc = H5Soffset_simple(h5dset_space_id, h5offset);
VRFY((hrc >= 0), "H5Soffset_simple");
/* Write the buffer out */
hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id,
h5dset_space_id, h5dxpl, buffer);
VRFY((hrc >= 0), "H5Dread");
/* Increment number of bytes transferred */
nbytes_xfer += buf_size*blk_size;
} /* end else */
break;
} /* switch (parms->io_type) */
/* Verify raw data, if asked */
if (parms->verify) {
/* Verify data read */
unsigned char *ucharptr = (unsigned char *)buffer;
size_t i;
int nerror=0;
for (i = 0; i < bsize; ++i){
if (*ucharptr++ != pio_mpi_rank_g+1) {
if (++nerror < 20){
/* report at most 20 errors */
HDprint_rank(output);
HDfprintf(output, "read data error, expected (%d), "
"got (%d)\n",
pio_mpi_rank_g+1,
(int)*(ucharptr-1));
} /* end if */
} /* end if */
} /* end for */
if (nerror >= 20) {
HDprint_rank(output);
HDfprintf(output, "...");
HDfprintf(output, "total read data errors=%d\n",
nerror);
} /* end if */
} /* if (parms->verify) */
} /* end while */
/* Stop "raw data" read timer */
set_time(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP);
/* Calculate read time */
/* Close dataset. Only HDF5 needs to do an explicit close. */
if (parms->io_type == PHDF5) {
hrc = H5Dclose(h5ds_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Close failed\n");
GOTOERROR(FAIL);
}
h5ds_id = -1;
} /* end if */
} /* end for */
done:
/* release MPI-I/O objects */
if (parms->io_type == MPIO) {
/* 1D dataspace */
if (!parms->dim2d){
/* Free file type */
mrc = MPI_Type_free( &mpi_file_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free buffer type */
mrc = MPI_Type_free( &mpi_blk_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
} /* end if */
/* 2D dataspace */
else {
/* Free partial buffer type for contiguous access */
mrc = MPI_Type_free( &mpi_partial_buffer_cont );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free contiguous file type */
mrc = MPI_Type_free( &mpi_cont_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free partial buffer type for interleaved access */
mrc = MPI_Type_free( &mpi_partial_buffer_inter );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free interleaved file type */
mrc = MPI_Type_free( &mpi_inter_type );
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free full buffer type */
mrc = MPI_Type_free(&mpi_full_buffer);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free full chunk type */
mrc = MPI_Type_free(&mpi_full_chunk);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
/* Free chunk interleaved file type */
mrc = MPI_Type_free(&mpi_chunk_inter_type);
VRFY((mrc==MPI_SUCCESS), "MPIO_TYPE_FREE");
} /* end else */
} /* end if */
/* release HDF5 objects */
if (h5dset_space_id != -1) {
hrc = H5Sclose(h5dset_space_id);
if (hrc < 0){
fprintf(stderr, "HDF5 Dataset Space Close failed\n");
ret_code = FAIL;
} else {
h5dset_space_id = -1;
}
}
if (h5mem_space_id != -1) {
hrc = H5Sclose(h5mem_space_id);
if (hrc < 0) {
fprintf(stderr, "HDF5 Memory Space Close failed\n");
ret_code = FAIL;
} else {
h5mem_space_id = -1;
}
}
if (h5dxpl != -1) {
hrc = H5Pclose(h5dxpl);
if (hrc < 0) {
fprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n");
ret_code = FAIL;
} else {
h5dxpl = -1;
}
}
return ret_code;
}
/*
* Function: do_fopen
* Purpose: Open the specified file.
* Return: SUCCESS or FAIL
* Programmer: Albert Cheng, Bill Wendling, 2001/12/13
* Modifications:
*/
static herr_t
do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags)
{
int ret_code = SUCCESS, mrc;
hid_t acc_tpl = -1; /* file access templates */
switch (param->io_type) {
case POSIXIO:
if (flags & (PIO_CREATE | PIO_WRITE))
fd->posixfd = POSIXCREATE(fname);
else
fd->posixfd = POSIXOPEN(fname, O_RDONLY);
if (fd->posixfd < 0 ) {
fprintf(stderr, "POSIX File Open failed(%s)\n", fname);
GOTOERROR(FAIL);
}
/* The perils of POSIX I/O in a parallel environment. The problem is:
*
* - Process n opens a file with truncation and then starts
* writing to the file.
* - Process m also opens the file with truncation, but after
* process n has already started to write to the file. Thus,
* all of the stuff process n wrote is now lost.
*/
MPI_Barrier(pio_comm_g);
break;
case MPIO:
if (flags & (PIO_CREATE | PIO_WRITE)) {
MPI_File_delete(fname, h5_io_info_g);
mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_CREATE | MPI_MODE_RDWR,
h5_io_info_g, &fd->mpifd);
if (mrc != MPI_SUCCESS) {
fprintf(stderr, "MPI File Open failed(%s)\n", fname);
GOTOERROR(FAIL);
}
/*since MPI_File_open with MPI_MODE_CREATE does not truncate */
/*filesize , set size to 0 explicitedly. */
mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0);
if (mrc != MPI_SUCCESS) {
fprintf(stderr, "MPI_File_set_size failed\n");
GOTOERROR(FAIL);
}
} else {
mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_RDONLY, h5_io_info_g, &fd->mpifd);
if (mrc != MPI_SUCCESS) {
fprintf(stderr, "MPI File Open failed(%s)\n", fname);
GOTOERROR(FAIL);
}
}
break;
case PHDF5:
if ((acc_tpl = H5Pcreate(H5P_FILE_ACCESS)) < 0) {
fprintf(stderr, "HDF5 Property List Create failed\n");
GOTOERROR(FAIL);
}
/* Set the file driver to the MPI-IO driver */
if (H5Pset_fapl_mpio(acc_tpl, pio_comm_g, h5_io_info_g) < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
}
/* Set the alignment of objects in HDF5 file */
if (H5Pset_alignment(acc_tpl, param->h5_thresh, param->h5_align) < 0) {
fprintf(stderr, "HDF5 Property List Set failed\n");
GOTOERROR(FAIL);
}
/* create the parallel file */
if (flags & (PIO_CREATE | PIO_WRITE))
fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl);
else
fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, acc_tpl);
if (fd->h5fd < 0) {
fprintf(stderr, "HDF5 File Create failed(%s)\n", fname);
GOTOERROR(FAIL);
}
/* verifying the close of the acc_tpl */
if (H5Pclose(acc_tpl) < 0) {
fprintf(stderr, "HDF5 Property List Close failed\n");
GOTOERROR(FAIL);
}
break;
}
done:
return ret_code;
}
/*
* Function: do_fclose
* Purpose: Close the specified file descriptor.
* Return: SUCCESS or FAIL
* Programmer: Albert Cheng, Bill Wendling, 2001/12/13
* Modifications:
*/
static herr_t
do_fclose(iotype iot, file_descr *fd /*out*/)
{
herr_t ret_code = SUCCESS, hrc;
int mrc = 0, rc = 0;
switch (iot) {
case POSIXIO:
rc = POSIXCLOSE(fd->posixfd);
if (rc != 0){
fprintf(stderr, "POSIX File Close failed\n");
GOTOERROR(FAIL);
}
fd->posixfd = -1;
break;
case MPIO:
mrc = MPI_File_close(&fd->mpifd);
if (mrc != MPI_SUCCESS){
fprintf(stderr, "MPI File close failed\n");
GOTOERROR(FAIL);
}
fd->mpifd = MPI_FILE_NULL;
break;
case PHDF5:
hrc = H5Fclose(fd->h5fd);
if (hrc < 0) {
fprintf(stderr, "HDF5 File Close failed\n");
GOTOERROR(FAIL);
}
fd->h5fd = -1;
break;
}
done:
return ret_code;
}
/*
* Function: do_fclose
* Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set.
* Only Proc 0 of the PIO communicator will do the cleanup.
* Other processes just return.
* Return: void
* Programmer: Albert Cheng 2001/12/12
* Modifications:
*/
static void
do_cleanupfile(iotype iot, char *fname)
{
if (pio_mpi_rank_g != 0)
return;
if (clean_file_g == -1)
clean_file_g = (getenv("HDF5_NOCLEANUP")==NULL) ? 1 : 0;
if (clean_file_g){
switch (iot){
case POSIXIO:
remove(fname);
break;
case MPIO:
case PHDF5:
MPI_File_delete(fname, h5_io_info_g);
break;
}
}
}
#ifdef TIME_MPI
/* instrument the MPI_File_wrirte_xxx and read_xxx calls to measure
* pure time spent in MPI_File code.
*/
int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status)
{
int err;
set_time(timer_g, HDF5_MPI_READ, TSTART);
err=PMPI_File_read_at(fh, offset, buf, count, datatype, status);
set_time(timer_g, HDF5_MPI_READ, TSTOP);
return err;
}
int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status)
{
int err;
set_time(timer_g, HDF5_MPI_READ, TSTART);
err=PMPI_File_read_at_all(fh, offset, buf, count, datatype, status);
set_time(timer_g, HDF5_MPI_READ, TSTOP);
return err;
}
int MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status)
{
int err;
set_time(timer_g, HDF5_MPI_WRITE, TSTART);
err=PMPI_File_write_at(fh, offset, buf, count, datatype, status);
set_time(timer_g, HDF5_MPI_WRITE, TSTOP);
return err;
}
int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void *buf,
int count, MPI_Datatype datatype, MPI_Status *status)
{
int err;
set_time(timer_g, HDF5_MPI_WRITE, TSTART);
err=PMPI_File_write_at_all(fh, offset, buf, count, datatype, status);
set_time(timer_g, HDF5_MPI_WRITE, TSTOP);
return err;
}
#endif /* TIME_MPI */
#endif /* H5_HAVE_PARALLEL */
Reference in New Issue View Git Blame Copy Permalink