mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-30 15:32:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

[svn-r5407] Purpose:

Browse Source 
Bug fix
Description:
    Was not able to handle data size (file size) larger than 32bits.
    Was using long, which is only 4 bytes big in SP, thus overflowing
    into negative when trying to address 2GB or larger.
Solution:
    Changed those variables involved in file size/offset calculation
    to type off_t.  (If a certain system/compiler has off_t defined
    as 4 bytes, it can't write to file size larger than 2GB anyway.)

    Note that the lseek of SP with -D_LARGE_FILE still fails for
    offset larger than 2GB (works for 2GB).  That has to be fixed
    soon.
Platforms tested:
   burrwhite (linux 2.4) and modi4 parallel.
This commit is contained in:
Albert Cheng 2002-05-13 14:55:33 -05:00
parent 4793f81ae1
commit d07e0dd9a3
3 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
perform/pio_engine.c
View File

@ -129,9 +129,9 @@ typedef union _file_descr {
static char *pio_create_filename(iotype iot, const char *base_name,
char *fullname, size_t size);
static herr_t do_write(file_descr *fd, iotype iot, long ndsets,
long nelmts, long buf_size, void *buffer);
long nelmts, size_t buf_size, void *buffer);
static herr_t do_read(file_descr *fd, iotype iot, long ndsets,
long nelmts, long buf_size, void *buffer /*out*/);
long nelmts, size_t buf_size, void *buffer /*out*/);
static herr_t do_fopen(iotype iot, char *fname, file_descr *fd /*out*/,
int flags);
static herr_t do_fclose(iotype iot, file_descr *fd);
@ -157,9 +157,10 @@ do_pio(parameters param)
char fname[FILENAME_MAX];
int maxprocs;
int nfiles, nf;
long ndsets, nelmts;
long ndsets;
long nelmts;
char *buffer = NULL; /*data buffer pointer */
long buf_size; /*data buffer size in bytes */
size_t buf_size; /*data buffer size in bytes */
/* HDF5 variables */
herr_t hrc; /*HDF5 return code */
@ -244,7 +245,7 @@ buf_size=MIN(1024*1024, buf_size);
#endif
/* allocate data buffer */
buffer = malloc((size_t)buf_size);
buffer = malloc(buf_size);
if (buffer == NULL){
fprintf(stderr, "malloc for data buffer size (%ld) failed\n",
@ -468,7 +469,7 @@ pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t si
*/
static herr_t
do_write(file_descr *fd, iotype iot, long ndsets,
long nelmts, long buf_size, void *buffer)
long nelmts, size_t buf_size, void *buffer)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
@ -480,7 +481,7 @@ do_write(file_descr *fd, iotype iot, long ndsets,
char dname[64];
off_t dset_offset; /*dataset offset in a file */
off_t file_offset; /*file offset of the next transfer */
long dset_size; /*one dataset size in bytes */
off_t dset_size; /*one dataset size in bytes */
long nelmts_in_buf;
long elmts_begin; /*first elmt this process transfer */
long elmts_count; /*number of elmts this process transfer */
@ -587,7 +588,9 @@ fprintf(stderr, "proc %d: elmts_begin=%ld, elmts_count=%ld\n",
/* Calculate offset of write within a dataset/file */
switch (iot) {
case RAWIO:
file_offset = dset_offset + (elmts_begin + nelmts_written)*ELMT_SIZE;
/* need to (off_t) the elmnts_begin expression because they */
/* may be of smaller sized integer types */
file_offset = dset_offset + (off_t)(elmts_begin + nelmts_written)*ELMT_SIZE;
#if AKCDEBUG
fprintf(stderr, "proc %d: writes %ld bytes at file-offset %ld\n",
@ -692,7 +695,7 @@ done:
*/
static herr_t
do_read(file_descr *fd, iotype iot, long ndsets,
long nelmts, long buf_size, void *buffer /*out*/)
long nelmts, size_t buf_size, void *buffer /*out*/)
{
int ret_code = SUCCESS;
int rc; /*routine return code */
@ -704,7 +707,7 @@ do_read(file_descr *fd, iotype iot, long ndsets,
char dname[64];
off_t dset_offset; /*dataset offset in a file */
off_t file_offset; /*file offset of the next transfer */
long dset_size; /*one dataset size in bytes */
off_t dset_size; /*one dataset size in bytes */
long nelmts_in_buf;
long elmts_begin; /*first elmt this process transfer */
long elmts_count; /*number of elmts this process transfer */
@ -797,7 +800,9 @@ fprintf(stderr, "proc %d: elmts_begin=%ld, elmts_count=%ld\n",
/* Calculate offset of read within a dataset/file */
switch (iot){
case RAWIO:
file_offset = dset_offset + (elmts_begin + nelmts_read)*ELMT_SIZE;
/* need to (off_t) the elmnts_begin expression because they */
/* may be of smaller sized integer types */
file_offset = dset_offset + (off_t)(elmts_begin + nelmts_read)*ELMT_SIZE;
#if AKCDEBUG
fprintf(stderr, "proc %d: read %ld bytes at file-offset %ld\n",

31
perform/pio_perf.c
View File

@ -203,14 +203,14 @@ static struct long_options l_opts[] = {
struct options {
long io_types; /* bitmask of which I/O types to test */
const char *output_file; /* file to print report to */
long file_size; /* size of file */
off_t file_size; /* size of file */
long num_dsets; /* number of datasets */
long num_files; /* number of files */
long num_iters; /* number of iterations */
long max_num_procs; /* maximum number of processes to use */
long min_num_procs; /* minimum number of processes to use */
long max_xfer_size; /* maximum transfer buffer size */
long min_xfer_size; /* minimum transfer buffer size */
size_t max_xfer_size; /* maximum transfer buffer size */
size_t min_xfer_size; /* minimum transfer buffer size */
};
typedef struct _minmax {
@ -221,13 +221,13 @@ typedef struct _minmax {
} minmax;
/* local functions */
static long parse_size_directive(const char *size);
static off_t parse_size_directive(const char *size);
static struct options *parse_command_line(int argc, char *argv[]);
static void run_test_loop(struct options *options);
static int run_test(iotype iot, parameters parms);
static void output_all_info(minmax *mm, int count, int indent_level);
static void get_minmax(minmax *mm, double val);
static minmax accumulate_minmax_stuff(minmax *mm, long raw_size, int count);
static minmax accumulate_minmax_stuff(minmax *mm, off_t raw_size, int count);
static int create_comm_world(int num_procs, int *doing_pio);
static int destroy_comm_world(void);
static void output_report(const char *fmt, ...);
@ -296,7 +296,6 @@ main(int argc, char **argv)
goto finish;
}
}
run_test_loop(opts);
finish:
@ -355,7 +354,7 @@ run_test_loop(struct options *opts)
/* if performance needs restart, fewer processes may be needed. */
for (num_procs = opts->max_num_procs;
num_procs >= opts->min_num_procs; num_procs >>= 1) {
register long buf_size;
register size_t buf_size;
parms.num_procs = num_procs;
@ -411,7 +410,7 @@ run_test(iotype iot, parameters parms)
results res;
register int i, ret_value = SUCCESS;
int comm_size;
long raw_size;
off_t raw_size;
minmax total_mm;
minmax *write_mpi_mm_table;
minmax *write_mm_table;
@ -484,7 +483,7 @@ run_test(iotype iot, parameters parms)
read_gross_mm_table[i].num = 0;
}
/* call Albert's testing here */
/* Do IO iteration times, collecting statics each time */
for (i = 0; i < parms.num_iters; ++i) {
double t;
@ -529,6 +528,9 @@ run_test(iotype iot, parameters parms)
pio_time_destroy(res.timers);
}
/*
* Show various statics
*/
/* show mpi write statics */
if (pio_debug_level >= 3) {
/* output all of the times for all iterations */
@ -691,7 +693,7 @@ get_minmax(minmax *mm, double val)
* Modifications:
*/
static minmax
accumulate_minmax_stuff(minmax *mm, long raw_size, int count)
accumulate_minmax_stuff(minmax *mm, off_t raw_size, int count)
{
register int i;
minmax total_mm;
@ -944,15 +946,16 @@ parse_command_line(int argc, char *argv[])
* M, m - Megabyte
* G, g - Gigabyte
*
* Return: The size as a LONG. If an unknown size indicator is used, then
* the program will exit with EXIT_FAILURE as the return value.
* Return: The size as a off_t because this is related to file size.
* If an unknown size indicator is used, then the program will
* exit with EXIT_FAILURE as the return value.
* Programmer: Bill Wendling, 18. December 2001
* Modifications:
*/
static long
static off_t
parse_size_directive(const char *size)
{
long s;
off_t s;
char *endptr;
s = strtol(size, &endptr, 10);

2
perform/pio_perf.h
View File

@ -24,7 +24,7 @@ typedef struct parameters_ {
long num_dsets; /* Number of datasets to create */
long num_elmts; /* Number of native ints in each dset */
int num_iters; /* Number of times to loop doing the IO */
long buf_size; /* Buffer size */
size_t buf_size; /* Buffer size */
} parameters;
typedef struct results_ {