mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-21 03:13:42 +08:00
Merge pull request #1555 from NOAA-GSD/ejh_bm_file
update of benchmark program bm_file to prevent integer overflows on very big files
This commit is contained in:
Ward Fisher
GitHub
commit
e0717731b9
@ -28,7 +28,8 @@
|
||||
#define MILLION 1000000
|
||||
#define BAD -99
|
||||
#define NOMEM -98
|
||||
#define MAX_VO 50 /* Max number of var options on command line. */
|
||||
#define MAX_VO 255 /* Max number of var options on command line. */
|
||||
#define MAX_VO_PRINTED 3
|
||||
#define MAX_DIMS 7 /* Max dim for variables in input file. */
|
||||
|
||||
/* This struct holds data about what options we want to apply to
|
||||
@ -46,20 +47,20 @@ typedef struct {
|
||||
/* This macro prints an error message with line number and name of
|
||||
* test program. */
|
||||
#define ERR1(n) do { \
|
||||
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
|
||||
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d - %s\n", \
|
||||
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
|
||||
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d - %s\n", \
|
||||
__FILE__, __LINE__, nc_strerror(n)); \
|
||||
return n; \
|
||||
} while (0)
|
||||
return n; \
|
||||
} while (0)
|
||||
|
||||
#ifdef USE_PARALLEL
|
||||
/* Error handling code for MPI calls. */
|
||||
#define MPIERR(e) do { \
|
||||
MPI_Error_string(e, err_buffer, &resultlen); \
|
||||
printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
|
||||
MPI_Finalize(); \
|
||||
return 2; \
|
||||
} while (0)
|
||||
MPI_Error_string(e, err_buffer, &resultlen); \
|
||||
printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
|
||||
MPI_Finalize(); \
|
||||
return 2; \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/* Prototype from tst_utils.c. */
|
||||
@ -234,7 +235,7 @@ check_att(int ncid1, int ncid2, int varid, int a)
|
||||
ret = BAD;
|
||||
}
|
||||
|
||||
exit:
|
||||
exit:
|
||||
/* Free up our resources. */
|
||||
if (d)
|
||||
free(d);
|
||||
@ -246,7 +247,7 @@ check_att(int ncid1, int ncid2, int varid, int a)
|
||||
|
||||
/* Do two files contain the same data and metadata? */
|
||||
static int
|
||||
cmp_file(char *file1, char *file2, int *meta_read_us, int *data_read_us,
|
||||
cmp_file(char *file1, char *file2, int *meta_read_us, size_t *data_read_us,
|
||||
int use_par, int par_access, int do_cmp, int p, int my_rank,
|
||||
int slow_count, int verbose, int num_vo, VAR_OPTS_T *vo, int use_scs)
|
||||
{
|
||||
@ -424,7 +425,7 @@ cmp_file(char *file1, char *file2, int *meta_read_us, int *data_read_us,
|
||||
*data_read_us += (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
|
||||
#endif
|
||||
if (verbose)
|
||||
printf("%d: reading copy step %d, var %d took %d micro-seconds\n",
|
||||
printf("%d: reading copy step %d, var %d took %ld micro-seconds\n",
|
||||
my_rank, step, v, *data_read_us);
|
||||
|
||||
/* Check data. */
|
||||
@ -458,8 +459,8 @@ cmp_file(char *file1, char *file2, int *meta_read_us, int *data_read_us,
|
||||
static
|
||||
int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
|
||||
int num_vo, VAR_OPTS_T *vo, int *meta_read_us, int *meta_write_us,
|
||||
int *data_read_us, int *data_write_us, int *in_format, int use_par,
|
||||
int par_access, long long *num_bytes, int p, int my_rank,
|
||||
size_t *data_read_us, int *data_write_us, int *in_format, int use_par,
|
||||
int par_access, float *num_bytes, int p, int my_rank,
|
||||
int slow_count, int verbose, int use_scs, int endianness,
|
||||
int convert_unlim)
|
||||
{
|
||||
@ -645,7 +646,7 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
|
||||
char type_name[NC_MAX_NAME+1];
|
||||
int start_inc;
|
||||
int step, num_steps;
|
||||
int var_num_bytes;
|
||||
float var_num_bytes;
|
||||
|
||||
/* Learn about this var. */
|
||||
if ((ret = nc_inq_var(ncid_in, v, name, &xtype, &ndims, dimids, &natts)))
|
||||
@ -729,7 +730,7 @@ int copy_file(char *file_name_in, char *file_name_out, int cmode_out,
|
||||
*data_read_us += (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
|
||||
#endif
|
||||
if (verbose)
|
||||
printf("%d: reading step %d, var %d took %d micro-seconds\n",
|
||||
printf("%d: reading step %d, var %d took %ld micro-seconds\n",
|
||||
my_rank, step, v, *data_read_us);
|
||||
|
||||
/* Write the data to the output file. */
|
||||
@ -816,7 +817,7 @@ main(int argc, char **argv)
|
||||
extern int opterr;
|
||||
extern char *optarg;
|
||||
char file_in[NC_MAX_NAME + 1], file_out[NC_MAX_NAME + 1] = {""};
|
||||
int c;
|
||||
char file_out_2[NC_MAX_NAME + 1];
|
||||
int out_format, in_format, header = 0, doublecheck = 0;
|
||||
int convert_unlim = 0;
|
||||
char *str1, *str2, *token, *subtoken;
|
||||
@ -824,20 +825,25 @@ main(int argc, char **argv)
|
||||
int i, ndims, o1;
|
||||
int cmode = 0;
|
||||
int mpiio = 0;
|
||||
int meta_read_us = 0, meta_write_us = 0, data_read_us = 0, data_write_us = 0;
|
||||
int meta_read2_us = 0, data_read2_us = 0;
|
||||
int tmeta_read_us = 0, tmeta_write_us = 0, tdata_read_us = 0, tdata_write_us = 0;
|
||||
int tmeta_read2_us = 0, tdata_read2_us = 0;
|
||||
int meta_read_us = 0, meta_write_us = 0, data_write_us = 0;
|
||||
size_t data_read_us = 0;
|
||||
int meta_read2_us = 0;
|
||||
size_t data_read2_us = 0;
|
||||
int tmeta_read_us = 0, tmeta_write_us = 0, tdata_write_us = 0;
|
||||
size_t tdata_read_us = 0;
|
||||
int tmeta_read2_us = 0;
|
||||
size_t tdata_read2_us = 0;
|
||||
VAR_OPTS_T vo[MAX_VO];
|
||||
int use_par = 0, par_access = 0;
|
||||
int do_cmp = 0, verbose = 0;
|
||||
int ret;
|
||||
float read_rate, write_rate, reread_rate;
|
||||
int slow_count = 10, use_scs = 0;
|
||||
int endianness = 0;
|
||||
long long num_bytes = 0;
|
||||
float num_bytes = 0;
|
||||
int p = 1, my_rank = 0;
|
||||
int c;
|
||||
int v, d;
|
||||
int ret;
|
||||
|
||||
#ifdef USE_PARALLEL
|
||||
MPI_Init(&argc, &argv);
|
||||
@ -1061,10 +1067,19 @@ main(int argc, char **argv)
|
||||
* file is exactly the same. */
|
||||
if (doublecheck)
|
||||
{
|
||||
/* We need a string long enough for the copy command. */
|
||||
char cmd[NC_MAX_NAME * 2 + 5];
|
||||
|
||||
#ifdef USE_PARALLEL
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
if ((ret = cmp_file(file_in, file_out, &meta_read2_us, &data_read2_us,
|
||||
/* Create a copy of file_out. This will defeat any buffering
|
||||
* that may exist from the fact that we just wrote file_out. */
|
||||
sprintf(file_out_2, "tst_copy_%s", file_out);
|
||||
sprintf(cmd, "cp %s %s\n", file_out, file_out_2);
|
||||
system(cmd);
|
||||
|
||||
if ((ret = cmp_file(file_in, file_out_2, &meta_read2_us, &data_read2_us,
|
||||
use_par, par_access, do_cmp, p, my_rank, slow_count,
|
||||
verbose, num_vo, vo, use_scs)))
|
||||
return ret;
|
||||
@ -1075,9 +1090,9 @@ main(int argc, char **argv)
|
||||
#ifdef USE_PARALLEL
|
||||
MPI_Reduce(&meta_read_us, &tmeta_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&meta_write_us, &tmeta_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&data_read_us, &tdata_read_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&data_read_us, &tdata_read_us, 1, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&data_write_us, &tdata_write_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&data_read2_us, &tdata_read2_us, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
MPI_Reduce(&data_read2_us, &tdata_read2_us, 1, MPI_OFFSET, MPI_MAX, 0, MPI_COMM_WORLD);
|
||||
#else
|
||||
return NC_EPARINIT;
|
||||
#endif
|
||||
@ -1093,11 +1108,11 @@ main(int argc, char **argv)
|
||||
}
|
||||
|
||||
if (verbose)
|
||||
printf("num_bytes=%lld tdata_read_us=%d\n", num_bytes, tdata_read_us);
|
||||
printf("num_bytes=%g tdata_read_us=%ld\n", num_bytes, tdata_read_us);
|
||||
|
||||
read_rate = (float)num_bytes/((float)tdata_read_us/p);
|
||||
write_rate = (float)num_bytes/((float)tdata_write_us/p);
|
||||
reread_rate = (float)num_bytes/((float)tdata_read2_us/p);
|
||||
read_rate = num_bytes/((float)tdata_read_us/p);
|
||||
write_rate = num_bytes/((float)tdata_write_us/p);
|
||||
reread_rate = num_bytes/((float)tdata_read2_us/p);
|
||||
if (verbose)
|
||||
printf("%d: read rate %g, write rate %g, reread_rate %g\n", my_rank, read_rate,
|
||||
write_rate, reread_rate);
|
||||
@ -1121,11 +1136,11 @@ main(int argc, char **argv)
|
||||
"chunksize[3]\n");
|
||||
}
|
||||
|
||||
printf("%d, %d, %ld, %ld, %d, %d, %d, %d, %d, ", in_format, out_format, file_size(file_in),
|
||||
printf("%d, %d, %ld, %ld, %d, %d, %ld, %d, %d, ", in_format, out_format, file_size(file_in),
|
||||
file_size(file_out), tmeta_read_us, tmeta_write_us, tdata_read_us, tdata_write_us,
|
||||
endianness);
|
||||
if (doublecheck)
|
||||
printf("%d, %d, %g, %g, %g, ", tmeta_read2_us, tdata_read2_us, read_rate, write_rate,
|
||||
printf("%d, %ld, %g, %g, %g, ", tmeta_read2_us, tdata_read2_us, read_rate, write_rate,
|
||||
reread_rate);
|
||||
else
|
||||
printf("%g, %g, ", read_rate, write_rate);
|
||||
@ -1135,6 +1150,8 @@ main(int argc, char **argv)
|
||||
{
|
||||
printf("%d, %d, %d, %d, %d, %d ", vo[o1].deflate_num, vo[o1].shuffle,
|
||||
(int)vo[o1].chunksize[0], (int)vo[o1].chunksize[1], (int)vo[o1].chunksize[2], (int)vo[o1].chunksize[3]);
|
||||
if (o1 >= MAX_VO_PRINTED)
|
||||
break;
|
||||
if (o1 != num_vo - 1)
|
||||
printf(", ");
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user