mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-01 17:06:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Edward Hartnett 2020-07-05 08:54:50 -06:00
parent 86c39cf7a2
commit 4a80190c64
1 changed files with 180 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

345
nc_perf/tst_gfs_data_1.c
View File

@ -52,38 +52,38 @@ get_file_size(char *filename, size_t *file_size)
{
FILE *fp;
assert(filename && file_size);
fp = fopen(filename, "r");
if (fp)
{
fseek(fp, 0 , SEEK_END);
*file_size = ftell(fp);
fclose(fp);
fseek(fp, 0 , SEEK_END);
*file_size = ftell(fp);
fclose(fp);
}
return 0;
}
/* Write all the metadata, including coordinate variable data. */
int
write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start,
float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc,
size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start,
size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count,
double *value_lat_loc, double *value_lon_loc, int my_rank)
write_meta(int ncid, int *data_varid, int s, int f, int deflate, int *dim_len, size_t phalf_loc_size, size_t phalf_start,
float *value_phalf_loc, size_t *data_start, size_t *data_count, float *value_pfull_loc,
size_t grid_xt_start, size_t grid_xt_loc_size, double *value_grid_xt_loc, size_t grid_yt_start,
size_t grid_yt_loc_size, double *value_grid_yt_loc, size_t *latlon_start, size_t *latlon_count,
double *value_lat_loc, double *value_lon_loc, int my_rank)
{
char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull",
"phalf", "time"};
"phalf", "time"};
int dimid[NDIM5];
int dimid_data[NDIM4];
char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt",
"lat", "pfull", "phalf", "time"};
"lat", "pfull", "phalf", "time"};
int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE,
NC_FLOAT, NC_FLOAT, NC_DOUBLE};
NC_FLOAT, NC_FLOAT, NC_DOUBLE};
int varid[NUM_META_VARS];
double value_time = 2.0;
int dv;
int res;
/* Turn off fill mode. */
if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR;
@ -116,7 +116,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le
if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR;
if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR;
/* Define dimension pfull. */
if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR;
@ -148,7 +148,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le
/* In NOAA code, do all processors write the single time value? */
if (my_rank == 0)
if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;;
if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;;
if (nc_redef(ncid)) ERR;
/* Write variable grid_xt data. */
@ -179,50 +179,50 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le
/* Define data variables. */
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
char data_var_name[NC_MAX_NAME + 1];
char data_var_name[NC_MAX_NAME + 1];
sprintf(data_var_name, "var_%d", dv);
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR;
sprintf(data_var_name, "var_%d", dv);
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR;
/* Setting any filter only will work for HDF5-1.10.3 and later */
/* versions. */
if (!f)
res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate);
else
{
res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0);
if (!res)
res = nc_def_var_szip(ncid, data_varid[dv], 32, 32);
}
/* Setting any filter only will work for HDF5-1.10.3 and later */
/* versions. */
if (!f)
res = nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate);
else
{
res = nc_def_var_deflate(ncid, data_varid[dv], s, 0, 0);
if (!res)
res = nc_def_var_szip(ncid, data_varid[dv], 32, 32);
}
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if (res) ERR;
if (res) ERR;
#else
if (res != NC_EINVAL) ERR;
if (res != NC_EINVAL) ERR;
#endif
if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_var_par_access(ncid, data_varid[dv], NC_COLLECTIVE)) ERR;
if (nc_enddef(ncid)) ERR;
}
if (nc_redef(ncid)) ERR;
if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR;
if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR;
if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR;
if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_redef(ncid)) ERR;
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
float compress_err = 42.22;
int nbits = 5;
if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR;
if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR;
float compress_err = 42.22;
int nbits = 5;
if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR;
if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR;
}
if (nc_enddef(ncid)) ERR;
return 0;
}
@ -231,7 +231,7 @@ write_metadata(int ncid, int *data_varid, int s, int f, int deflate, int *dim_le
* decomposition of the 2D lat/lon coordinate variables. */
int
decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start,
size_t *latlon_count)
size_t *latlon_count)
{
/* Size of local arrays (i.e. for this pe) lon and lat data. This
@ -239,70 +239,70 @@ decomp_2D(int my_rank, int mpi_size, int *dim_len, size_t *latlon_start,
latlon_count[0] = dim_len[0]/2;
if (my_rank == 0 || my_rank == 2)
{
latlon_start[0] = 0;
latlon_start[0] = 0;
}
else
{
latlon_start[0] = dim_len[0]/2;
latlon_start[0] = dim_len[0]/2;
}
latlon_count[1] = dim_len[1]/2;
if (my_rank == 0 || my_rank == 1)
{
latlon_start[1] = 0;
latlon_start[1] = 0;
}
else
{
latlon_start[1] = dim_len[1]/2;
latlon_start[1] = dim_len[1]/2;
}
printf("%d: latlon_start %ld %ld latlon_count %ld %ld\n", my_rank, latlon_start[0],
latlon_start[1], latlon_count[0], latlon_count[1]);
latlon_start[1], latlon_count[0], latlon_count[1]);
return 0;
}
/* Based on the MPI rank and number of tasks, calculate the
/* Based on the MPI rank and number of tasks, calculate the
* decomposition of the 4D data. */
int
decomp_4D(int my_rank, int mpi_size, int *dim_len, size_t *start, size_t *count)
{
start[0] = 0;
count[0] = 1;
count[1] = dim_len[2]/mpi_size;
start[1] = my_rank * count[1];
/* Add any extra to the end. */
if (my_rank == mpi_size - 1)
count[1] = count[1] + dim_len[2] % mpi_size;
count[1] = count[1] + dim_len[2] % mpi_size;
if (mpi_size == 4)
{
if (my_rank == 0 || my_rank == 1)
{
start[2] = 0;
start[3] = 0;
}
else
{
start[2] = 768;
start[3] = 768;
}
count[2] = 768;
count[3] = 1536;
if (my_rank == 0 || my_rank == 1)
{
start[2] = 0;
start[3] = 0;
}
else
{
start[2] = 768;
start[3] = 768;
}
count[2] = 768;
count[3] = 1536;
}
else if (mpi_size == 36)
{
start[2] = my_rank * 256;
start[3] = my_rank * 512;
count[2] = 256;
count[3] = 512;
start[2] = my_rank * 256;
start[3] = my_rank * 512;
count[2] = 256;
count[3] = 512;
}
else
return ERR_AWFUL;
return ERR_AWFUL;
printf("%d: start %ld %ld %ld %ld count %ld %ld %ld %ld\n", my_rank, start[0],
start[1], start[2], start[3], count[0], count[1], count[2], count[3]);
start[1], start[2], start[3], count[0], count[1], count[2], count[3]);
return 0;
}
@ -317,25 +317,25 @@ main(int argc, char **argv)
/* For timing. */
double meta_start_time, meta_stop_time;
double data_start_time, data_stop_time;
int ncid;
size_t latlon_start[NDIM2], latlon_count[NDIM2];
size_t data_start[NDIM4], data_count[NDIM4];
int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN,
TIME_LEN};
TIME_LEN};
/* Variables. */
int data_varid[NUM_DATA_VARS];
float *value_pfull_loc;
size_t phalf_loc_size, phalf_start;
float *value_phalf_loc;
size_t grid_xt_loc_size, grid_xt_start;
double *value_grid_xt_loc;
size_t grid_yt_loc_size, grid_yt_start;
double *value_grid_yt_loc;
double *value_lon_loc;
double *value_lat_loc;
float *pfull = NULL;
size_t phalf_size, phalf_start;
float *phalf = NULL;
size_t grid_xt_size, grid_xt_start;
double *grid_xt = NULL;
size_t grid_yt_size, grid_yt_start;
double *grid_yt = NULL;
double *lon = NULL;
double *lat = NULL;
float *value_data;
/* int deflate_level[NUM_DEFLATE_LEVELS] = {1, 4, 9}; */
int deflate_level[NUM_DEFLATE_LEVELS] = {1};
@ -355,58 +355,59 @@ main(int argc, char **argv)
if (decomp_2D(my_rank, mpi_size, dim_len, latlon_start, latlon_count)) ERR;
/* Size of local (i.e. for this pe) grid_xt data. */
grid_xt_loc_size = dim_len[0]/mpi_size;
grid_xt_start = my_rank * grid_xt_loc_size;
grid_xt_size = dim_len[0]/mpi_size;
grid_xt_start = my_rank * grid_xt_size;
if (my_rank == mpi_size - 1)
grid_xt_loc_size = grid_xt_loc_size + dim_len[0] % mpi_size;
grid_xt_size = grid_xt_size + dim_len[0] % mpi_size;
/* Size of local (i.e. for this pe) grid_yt data. */
grid_yt_loc_size = dim_len[1]/mpi_size;
grid_yt_start = my_rank * grid_yt_loc_size;
grid_yt_size = dim_len[1]/mpi_size;
grid_yt_start = my_rank * grid_yt_size;
if (my_rank == mpi_size - 1)
grid_yt_loc_size = grid_yt_loc_size + dim_len[1] % mpi_size;
grid_yt_size = grid_yt_size + dim_len[1] % mpi_size;
/* Size of local (i.e. for this pe) phalf data. */
phalf_loc_size = dim_len[3]/mpi_size;
phalf_start = my_rank * phalf_loc_size;
phalf_size = dim_len[3]/mpi_size;
phalf_start = my_rank * phalf_size;
if (my_rank == mpi_size - 1)
phalf_loc_size = phalf_loc_size + dim_len[3] % mpi_size;
phalf_size = phalf_size + dim_len[3] % mpi_size;
/* Allocate space on this pe to hold the coordinate var data for this pe. */
if (!(value_pfull_loc = malloc(data_count[1] * sizeof(float)))) ERR;
if (!(value_phalf_loc = malloc(phalf_loc_size * sizeof(float)))) ERR;
if (!(value_grid_xt_loc = malloc(grid_xt_loc_size * sizeof(double)))) ERR;
if (!(value_grid_yt_loc = malloc(grid_yt_loc_size * sizeof(double)))) ERR;
if (!(value_lon_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (!(value_lat_loc = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (!(pfull = malloc(data_count[1] * sizeof(float)))) ERR;
if (!(phalf = malloc(phalf_size * sizeof(float)))) ERR;
if (!(grid_xt = malloc(grid_xt_size * sizeof(double)))) ERR;
if (!(grid_yt = malloc(grid_yt_size * sizeof(double)))) ERR;
if (!(lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (!(lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
/* Allocate space to hold the data. */
if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR;
/* Some fake data for this pe to write. */
for (i = 0; i < data_count[1]; i++)
value_pfull_loc[i] = my_rank * 100 + i;
for (i = 0; i < phalf_loc_size; i++)
value_phalf_loc[i] = my_rank * 100 + i;
for (i = 0; i < grid_xt_loc_size; i++)
value_grid_xt_loc[i] = my_rank * 100 + i;
for (i = 0; i < grid_yt_loc_size; i++)
value_grid_yt_loc[i] = my_rank * 100 + i;
pfull[i] = my_rank * 100 + i;
for (i = 0; i < phalf_size; i++)
phalf[i] = my_rank * 100 + i;
for (i = 0; i < grid_xt_size; i++)
grid_xt[i] = my_rank * 100 + i;
for (i = 0; i < grid_yt_size; i++)
grid_yt[i] = my_rank * 100 + i;
for (j = 0; j < latlon_count[1]; j++)
{
for(i = 0; i < latlon_count[0]; i++)
{
value_lon_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j;
value_lat_loc[j * latlon_count[0] + i] = my_rank * 100 + i + j;
for (k = 0; k < data_count[1]; k++)
value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k;
}
for(i = 0; i < latlon_count[0]; i++)
{
lon[j * latlon_count[0] + i] = my_rank * 100 + i + j;
lat[j * latlon_count[0] + i] = my_rank * 100 + i + j;
for (k = 0; k < data_count[1]; k++)
value_data[j * latlon_count[0] + i] = my_rank * 100 + i + j + k;
}
}
if (my_rank == 0)
{
printf("Benchmarking creation of UFS file.\n");
printf("comp, level, shuffle, meta wr time (s), data wr time (s), file size\n");
printf("Benchmarking creation of UFS file.\n");
printf("comp, level, shuffle, meta wr time (s), data wr time (s), "
"file size\n");
}
{
int s;
@ -414,77 +415,91 @@ main(int argc, char **argv)
{
for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
{
for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++)
{
size_t file_size;
for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++)
{
size_t file_size;
/* No deflate levels for szip. */
if (f && dl) continue;
/* nc_set_log_level(3); */
/* Create a parallel netcdf-4 file. */
meta_start_time = MPI_Wtime();
if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
/* No deflate levels for szip. */
if (f && dl) continue;
if (write_metadata(ncid, data_varid, s, f, deflate_level[dl], dim_len, phalf_loc_size, phalf_start, value_phalf_loc, data_start, data_count, value_pfull_loc,
grid_xt_start, grid_xt_loc_size, value_grid_xt_loc, grid_yt_start, grid_yt_loc_size, value_grid_yt_loc,
latlon_start, latlon_count, value_lat_loc, value_lon_loc, my_rank)) ERR;
MPI_Barrier(MPI_COMM_WORLD);
meta_stop_time = MPI_Wtime();
data_start_time = MPI_Wtime();
/* nc_set_log_level(3); */
/* Create a parallel netcdf-4 file. */
meta_start_time = MPI_Wtime();
if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info,
&ncid)) ERR;
/* Write one record each of the data variables. */
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
int r;
if ((r = nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)))
{
printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl);
printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", my_rank, data_start[0], data_start[1],
data_start[2], data_start[3], data_count[0], data_count[1], data_count[2], data_count[3]);
ERR;
}
if (nc_redef(ncid)) ERR;
}
if (write_meta(ncid, data_varid, s, f, deflate_level[dl],
dim_len, phalf_size, phalf_start, phalf,
data_start, data_count, pfull, grid_xt_start,
grid_xt_size, grid_xt, grid_yt_start,
grid_yt_size, grid_yt, latlon_start,
latlon_count, lat, lon, my_rank)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
/* Stop the timer for metadata writes. */
MPI_Barrier(MPI_COMM_WORLD);
meta_stop_time = MPI_Wtime();
data_start_time = MPI_Wtime();
/* Stop the data timer. */
MPI_Barrier(MPI_COMM_WORLD);
data_stop_time = MPI_Wtime();
/* Write one record each of the data variables. */
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
int r;
if ((r = nc_put_vara_float(ncid, data_varid[dv],
data_start, data_count, value_data)))
{
printf("%d: r %d f %d s %d dl %d\n", my_rank, r, f, s, dl);
printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n",
my_rank, data_start[0], data_start[1],
data_start[2], data_start[3], data_count[0],
data_count[1], data_count[2], data_count[3]);
ERR;
}
if (nc_redef(ncid)) ERR;
}
/* Get the file size. */
if (get_file_size(FILE_NAME, &file_size)) ERR;
/* Close the file. */
if (nc_close(ncid)) ERR;
/* Print out results. */
if (my_rank == 0)
printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"), deflate_level[dl], s, meta_stop_time - meta_start_time,
data_stop_time - data_start_time, file_size);
} /* next deflate level */
/* Stop the data timer. */
MPI_Barrier(MPI_COMM_WORLD);
data_stop_time = MPI_Wtime();
/* Get the file size. */
if (get_file_size(FILE_NAME, &file_size)) ERR;
/* Print out results. */
if (my_rank == 0)
printf("%s, %d, %d, %g, %g, %ld\n", (f ? "szip" : "zlib"),
deflate_level[dl], s, meta_stop_time - meta_start_time,
data_stop_time - data_start_time, file_size);
} /* next deflate level */
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
/* Free resources. */
free(value_grid_xt_loc);
free(value_grid_yt_loc);
free(value_pfull_loc);
free(value_phalf_loc);
free(value_lon_loc);
free(value_lat_loc);
free(value_data);
/* Free resources. */
if (grid_xt)
free(grid_xt);
if (grid_yt)
free(grid_yt);
if (pfull)
free(pfull);
if (phalf)
free(phalf);
if (lon)
free(lon);
if (lat)
free(lat);
free(value_data);
}
if (!my_rank)
SUMMARIZE_ERR;
/* Shut down MPI. */
MPI_Finalize();
if (!my_rank)
FINAL_RESULTS;
FINAL_RESULTS;
return 0;
}