/* Copyright 2021, UCAR/Unidata See COPYRIGHT file for copying and redistribution conditions. This program tests and benchmarks netcdf-4 I/O doing compression. This is a sequential version of tst_compress_par.c. Also see the file gfs_sample.cdl to see what is being produced by this program. Ed Hartnett, 11/27/21 */ #include #include #include #include /* Extra high precision time info. */ #include "err_macros.h" #include #include #include #include #include #include #define TEST_NAME "tst_compress" #define NUM_META_VARS 7 #define NUM_META_TRIES 2 #define NDIM2 2 #define NDIM4 4 #define NDIM5 5 #define NUM_PROC 4 #define NUM_SHUFFLE_SETTINGS 1 /* #define NUM_DEFLATE_LEVELS 3 */ #define NUM_DEFLATE_LEVELS 1 #define NUM_UNLIM_TRIES 1 #define NUM_NSD_SETTINGS 2 #define THOUSAND 1000 #define NUM_DATA_VARS 3 #define ERR_AWFUL 1 #define NUM_TRIES 2 #define USE_SMALL 1 #ifdef USE_SMALL #define GRID_XT_LEN 8 #define GRID_YT_LEN 4 #define PFULL_LEN 4 #define PHALF_LEN 5 #else #define GRID_XT_LEN 3072 #define GRID_YT_LEN 1536 #define PFULL_LEN 127 #define PHALF_LEN 128 #endif /* USE_SMALL */ #define TIME_LEN 1 #define MAX_COMPRESSION_FILTERS 4 char compression_filter_name[MAX_COMPRESSION_FILTERS][NC_MAX_NAME + 1]; int deflate_level[MAX_COMPRESSION_FILTERS][NUM_DEFLATE_LEVELS]; int nsd[NUM_NSD_SETTINGS] = {0, 4}; char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull", "phalf", "time"}; char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt", "lat", "pfull", "phalf", "time"}; int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_FLOAT, NC_FLOAT, NC_DOUBLE}; int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN, TIME_LEN}; /* Prototype from tst_utils.c. */ int nc4_timeval_subtract(struct timeval *result, struct timeval *x, struct timeval *y); /* Get the size of a file in bytes. */ int get_file_size(char *filename, size_t *file_size) { FILE *fp; assert(filename && file_size); fp = NCfopen(filename, "r"); if (fp) { fseek(fp, 0 , SEEK_END); *file_size = ftell(fp); fclose(fp); } return 0; } /* Check all the metadata, including coordinate variable data. */ int check_meta(int ncid, int *data_varid, int s, int f, int deflate, int u, size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, size_t *data_count, size_t pfull_start, size_t pfull_size, float *pfull, size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, double *lat, double *lon) { int ndims, nvars, natts, unlimdimid; char name_in[NC_MAX_NAME + 1]; int xtype_in; int ndims_in; int dimids_in[NDIM4]; size_t len_in; double *grid_xt_in, *grid_yt_in; double *lat_in, *lon_in; float *phalf_in, *pfull_in; int d, v, i; /* Check number of dims, vars, atts. */ if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; if (ndims != NDIM5 || nvars != NUM_META_VARS + NUM_DATA_VARS || natts != 0) ERR; if (unlimdimid != (u ? 4 : -1)) ERR; /* Check the dimensions. */ for (d = 0; d < NDIM5; d++) { if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR; if (strcmp(name_in, dim_name[d]) || len_in != dim_len[d]) ERR; } /* Check metadata vars. */ for (v = 0; v < NUM_META_VARS; v++) { if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in, &natts)) ERR; if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v]) ERR; } /* Check the values for grid_xt. */ if (!(grid_xt_in = malloc(grid_xt_size * sizeof(double)))) ERR; if (nc_get_vara_double(ncid, 0, &grid_xt_start, &grid_xt_size, grid_xt_in)) ERR; for (i = 0; i < grid_xt_size; i++) if (grid_xt_in[i] != grid_xt[i]) ERR; free(grid_xt_in); /* Check the values for lon. */ if (!(lon_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lon_in)) ERR; for (i = 0; i < latlon_count[0] * latlon_count[1]; i++) if (lon_in[i] != lon[i]) ERR; free(lon_in); /* Check the values for grid_yt. */ if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR; if (nc_get_vara_double(ncid, 2, &grid_yt_start, &grid_yt_size, grid_yt_in)) ERR; for (i = 0; i < grid_yt_size; i++) if (grid_yt_in[i] != grid_yt[i]) ERR; free(grid_yt_in); /* Check the values for lat. */ if (!(lat_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lat_in)) ERR; for (i = 0; i < latlon_count[0] * latlon_count[1]; i++) if (lat_in[i] != lat[i]) ERR; free(lat_in); /* Check the values for pfull. */ if (!(pfull_in = malloc(pfull_size * sizeof(float)))) ERR; if (nc_get_vara_float(ncid, 4, &pfull_start, &pfull_size, pfull_in)) ERR; for (i = 0; i < pfull_size; i++) if (pfull_in[i] != pfull[i]) ERR; free(pfull_in); /* Check the values for phalf. */ if (!(phalf_in = malloc(phalf_size * sizeof(float)))) ERR; if (nc_get_vara_float(ncid, 5, &phalf_start, &phalf_size, phalf_in)) ERR; for (i = 0; i < phalf_size; i++) if (phalf_in[i] != phalf[i]) ERR; free(phalf_in); return 0; } /* Write all the metadata, including coordinate variable data. */ int write_meta(int ncid, int *data_varid, int s, int f, int nsd, int deflate, int u, size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start, size_t *data_count, size_t pfull_start, size_t pfull_size, float *pfull, size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start, size_t grid_yt_size, double *grid_yt, size_t *latlon_start, size_t *latlon_count, double *lat, double *lon) { int dimid[NDIM5]; int dimid_data[NDIM4]; int varid[NUM_META_VARS]; double value_time = 2.0; int dv; /* Turn off fill mode. */ if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; /* Define dimension grid_xt. */ if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR; /* Define dimension grid_yt. */ if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR; /* Define variable grid_xt. */ if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR; /* Define variable lon. */ if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR; if (nc_put_att_text(ncid, varid[1], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; if (nc_put_att_text(ncid, varid[1], "units", strlen("degrees_E"), "degrees_E")) ERR; if (nc_put_att_text(ncid, varid[0], "cartesian_axis", strlen("X"), "X")) ERR; /* Define variable grid_yt. */ if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR; /* Define variable lat. */ if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR; if (nc_put_att_text(ncid, varid[3], "long_name", strlen("T-cell latitude"), "T-cell latitude")) ERR; if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR; if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR; /* Define dimension pfull. */ if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR; /* Define variable pfull and write data. */ if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR; if (nc_enddef(ncid)) ERR; if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_size, pfull)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension phalf. This dim is only used by the phalf coord var. */ if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR; /* Define coord variable phalf and write data. */ if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR; if (nc_enddef(ncid)) ERR; if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_size, phalf)) ERR; if (nc_redef(ncid)) ERR; /* Define dimension time, sometimes the unlimited dimension, * sometimes a fixed dim of 1. */ if (nc_def_dim(ncid, dim_name[4], (u ? NC_UNLIMITED : 1), &dimid[4])) ERR; /* Define variable time and write data. */ if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR; if (nc_enddef(ncid)) ERR; /* In NOAA code, do all processors write the single time value? */ if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;; if (nc_redef(ncid)) ERR; /* Write variable grid_xt data. */ if (nc_enddef(ncid)) ERR; if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_size, grid_xt)) ERR; if (nc_redef(ncid)) ERR; /* Write lon data. */ if (nc_enddef(ncid)) ERR; if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, lon)) ERR; if (nc_redef(ncid)) ERR; /* Write grid_yt data. */ if (nc_enddef(ncid)) ERR; if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_size, grid_yt)) ERR; if (nc_redef(ncid)) ERR; /* Write lat data. */ if (nc_enddef(ncid)) ERR; if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, lat)) ERR; /* Specify dimensions for our data vars. */ dimid_data[0] = dimid[4]; dimid_data[1] = dimid[2]; dimid_data[2] = dimid[1]; dimid_data[3] = dimid[0]; /* Define data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { char data_var_name[NC_MAX_NAME + 1]; sprintf(data_var_name, "var_%d", dv); if (nc_redef(ncid)) ERR; if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; if (nsd) if (nc_def_var_quantize(ncid, data_varid[dv], NC_QUANTIZE_BITGROOM, nsd)) ERR; /* Setting any filter only will work for HDF5-1.10.3 and later */ /* versions. Do nothing for "none". */ if (!strcmp(compression_filter_name[f], "zlib")) if (nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate)) ERR; #if NC_HAS_SZIP_WRITE if (!strcmp(compression_filter_name[f], "szip")) if (nc_def_var_szip(ncid, data_varid[dv], 32, 32)) ERR; #endif /* NC_HAS_SZIP_WRITE */ if (nc_enddef(ncid)) ERR; } if (nc_redef(ncid)) ERR; if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR; if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR; if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR; if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR; if (nc_enddef(ncid)) ERR; if (nc_redef(ncid)) ERR; for (dv = 0; dv < NUM_DATA_VARS; dv++) { float compress_err = 42.22; int nbits = 5; if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR; if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR; } if (nc_enddef(ncid)) ERR; return 0; } /* Calculate the decomposition of the 2D lat/lon coordinate * variables. */ int decomp_latlon(int *dim_len, size_t *latlon_start, size_t *latlon_count, double **lat, double **lon) { int i, j; assert(dim_len && latlon_start && latlon_count && lat && lon && !*lat && !*lon); /* Size of local arrays (i.e. for this pe) lon and lat data. */ latlon_start[0] = 0; latlon_start[1] = 0; latlon_count[0] = dim_len[0]; latlon_count[1] = dim_len[1]; /* Allocate storage. */ if (!(*lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; if (!(*lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR; /* Now calculate some latlon values to write. */ for (i = 0; i < latlon_count[0]; i++) { for (j = 0; j < latlon_count[1]; j++) { (*lon)[j * latlon_count[0] + i] = 100 + i + j; (*lat)[j * latlon_count[0] + i] = 100 + i + j; } } return 0; } /* Based on the MPI rank and number of tasks, calculate the * decomposition of the 4D data. */ int decomp_4D(int *dim_len, size_t *start, size_t *count) { /* Time dimension. */ start[0] = 0; count[0] = 1; /* Vertical dimension (pfull). */ count[1] = dim_len[2]; start[1] = 0; start[2] = 0; start[3] = 0; count[2] = dim_len[1]; count[3] = dim_len[0]; return 0; } /* Decompose the grid_xt and grid_yt coordinate vars, and also come up * with some data. */ int decomp_grid(int *dim_len, size_t *grid_xt_start, size_t *grid_xt_size, size_t *grid_yt_start, size_t *grid_yt_size, double **grid_xt, double **grid_yt) { int i; /* Size of local (i.e. for this pe) grid_xt data. */ *grid_xt_size = dim_len[0]; *grid_xt_start = 0; /* Size of local (i.e. for this pe) grid_yt data. */ *grid_yt_size = dim_len[1]; *grid_yt_start = 0; /* Allocate storage for the grid_xy and grid_yt coordinate * variable data. */ if (!(*grid_xt = malloc(*grid_xt_size * sizeof(double)))) ERR; if (!(*grid_yt = malloc(*grid_yt_size * sizeof(double)))) ERR; /* Fill the grid_xt and grid_yt coordinate data arrays. */ for (i = 0; i < *grid_xt_size; i++) (*grid_xt)[i] = 100 + i; for (i = 0; i < *grid_yt_size; i++) (*grid_yt)[i] = 100 + i; return 0; } /* Decompose the pfull and phalf coordinate vars. */ int decomp_p(size_t *data_count, int *dim_len, size_t *phalf_start, size_t *phalf_size, float **phalf, size_t *pfull_start, size_t *pfull_size, float **pfull) { int i; /* Size of local (i.e. for this pe) phalf data. */ *phalf_size = dim_len[3]; *phalf_start = 0; *pfull_size = dim_len[2]; *pfull_start = 0; /* Allocate space on this pe to hold the coordinate var data for this pe. */ if (!(*pfull = malloc(data_count[1] * sizeof(float)))) ERR; if (!(*phalf = malloc(*phalf_size * sizeof(float)))) ERR; /* Some fake data for this pe to write. */ for (i = 0; i < data_count[1]; i++) (*pfull)[i] = 100 + i; for (i = 0; i < *phalf_size; i++) (*phalf)[i] = 100 + i; return 0; } /* Determine what compression filters are present. */ int find_filters(int *num_compression_filters, char compression_filter_name[][NC_MAX_NAME + 1], int deflate_level[][NUM_DEFLATE_LEVELS]) { int nfilters = 0; int i; /* Try with no compression. */ strcpy(compression_filter_name[nfilters], "none"); nfilters++; /* zlib is always present. */ strcpy(compression_filter_name[nfilters], "zlib"); for (i = 0; i < NUM_DEFLATE_LEVELS; i++) deflate_level[nfilters][i] = i + 1; /* deflate_level[nfilters][0] = 1; */ /* deflate_level[nfilters][1] = 4; */ /* deflate_level[nfilters][2] = 9; */ nfilters++; /* szip is optionally present. */ #if NC_HAS_SZIP_WRITE strcpy(compression_filter_name[nfilters], "szip"); nfilters++; #endif /* NC_HAS_SZIP_WRITE */ *num_compression_filters = nfilters; return 0; } int main(int argc, char **argv) { /* For timing. */ struct timeval start_time, end_time, diff_time; int write_1_us; int ncid; size_t latlon_start[NDIM2], latlon_count[NDIM2]; size_t data_start[NDIM4], data_count[NDIM4]; /* Variables. */ int data_varid[NUM_DATA_VARS]; size_t pfull_size, pfull_start; float *pfull = NULL; size_t phalf_size, phalf_start; float *phalf = NULL; size_t grid_xt_size, grid_xt_start; double *grid_xt = NULL; size_t grid_yt_size, grid_yt_start; double *grid_yt = NULL; double *lon = NULL; double *lat = NULL; float *value_data; /* Compression filter info. */ int num_compression_filters; int f, s, n, try; int i, j, k, dv, dl; int ret; /* Determine what compression filters are present. */ if ((ret = find_filters(&num_compression_filters, compression_filter_name, deflate_level))) return ret; /* Determine 4D data decomposition to write data vars. */ if (decomp_4D(dim_len, data_start, data_count)) ERR; /* Determine 2D data decomposition to write lat/lon coordinate vars. */ if (decomp_latlon(dim_len, latlon_start, latlon_count, &lat, &lon)) ERR; /* Decompose grid_xt and grid_yt coordiate vars. */ if (decomp_grid(dim_len, &grid_xt_start, &grid_xt_size, &grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR; /* Decompose phalf and pfull. */ if (decomp_p(data_count, dim_len, &phalf_start, &phalf_size, &phalf, &pfull_start, &pfull_size, &pfull)) ERR; /* printf("%d: data_count[3] %ld data_count[2] %ld data_count[1] %ld\n", my_rank, */ /* data_count[3], data_count[2], data_count[1]); */ /* Allocate space to hold the data. */ if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] * sizeof(float)))) ERR; /* Create some data. */ size_t cnt = 0; for (k = 0; k < data_count[1]; k++) { for (j = 0; j < data_count[2]; j++) { for(i = 0; i < data_count[3]; i++) { /* value_data[cnt] = (-1 * i%2) * my_rank * 1000 + cnt / sqrt(my_rank + cnt + 1) - (-1 * i%3 * i); */ value_data[cnt] = (-1 * i%2) + cnt / sqrt(cnt + 1) - (-1 * i%2 * i); /* printf("%d: value_data[%ld] %g\n", my_rank, cnt, value_data[cnt]); */ cnt++; } } } printf("Benchmarking creation of file similar to one produced by the UFS.\n"); printf("comp, level, nsd, shuffle, data wr rate (MB/s), file size (MB)\n"); /* for (f = 0; f < num_compression_filters; f++) */ for (try = 0; try < NUM_TRIES; try++) { printf("try %d:\n", try); for (f = 0; f < 2; f++) { /* for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) */ for (s = 0; s < 1; s++) { /* for (n = 0; n < NUM_NSD_SETTINGS; n++) */ for (n = 0; n < 1; n++) { for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++) { size_t file_size; char file_name[NC_MAX_NAME * 3 + 1]; /* No deflate levels for szip or none. */ if (!strcmp(compression_filter_name[f], "szip") && dl) continue; if (!strcmp(compression_filter_name[f], "none") && dl) continue; /* Use the same filename every time, so we don't * create many large files, just one. ;-) */ sprintf(file_name, "%s.nc", TEST_NAME); /* Remove the last file. Ignore errors. */ remove(file_name); /* nc_set_log_level(3); */ /* Create a netcdf-4 file. */ if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR; if (write_meta(ncid, data_varid, s, f, nsd[n], deflate_level[f][dl], 0, phalf_size, phalf_start, phalf, data_start, data_count, pfull_start, pfull_size, pfull, grid_xt_start, grid_xt_size, grid_xt, grid_yt_start, grid_yt_size, grid_yt, latlon_start, latlon_count, lat, lon)) ERR; if (gettimeofday(&start_time, NULL)) ERR; /* Write one record each of the data variables. */ for (dv = 0; dv < NUM_DATA_VARS; dv++) { /* printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", my_rank, data_start[0], data_start[1], */ /* data_start[2], data_start[3], data_count[0], data_count[1], data_count[2], data_count[3]); */ if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count, value_data)) ERR; if (nc_redef(ncid)) ERR; } /* Close the file. */ if (nc_close(ncid)) ERR; /* Stop the data timer. */ if (gettimeofday(&end_time, NULL)) ERR; if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; write_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec; /* printf("write_1_us %d\n", write_1_us); */ /* Get the file size. */ if (get_file_size(file_name, &file_size)) ERR; /* Check the file metadata for correctness. */ if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR; if (check_meta(ncid, data_varid, s, f, deflate_level[f][dl], 0, phalf_size, phalf_start, phalf, data_start, data_count, pfull_start, pfull_size, pfull, grid_xt_start, grid_xt_size, grid_xt, grid_yt_start, grid_yt_size, grid_yt, latlon_start, latlon_count, lat, lon)) ERR; if (nc_close(ncid)) ERR; /* Print out results. */ { float data_size, data_rate; data_size = (NUM_DATA_VARS * dim_len[0] * dim_len[1] * dim_len[2] * dim_len[4] * sizeof(float))/MILLION; /* printf("data_size %f write_1_us / MILLION %g\n", data_size, (float)write_1_us/MILLION); */ data_rate = (float)data_size / ((float)write_1_us / MILLION); printf("%s, %d, %d, %d, %g, %g\n", compression_filter_name[f], deflate_level[f][dl], nsd[n], s, data_rate, (float)file_size/MILLION); } } /* next deflate level */ } /* next nsd */ } /* next shuffle filter test */ } /* next compression filter (zlib and szip) */ } /* next try */ /* Free resources. */ if (grid_xt) free(grid_xt); if (grid_yt) free(grid_yt); if (pfull) free(pfull); if (phalf) free(phalf); if (lon) free(lon); if (lat) free(lat); free(value_data); SUMMARIZE_ERR; FINAL_RESULTS; return 0; }