mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-12-21 08:39:46 +08:00
Code Issues Packages Projects Releases Wiki Activity
d1a3f6fbe6
netcdf-c/nc_perf/tst_compress.c
Dennis Heimbigner 36102e3c32 Improve UTF8 Support On Windows 
re: Issue https://github.com/Unidata/netcdf-c/issues/2190

The primary purpose of this PR is to improve the utf8 support
for windows. This is persuant to a change in Windows that
supports utf8 natively (almost). The almost means that it is
still utf16 internally and the set of characters representable
by utf8 is larger than those representable by utf16.

This leaves open the question in the Issue about handling
the Windows 1252 character set.

This required the following changes:

1. Test the Windows build and major version in order to see if
   native utf8 is supported.
2. If native utf8 is supported, Modify dpathmgr.c to call the 8-bit
   version of the windows fopen() and open() functions.
3. In support of this, programs that use XGetOpt (Windows versions)
   need to get the command line as utf8 and then parse to
   arc+argv as utf8. This requires using a homegrown command line parser
   named XCommandLineToArgvA.
4. Add a utility program called "acpget" that prints out the
   current Windows code page and locale.

Additionally, some technical debt was cleaned up as follows:

1. Unify all the places which attempt to read all or a part
   of a file into the dutil.c#NC_readfile code.
2. Similary unify all the code that creates temp files into
   dutil.c#NC_mktmp code.
3. Convert almost all remaining calls to fopen() and open()
   to NCfopen() and NCopen3(). This is to ensure that path management
   is used consistently. This touches a number of files.
4. extern->EXTERNL as needed to get it to work under Windows.
2022-02-08 20:53:30 -07:00

657 lines
23 KiB
C
Raw Blame History

/*
Copyright 2021, UCAR/Unidata See COPYRIGHT file for copying and
redistribution conditions.
This program tests and benchmarks netcdf-4 I/O doing compression.
This is a sequential version of tst_compress_par.c.
Also see the file gfs_sample.cdl to see what is being produced by
this program.
Ed Hartnett, 11/27/21
*/
#include <config.h>
#include <nc_tests.h>
#include <time.h>
#include <sys/time.h> /* Extra high precision time info. */
#include "err_macros.h"
#include <H5public.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include <netcdf.h>
#include <netcdf_par.h>
#include <netcdf_meta.h>
#define TEST_NAME "tst_compress"
#define NUM_META_VARS 7
#define NUM_META_TRIES 2
#define NDIM2 2
#define NDIM4 4
#define NDIM5 5
#define NUM_PROC 4
#define NUM_SHUFFLE_SETTINGS 1
/* #define NUM_DEFLATE_LEVELS 3 */
#define NUM_DEFLATE_LEVELS 1
#define NUM_UNLIM_TRIES 1
#define NUM_NSD_SETTINGS 2
#define THOUSAND 1000
#define NUM_DATA_VARS 3
#define ERR_AWFUL 1
#define NUM_TRIES 2
#define USE_SMALL 1
#ifdef USE_SMALL
#define GRID_XT_LEN 8
#define GRID_YT_LEN 4
#define PFULL_LEN 4
#define PHALF_LEN 5
#else
#define GRID_XT_LEN 3072
#define GRID_YT_LEN 1536
#define PFULL_LEN 127
#define PHALF_LEN 128
#endif /* USE_SMALL */
#define TIME_LEN 1
#define MAX_COMPRESSION_FILTERS 4
char compression_filter_name[MAX_COMPRESSION_FILTERS][NC_MAX_NAME + 1];
int deflate_level[MAX_COMPRESSION_FILTERS][NUM_DEFLATE_LEVELS];
int nsd[NUM_NSD_SETTINGS] = {0, 4};
char dim_name[NDIM5][NC_MAX_NAME + 1] = {"grid_xt", "grid_yt", "pfull",
"phalf", "time"};
char var_name[NUM_META_VARS][NC_MAX_NAME + 1] = {"grid_xt", "lon", "grid_yt",
"lat", "pfull", "phalf", "time"};
int var_type[NUM_META_VARS] = {NC_DOUBLE, NC_DOUBLE, NC_DOUBLE, NC_DOUBLE,
NC_FLOAT, NC_FLOAT, NC_DOUBLE};
int dim_len[NDIM5] = {GRID_XT_LEN, GRID_YT_LEN, PFULL_LEN, PHALF_LEN,
TIME_LEN};
/* Prototype from tst_utils.c. */
int nc4_timeval_subtract(struct timeval *result, struct timeval *x,
struct timeval *y);
/* Get the size of a file in bytes. */
int
get_file_size(char *filename, size_t *file_size)
{
FILE *fp;
assert(filename && file_size);
fp = NCfopen(filename, "r");
if (fp)
{
fseek(fp, 0 , SEEK_END);
*file_size = ftell(fp);
fclose(fp);
}
return 0;
}
/* Check all the metadata, including coordinate variable data. */
int
check_meta(int ncid, int *data_varid, int s, int f, int deflate, int u,
size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start,
size_t *data_count, size_t pfull_start, size_t pfull_size, float *pfull,
size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start,
size_t grid_yt_size, double *grid_yt, size_t *latlon_start,
size_t *latlon_count, double *lat, double *lon)
{
int ndims, nvars, natts, unlimdimid;
char name_in[NC_MAX_NAME + 1];
int xtype_in;
int ndims_in;
int dimids_in[NDIM4];
size_t len_in;
double *grid_xt_in, *grid_yt_in;
double *lat_in, *lon_in;
float *phalf_in, *pfull_in;
int d, v, i;
/* Check number of dims, vars, atts. */
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
if (ndims != NDIM5 || nvars != NUM_META_VARS + NUM_DATA_VARS ||
natts != 0) ERR;
if (unlimdimid != (u ? 4 : -1)) ERR;
/* Check the dimensions. */
for (d = 0; d < NDIM5; d++)
{
if (nc_inq_dim(ncid, d, name_in, &len_in)) ERR;
if (strcmp(name_in, dim_name[d]) || len_in != dim_len[d]) ERR;
}
/* Check metadata vars. */
for (v = 0; v < NUM_META_VARS; v++)
{
if (nc_inq_var(ncid, v, name_in, &xtype_in, &ndims_in, dimids_in,
&natts)) ERR;
if (strcmp(name_in, var_name[v]) || xtype_in != var_type[v]) ERR;
}
/* Check the values for grid_xt. */
if (!(grid_xt_in = malloc(grid_xt_size * sizeof(double)))) ERR;
if (nc_get_vara_double(ncid, 0, &grid_xt_start, &grid_xt_size, grid_xt_in)) ERR;
for (i = 0; i < grid_xt_size; i++)
if (grid_xt_in[i] != grid_xt[i]) ERR;
free(grid_xt_in);
/* Check the values for lon. */
if (!(lon_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lon_in)) ERR;
for (i = 0; i < latlon_count[0] * latlon_count[1]; i++)
if (lon_in[i] != lon[i]) ERR;
free(lon_in);
/* Check the values for grid_yt. */
if (!(grid_yt_in = malloc(grid_yt_size * sizeof(double)))) ERR;
if (nc_get_vara_double(ncid, 2, &grid_yt_start, &grid_yt_size, grid_yt_in)) ERR;
for (i = 0; i < grid_yt_size; i++)
if (grid_yt_in[i] != grid_yt[i]) ERR;
free(grid_yt_in);
/* Check the values for lat. */
if (!(lat_in = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (nc_get_vara_double(ncid, 1, latlon_start, latlon_count, lat_in)) ERR;
for (i = 0; i < latlon_count[0] * latlon_count[1]; i++)
if (lat_in[i] != lat[i]) ERR;
free(lat_in);
/* Check the values for pfull. */
if (!(pfull_in = malloc(pfull_size * sizeof(float)))) ERR;
if (nc_get_vara_float(ncid, 4, &pfull_start, &pfull_size, pfull_in)) ERR;
for (i = 0; i < pfull_size; i++)
if (pfull_in[i] != pfull[i]) ERR;
free(pfull_in);
/* Check the values for phalf. */
if (!(phalf_in = malloc(phalf_size * sizeof(float)))) ERR;
if (nc_get_vara_float(ncid, 5, &phalf_start, &phalf_size, phalf_in)) ERR;
for (i = 0; i < phalf_size; i++)
if (phalf_in[i] != phalf[i]) ERR;
free(phalf_in);
return 0;
}
/* Write all the metadata, including coordinate variable data. */
int
write_meta(int ncid, int *data_varid, int s, int f, int nsd, int deflate, int u,
size_t phalf_size, size_t phalf_start, float *phalf, size_t *data_start,
size_t *data_count, size_t pfull_start, size_t pfull_size, float *pfull,
size_t grid_xt_start, size_t grid_xt_size, double *grid_xt, size_t grid_yt_start,
size_t grid_yt_size, double *grid_yt, size_t *latlon_start,
size_t *latlon_count, double *lat, double *lon)
{
int dimid[NDIM5];
int dimid_data[NDIM4];
int varid[NUM_META_VARS];
double value_time = 2.0;
int dv;
/* Turn off fill mode. */
if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR;
/* Define dimension grid_xt. */
if (nc_def_dim(ncid, dim_name[0], dim_len[0], &dimid[0])) ERR;
/* Define dimension grid_yt. */
if (nc_def_dim(ncid, dim_name[1], dim_len[1], &dimid[1])) ERR;
/* Define variable grid_xt. */
if (nc_def_var(ncid, var_name[0], var_type[0], 1, &dimid[0], &varid[0])) ERR;
/* Define variable lon. */
if (nc_def_var(ncid, var_name[1], var_type[1], 2, dimid, &varid[1])) ERR;
if (nc_put_att_text(ncid, varid[1], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR;
if (nc_put_att_text(ncid, varid[1], "units", strlen("degrees_E"), "degrees_E")) ERR;
if (nc_put_att_text(ncid, varid[0], "cartesian_axis", strlen("X"), "X")) ERR;
/* Define variable grid_yt. */
if (nc_def_var(ncid, var_name[2], var_type[2], 1, &dimid[1], &varid[2])) ERR;
/* Define variable lat. */
if (nc_def_var(ncid, var_name[3], var_type[3], 2, dimid, &varid[3])) ERR;
if (nc_put_att_text(ncid, varid[3], "long_name", strlen("T-cell latitude"), "T-cell latitude")) ERR;
if (nc_put_att_text(ncid, varid[3], "units", strlen("degrees_N"), "degrees_N")) ERR;
if (nc_put_att_text(ncid, varid[2], "cartesian_axis", strlen("Y"), "Y")) ERR;
/* Define dimension pfull. */
if (nc_def_dim(ncid, dim_name[2], dim_len[2], &dimid[2])) ERR;
/* Define variable pfull and write data. */
if (nc_def_var(ncid, var_name[4], var_type[4], 1, &dimid[2], &varid[4])) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_float(ncid, varid[4], &pfull_start, &pfull_size, pfull)) ERR;
if (nc_redef(ncid)) ERR;
/* Define dimension phalf. This dim is only used by the phalf coord var. */
if (nc_def_dim(ncid, dim_name[3], dim_len[3], &dimid[3])) ERR;
/* Define coord variable phalf and write data. */
if (nc_def_var(ncid, var_name[5], var_type[5], 1, &dimid[3], &varid[5])) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_float(ncid, varid[5], &phalf_start, &phalf_size, phalf)) ERR;
if (nc_redef(ncid)) ERR;
/* Define dimension time, sometimes the unlimited dimension,
* sometimes a fixed dim of 1. */
if (nc_def_dim(ncid, dim_name[4], (u ? NC_UNLIMITED : 1), &dimid[4])) ERR;
/* Define variable time and write data. */
if (nc_def_var(ncid, var_name[6], var_type[6], 1, &dimid[4], &varid[6])) ERR;
if (nc_enddef(ncid)) ERR;
/* In NOAA code, do all processors write the single time value? */
if (nc_put_var_double(ncid, varid[6], &value_time)) ERR;;
if (nc_redef(ncid)) ERR;
/* Write variable grid_xt data. */
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_double(ncid, varid[0], &grid_xt_start, &grid_xt_size, grid_xt)) ERR;
if (nc_redef(ncid)) ERR;
/* Write lon data. */
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_double(ncid, varid[1], latlon_start, latlon_count, lon)) ERR;
if (nc_redef(ncid)) ERR;
/* Write grid_yt data. */
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_double(ncid, varid[2], &grid_yt_start, &grid_yt_size, grid_yt)) ERR;
if (nc_redef(ncid)) ERR;
/* Write lat data. */
if (nc_enddef(ncid)) ERR;
if (nc_put_vara_double(ncid, varid[3], latlon_start, latlon_count, lat)) ERR;
/* Specify dimensions for our data vars. */
dimid_data[0] = dimid[4];
dimid_data[1] = dimid[2];
dimid_data[2] = dimid[1];
dimid_data[3] = dimid[0];
/* Define data variables. */
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
char data_var_name[NC_MAX_NAME + 1];
sprintf(data_var_name, "var_%d", dv);
if (nc_redef(ncid)) ERR;
if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR;
if (nsd)
if (nc_def_var_quantize(ncid, data_varid[dv], NC_QUANTIZE_BITGROOM, nsd)) ERR;
/* Setting any filter only will work for HDF5-1.10.3 and later */
/* versions. Do nothing for "none". */
if (!strcmp(compression_filter_name[f], "zlib"))
if (nc_def_var_deflate(ncid, data_varid[dv], s, 1, deflate)) ERR;
#if NC_HAS_SZIP_WRITE
if (!strcmp(compression_filter_name[f], "szip"))
if (nc_def_var_szip(ncid, data_varid[dv], 32, 32)) ERR;
#endif /* NC_HAS_SZIP_WRITE */
if (nc_enddef(ncid)) ERR;
}
if (nc_redef(ncid)) ERR;
if (nc_put_att_text(ncid, varid[0], "long_name", strlen("T-cell longitude"), "T-cell longitude")) ERR;
if (nc_put_att_text(ncid, varid[0], "units", strlen("degrees_E"), "degrees_E")) ERR;
if (nc_put_att_text(ncid, varid[2], "long_name", strlen("T-cell latiitude"), "T-cell latiitude")) ERR;
if (nc_put_att_text(ncid, varid[2], "units", strlen("degrees_N"), "degrees_N")) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_redef(ncid)) ERR;
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
float compress_err = 42.22;
int nbits = 5;
if (nc_put_att_float(ncid, data_varid[dv], "max_abs_compression_error", NC_FLOAT, 1, &compress_err)) ERR;
if (nc_put_att_int(ncid, data_varid[dv], "nbits", NC_INT, 1, &nbits)) ERR;
}
if (nc_enddef(ncid)) ERR;
return 0;
}
/* Calculate the decomposition of the 2D lat/lon coordinate
* variables. */
int
decomp_latlon(int *dim_len, size_t *latlon_start,
size_t *latlon_count, double **lat, double **lon)
{
int i, j;
assert(dim_len && latlon_start && latlon_count && lat && lon && !*lat &&
!*lon);
/* Size of local arrays (i.e. for this pe) lon and lat data. */
latlon_start[0] = 0;
latlon_start[1] = 0;
latlon_count[0] = dim_len[0];
latlon_count[1] = dim_len[1];
/* Allocate storage. */
if (!(*lon = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
if (!(*lat = malloc(latlon_count[0] * latlon_count[1] * sizeof(double)))) ERR;
/* Now calculate some latlon values to write. */
for (i = 0; i < latlon_count[0]; i++)
{
for (j = 0; j < latlon_count[1]; j++)
{
(*lon)[j * latlon_count[0] + i] = 100 + i + j;
(*lat)[j * latlon_count[0] + i] = 100 + i + j;
}
}
return 0;
}
/* Based on the MPI rank and number of tasks, calculate the
* decomposition of the 4D data. */
int
decomp_4D(int *dim_len, size_t *start, size_t *count)
{
/* Time dimension. */
start[0] = 0;
count[0] = 1;
/* Vertical dimension (pfull). */
count[1] = dim_len[2];
start[1] = 0;
start[2] = 0;
start[3] = 0;
count[2] = dim_len[1];
count[3] = dim_len[0];
return 0;
}
/* Decompose the grid_xt and grid_yt coordinate vars, and also come up
* with some data. */
int
decomp_grid(int *dim_len, size_t *grid_xt_start, size_t *grid_xt_size,
size_t *grid_yt_start, size_t *grid_yt_size, double **grid_xt, double **grid_yt)
{
int i;
/* Size of local (i.e. for this pe) grid_xt data. */
*grid_xt_size = dim_len[0];
*grid_xt_start = 0;
/* Size of local (i.e. for this pe) grid_yt data. */
*grid_yt_size = dim_len[1];
*grid_yt_start = 0;
/* Allocate storage for the grid_xy and grid_yt coordinate
* variable data. */
if (!(*grid_xt = malloc(*grid_xt_size * sizeof(double)))) ERR;
if (!(*grid_yt = malloc(*grid_yt_size * sizeof(double)))) ERR;
/* Fill the grid_xt and grid_yt coordinate data arrays. */
for (i = 0; i < *grid_xt_size; i++)
(*grid_xt)[i] = 100 + i;
for (i = 0; i < *grid_yt_size; i++)
(*grid_yt)[i] = 100 + i;
return 0;
}
/* Decompose the pfull and phalf coordinate vars. */
int
decomp_p(size_t *data_count, int *dim_len,
size_t *phalf_start, size_t *phalf_size, float **phalf,
size_t *pfull_start, size_t *pfull_size, float **pfull)
{
int i;
/* Size of local (i.e. for this pe) phalf data. */
*phalf_size = dim_len[3];
*phalf_start = 0;
*pfull_size = dim_len[2];
*pfull_start = 0;
/* Allocate space on this pe to hold the coordinate var data for this pe. */
if (!(*pfull = malloc(data_count[1] * sizeof(float)))) ERR;
if (!(*phalf = malloc(*phalf_size * sizeof(float)))) ERR;
/* Some fake data for this pe to write. */
for (i = 0; i < data_count[1]; i++)
(*pfull)[i] = 100 + i;
for (i = 0; i < *phalf_size; i++)
(*phalf)[i] = 100 + i;
return 0;
}
/* Determine what compression filters are present. */
int
find_filters(int *num_compression_filters, char compression_filter_name[][NC_MAX_NAME + 1],
int deflate_level[][NUM_DEFLATE_LEVELS])
{
int nfilters = 0;
int i;
/* Try with no compression. */
strcpy(compression_filter_name[nfilters], "none");
nfilters++;
/* zlib is always present. */
strcpy(compression_filter_name[nfilters], "zlib");
for (i = 0; i < NUM_DEFLATE_LEVELS; i++)
deflate_level[nfilters][i] = i + 1;
/* deflate_level[nfilters][0] = 1; */
/* deflate_level[nfilters][1] = 4; */
/* deflate_level[nfilters][2] = 9; */
nfilters++;
/* szip is optionally present. */
#if NC_HAS_SZIP_WRITE
strcpy(compression_filter_name[nfilters], "szip");
nfilters++;
#endif /* NC_HAS_SZIP_WRITE */
*num_compression_filters = nfilters;
return 0;
}
int
main(int argc, char **argv)
{
/* For timing. */
struct timeval start_time, end_time, diff_time;
int write_1_us;
int ncid;
size_t latlon_start[NDIM2], latlon_count[NDIM2];
size_t data_start[NDIM4], data_count[NDIM4];
/* Variables. */
int data_varid[NUM_DATA_VARS];
size_t pfull_size, pfull_start;
float *pfull = NULL;
size_t phalf_size, phalf_start;
float *phalf = NULL;
size_t grid_xt_size, grid_xt_start;
double *grid_xt = NULL;
size_t grid_yt_size, grid_yt_start;
double *grid_yt = NULL;
double *lon = NULL;
double *lat = NULL;
float *value_data;
/* Compression filter info. */
int num_compression_filters;
int f, s, n, try;
int i, j, k, dv, dl;
int ret;
/* Determine what compression filters are present. */
if ((ret = find_filters(&num_compression_filters, compression_filter_name, deflate_level)))
return ret;
/* Determine 4D data decomposition to write data vars. */
if (decomp_4D(dim_len, data_start, data_count)) ERR;
/* Determine 2D data decomposition to write lat/lon coordinate vars. */
if (decomp_latlon(dim_len, latlon_start, latlon_count, &lat, &lon)) ERR;
/* Decompose grid_xt and grid_yt coordiate vars. */
if (decomp_grid(dim_len, &grid_xt_start, &grid_xt_size,
&grid_yt_start, &grid_yt_size, &grid_xt, &grid_yt)) ERR;
/* Decompose phalf and pfull. */
if (decomp_p(data_count, dim_len, &phalf_start,
&phalf_size, &phalf, &pfull_start, &pfull_size, &pfull)) ERR;
/* printf("%d: data_count[3] %ld data_count[2] %ld data_count[1] %ld\n", my_rank, */
/* data_count[3], data_count[2], data_count[1]); */
/* Allocate space to hold the data. */
if (!(value_data = malloc(data_count[3] * data_count[2] * data_count[1] *
sizeof(float)))) ERR;
/* Create some data. */
size_t cnt = 0;
for (k = 0; k < data_count[1]; k++)
{
for (j = 0; j < data_count[2]; j++)
{
for(i = 0; i < data_count[3]; i++)
{
/* value_data[cnt] = (-1 * i%2) * my_rank * 1000 + cnt / sqrt(my_rank + cnt + 1) - (-1 * i%3 * i); */
value_data[cnt] = (-1 * i%2) + cnt / sqrt(cnt + 1) - (-1 * i%2 * i);
/* printf("%d: value_data[%ld] %g\n", my_rank, cnt, value_data[cnt]); */
cnt++;
}
}
}
printf("Benchmarking creation of file similar to one produced by the UFS.\n");
printf("comp, level, nsd, shuffle, data wr rate (MB/s), file size (MB)\n");
/* for (f = 0; f < num_compression_filters; f++) */
for (try = 0; try < NUM_TRIES; try++)
{
printf("try %d:\n", try);
for (f = 0; f < 2; f++)
{
/* for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++) */
for (s = 0; s < 1; s++)
{
/* for (n = 0; n < NUM_NSD_SETTINGS; n++) */
for (n = 0; n < 1; n++)
{
for (dl = 0; dl < NUM_DEFLATE_LEVELS; dl++)
{
size_t file_size;
char file_name[NC_MAX_NAME * 3 + 1];
/* No deflate levels for szip or none. */
if (!strcmp(compression_filter_name[f], "szip") && dl) continue;
if (!strcmp(compression_filter_name[f], "none") && dl) continue;
/* Use the same filename every time, so we don't
* create many large files, just one. ;-) */
sprintf(file_name, "%s.nc", TEST_NAME);
/* Remove the last file. Ignore errors. */
remove(file_name);
/* nc_set_log_level(3); */
/* Create a netcdf-4 file. */
if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR;
if (write_meta(ncid, data_varid, s, f, nsd[n], deflate_level[f][dl], 0,
phalf_size, phalf_start, phalf,
data_start, data_count, pfull_start, pfull_size, pfull, grid_xt_start,
grid_xt_size, grid_xt, grid_yt_start,
grid_yt_size, grid_yt, latlon_start,
latlon_count, lat, lon)) ERR;
if (gettimeofday(&start_time, NULL)) ERR;
/* Write one record each of the data variables. */
for (dv = 0; dv < NUM_DATA_VARS; dv++)
{
/* printf("%d: data_start %ld %ld %ld %ld data_count %ld %ld %ld %ld\n", my_rank, data_start[0], data_start[1], */
/* data_start[2], data_start[3], data_count[0], data_count[1], data_count[2], data_count[3]); */
if (nc_put_vara_float(ncid, data_varid[dv], data_start, data_count,
value_data)) ERR;
if (nc_redef(ncid)) ERR;
}
/* Close the file. */
if (nc_close(ncid)) ERR;
/* Stop the data timer. */
if (gettimeofday(&end_time, NULL)) ERR;
if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR;
write_1_us = (int)diff_time.tv_sec * MILLION + (int)diff_time.tv_usec;
/* printf("write_1_us %d\n", write_1_us); */
/* Get the file size. */
if (get_file_size(file_name, &file_size)) ERR;
/* Check the file metadata for correctness. */
if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR;
if (check_meta(ncid, data_varid, s, f, deflate_level[f][dl], 0,
phalf_size, phalf_start, phalf,
data_start, data_count, pfull_start, pfull_size,
pfull, grid_xt_start, grid_xt_size, grid_xt,
grid_yt_start, grid_yt_size, grid_yt, latlon_start,
latlon_count, lat, lon)) ERR;
if (nc_close(ncid)) ERR;
/* Print out results. */
{
float data_size, data_rate;
data_size = (NUM_DATA_VARS * dim_len[0] * dim_len[1] * dim_len[2] *
dim_len[4] * sizeof(float))/MILLION;
/* printf("data_size %f write_1_us / MILLION %g\n", data_size, (float)write_1_us/MILLION); */
data_rate = (float)data_size / ((float)write_1_us / MILLION);
printf("%s, %d, %d, %d, %g, %g\n", compression_filter_name[f],
deflate_level[f][dl], nsd[n], s,
data_rate, (float)file_size/MILLION);
}
} /* next deflate level */
} /* next nsd */
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
} /* next try */
/* Free resources. */
if (grid_xt)
free(grid_xt);
if (grid_yt)
free(grid_yt);
if (pfull)
free(pfull);
if (phalf)
free(phalf);
if (lon)
free(lon);
if (lat)
free(lat);
free(value_data);
SUMMARIZE_ERR;
FINAL_RESULTS;
return 0;
}
Reference in New Issue View Git Blame Copy Permalink