mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-09 08:11:38 +08:00
d2316f866c
Primary Fixes: * Add a whole variable optimization -- used in the rare case that nc_get/put_vara covers the whole of a variable and the variable has a single chunk. * Fix chunking error when stride causes whole chunks to be skipped. * Fix some memory leaks * Add test cases * Add one performance test to nczarr_test/. This uses the timer utils from unit_test: timer_utils.[ch]. * Move ncdumpchunks utility from ncdump to nczarr_test Misc. Other Changes: * Make check for aws libraries conditional on --enable-nczarr-s3 * Remove all but one bm tests from nczarr_test until they are working. * Remove another dependency on HDF5 from supposedly non-HDF5 specific code; specifically hdf5_log_hdf5. * Make the BAIL2 macro be hdf5 specific and replace elsewhere with an HDF5 independent equivalent. * Move hdf5cache.c to libsrc4/nc4cache.c because it is used by nczarr. * Modify unit_tests so that some of them are run even if using Windows. * Misc. small bug fixes and refactors and memory leaks. * Rename some conflicting tests for cmake. * Attempted to make nc_perf work with cmake and failed.
305 lines
9.4 KiB
C
305 lines
9.4 KiB
C
/*
|
|
Copyright 2007-2019, UCAR/Unidata
|
|
See COPYRIGHT file for copying and redistribution conditions.
|
|
|
|
This program creates some test files which are used for performance
|
|
testing.
|
|
|
|
Ed Hartnett
|
|
*/
|
|
#include <config.h>
|
|
#include <nc_tests.h>
|
|
#include "err_macros.h"
|
|
#include <netcdf.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include "ncdispatch.h"
|
|
#include <math.h>
|
|
|
|
/* We will create this file. */
|
|
#define FILE_NAME "tst_floats_1D.nc"
|
|
#define MAX_TYPE_NAME 6
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
printf("\n*** Create some files for testing benchmarks.\n");
|
|
|
|
#ifdef LARGE_FILE_TESTS
|
|
printf("*** Creating very large 3D file...");
|
|
{
|
|
#define LARGE_VAR_NAME "Really_Large_Variable"
|
|
#define LARGE_FILE "tst_large.nc"
|
|
#define NDIMS3 3
|
|
#define D0 400
|
|
#define D1 2000
|
|
#define D2 2000
|
|
|
|
int ncid, dimids[NDIMS3], varid;
|
|
size_t start[NDIMS3], count[NDIMS3];
|
|
size_t dim_len[NDIMS3] = {D0, D1, D2};
|
|
size_t chunk_sizes[NDIMS3] = {1, D1, D2};
|
|
float *data;
|
|
char file_name[NC_MAX_NAME * 2 + 1];
|
|
int d, i;
|
|
|
|
/* Initialize the data to random floats. */
|
|
if (!(data = (float *)malloc(D1 * D2 * sizeof(float)))) ERR;
|
|
for (i = 0; i < D1 * D2; i++)
|
|
data[i] = (float)rand();
|
|
|
|
/* User TEMP_LARGE as the directory. */
|
|
if (strlen(TEMP_LARGE) + strlen(LARGE_FILE) > NC_MAX_NAME * 2) ERR;
|
|
sprintf(file_name, "%s/%s", TEMP_LARGE, LARGE_FILE);
|
|
|
|
/* Create file with 3 dims, one variable. */
|
|
if (nc_create(file_name, NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, "d0", D0, &dimids[0])) ERR;
|
|
if (nc_def_dim(ncid, "d1", D1, &dimids[1])) ERR;
|
|
if (nc_def_dim(ncid, "d2", D2, &dimids[2])) ERR;
|
|
if (nc_def_var(ncid, LARGE_VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
|
|
#ifndef NOBUG
|
|
{int status;
|
|
status = nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunk_sizes);
|
|
if(status)
|
|
printf("nc_def_var_chunking fail: %d: %s\n",status,nc_strerror(status));
|
|
}
|
|
#else
|
|
if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunk_sizes)) ERR;
|
|
#endif
|
|
if (nc_enddef(ncid)) ERR;
|
|
|
|
/* Write the data one slice at a time. */
|
|
start[0] = 0;
|
|
count[0] = 1;
|
|
for (d = 1; d < NDIMS3; d++)
|
|
{
|
|
start[d] = 0;
|
|
count[d] = dim_len[d];
|
|
}
|
|
for ( ; start[0] < D0; (start[0])++)
|
|
if (nc_put_vara_float(ncid, varid, start, count, (const float *) data))
|
|
ERR_RET;
|
|
|
|
/* Close up shop. */
|
|
if (nc_close(ncid)) ERR;
|
|
free(data);
|
|
}
|
|
SUMMARIZE_ERR;
|
|
#endif /* LARGE_FILE_TESTS */
|
|
|
|
printf("*** Creating a file with floats...");
|
|
{
|
|
#define DIM_NAME "stupidity"
|
|
#define NUMDIMS 1
|
|
#define DIMLEN 10000
|
|
|
|
int ncid, dimids[NUMDIMS], varid;
|
|
char var_name[NC_MAX_NAME + 1] = {"Billy-Bob"};
|
|
int ndims, nvars, natts, unlimdimid;
|
|
nc_type xtype;
|
|
char name_in[NC_MAX_NAME + 1];
|
|
size_t len;
|
|
float data[DIMLEN], data_in[DIMLEN];
|
|
int i;
|
|
|
|
for (i = 0; i < DIMLEN; i++)
|
|
data[i] = ((float)rand() / (float)(RAND_MAX));
|
|
|
|
/* Create a netCDF netCDF-4/HDF5 format file, with 1 var. */
|
|
/* if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;*/
|
|
if (nc_create(FILE_NAME, 0, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, DIM_NAME, DIMLEN, dimids)) ERR;
|
|
if (nc_def_var(ncid, var_name, NC_FLOAT, NUMDIMS, dimids, &varid)) ERR;
|
|
if (nc_enddef(ncid)) ERR;
|
|
if (nc_put_var_float(ncid, varid, data)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
|
|
/* Reopen and check the file. */
|
|
if (nc_open(FILE_NAME, 0, &ncid)) ERR;
|
|
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
|
|
if (ndims != NUMDIMS || nvars != 1 || natts != 0 || unlimdimid != -1) ERR;
|
|
if (nc_inq_dimids(ncid, &ndims, dimids, 1)) ERR;
|
|
if (ndims != 1 || dimids[0] != 0) ERR;
|
|
if (nc_inq_dim(ncid, 0, name_in, &len)) ERR;
|
|
if (strcmp(name_in, DIM_NAME) || len != DIMLEN) ERR;
|
|
if (nc_inq_var(ncid, 0, name_in, &xtype, &ndims, dimids, &natts)) ERR;
|
|
if (strcmp(name_in, var_name) || xtype != NC_FLOAT || ndims != 1 ||
|
|
dimids[0] != 0 || natts != 0) ERR;
|
|
if (nc_get_var_float(ncid, 0, data_in)) ERR;
|
|
for (i = 0; i < DIMLEN; i++)
|
|
if (data_in[i] != data[i]) ERR;
|
|
|
|
if (nc_close(ncid)) ERR;
|
|
}
|
|
|
|
SUMMARIZE_ERR;
|
|
printf("*** Creating files of various dimensions with various types...");
|
|
{
|
|
#define TOTAL_SIZE 100000
|
|
#define MAX_DIMS 6
|
|
#define VAR_NAME "Unimaginatively_Named_Variable"
|
|
#define MAX_TYPES 3
|
|
|
|
int ncid, dimids[MAX_DIMS], varid;
|
|
char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1];
|
|
char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"};
|
|
int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT};
|
|
size_t len;
|
|
float fdata[TOTAL_SIZE];
|
|
int idata[TOTAL_SIZE];
|
|
short sdata[TOTAL_SIZE];
|
|
void *data[MAX_TYPES];
|
|
int ndims;
|
|
int i, d, t;
|
|
|
|
/* Initialize the data to random floats. */
|
|
for (i = 0; i < TOTAL_SIZE; i++)
|
|
{
|
|
fdata[i] = (float)rand();
|
|
idata[i] = rand();
|
|
sdata[i] = (short)rand();
|
|
}
|
|
data[0] = fdata;
|
|
data[1] = idata;
|
|
data[2] = sdata;
|
|
|
|
/* Do the file creation process below for each type. */
|
|
for (t = 0; t < MAX_TYPES; t++)
|
|
{
|
|
/* Create MAX_DIMS files, each with different number of
|
|
* dimensions. */
|
|
for (ndims = 1; ndims <= MAX_DIMS; ndims++)
|
|
{
|
|
sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims);
|
|
if (nc_create(file_name, 0, &ncid)) ERR;
|
|
for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++)
|
|
{
|
|
sprintf(dim_name, "dim_%d", d);
|
|
if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR;
|
|
}
|
|
if (nc_def_var(ncid, VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR;
|
|
if (nc_enddef(ncid)) ERR;
|
|
if (nc_put_var(ncid, varid, data[t])) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
} /* next ndims. */
|
|
} /* next type */
|
|
}
|
|
|
|
SUMMARIZE_ERR;
|
|
printf("*** Creating file like in http://hdfeos.org/workshops/ws06/presentations/Pourmal/HDF5_IO_Perf.pdf...");
|
|
{
|
|
#define XLEN 256
|
|
#define YLEN 256
|
|
#define ZLEN 1024
|
|
#define NDIMS 3
|
|
#define E_VAR_NAME "Like_Elenas_Benchmark"
|
|
#define ELENA_FILE_NAME "tst_elena_int_3D.nc"
|
|
#define E_TYPE_SIZE 4
|
|
|
|
int ncid, dimids[NDIMS], varid;
|
|
int *idata;
|
|
int i;
|
|
|
|
/* Initialize data to random int between 0 and 255. */
|
|
if (!(idata = malloc(XLEN * YLEN * ZLEN * E_TYPE_SIZE)))
|
|
return NC_ENOMEM;
|
|
for (i = 0; i < XLEN * YLEN * ZLEN; i++)
|
|
idata[i] = rand() % 255;
|
|
|
|
/* Create a 3D file with one var. */
|
|
if (nc_create(ELENA_FILE_NAME, NC_CLOBBER, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, "z", ZLEN, &dimids[0])) ERR;
|
|
if (nc_def_dim(ncid, "y", YLEN, &dimids[1])) ERR;
|
|
if (nc_def_dim(ncid, "x", XLEN, &dimids[2])) ERR;
|
|
if (nc_def_var(ncid, E_VAR_NAME, NC_INT, NDIMS, dimids, &varid)) ERR;
|
|
if (nc_enddef(ncid)) ERR;
|
|
|
|
/* Write the data. */
|
|
if (nc_put_var(ncid, varid, idata)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
free(idata);
|
|
}
|
|
|
|
SUMMARIZE_ERR;
|
|
printf("*** Creating super simple file to test non-sequential reads...");
|
|
{
|
|
#define DIM_LEN 10
|
|
#define NDIMS1 1
|
|
#define S_VAR_NAME "My_Favorite_Numbers_in_order"
|
|
#define S_FILE_NAME "tst_simple.nc"
|
|
|
|
int ncid, dimids[NDIMS1], varid;
|
|
int data[DIM_LEN];
|
|
int i;
|
|
|
|
/* Initialize data to my favorite numbers. */
|
|
for (i = 0; i < DIM_LEN; i++)
|
|
data[i] = i;
|
|
|
|
/* Create a file with one var. */
|
|
if (nc_create(S_FILE_NAME, NC_CLOBBER, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, "a", DIM_LEN, &dimids[0])) ERR;
|
|
if (nc_def_var(ncid, S_VAR_NAME, NC_INT, NDIMS1, dimids, &varid)) ERR;
|
|
if (nc_enddef(ncid)) ERR;
|
|
|
|
/* Write the data. */
|
|
if (nc_put_var(ncid, varid, data)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
}
|
|
|
|
SUMMARIZE_ERR;
|
|
printf("*** Creating very simple 3D file...");
|
|
{
|
|
#define SIMPLE_VAR_NAME "Paul_Mau_Dib"
|
|
#define MAX_TYPES 3
|
|
|
|
int ncid, dimids[MAX_DIMS], varid;
|
|
char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1];
|
|
char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"};
|
|
int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT};
|
|
size_t len;
|
|
float fdata[TOTAL_SIZE];
|
|
int idata[TOTAL_SIZE];
|
|
short sdata[TOTAL_SIZE];
|
|
void *data[MAX_TYPES];
|
|
int ndims;
|
|
int i, d, t;
|
|
|
|
/* Initialize the data to random floats. */
|
|
for (i = 0; i < TOTAL_SIZE; i++)
|
|
{
|
|
fdata[i] = (float)rand();
|
|
idata[i] = rand();
|
|
sdata[i] = (short)rand();
|
|
}
|
|
data[0] = fdata;
|
|
data[1] = idata;
|
|
data[2] = sdata;
|
|
|
|
/* Do the file creation process below for each type. */
|
|
for (t = 0; t < MAX_TYPES; t++)
|
|
{
|
|
/* Create MAX_DIMS files, each with different number of
|
|
* dimensions. */
|
|
for (ndims = 1; ndims <= MAX_DIMS; ndims++)
|
|
{
|
|
sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims);
|
|
if (nc_create(file_name, 0, &ncid)) ERR;
|
|
for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++)
|
|
{
|
|
sprintf(dim_name, "dim_%d", d);
|
|
if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR;
|
|
}
|
|
if (nc_def_var(ncid, SIMPLE_VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR;
|
|
if (nc_enddef(ncid)) ERR;
|
|
if (nc_put_var(ncid, varid, data[t])) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
} /* next ndims. */
|
|
} /* next type */
|
|
}
|
|
SUMMARIZE_ERR;
|
|
|
|
FINAL_RESULTS;
|
|
}
|