/* Copyright 2007-2019, UCAR/Unidata See COPYRIGHT file for copying and redistribution conditions. This program creates some test files which are used for performance testing. Ed Hartnett */ #include #include #include "err_macros.h" #include #include #include #include "ncdispatch.h" #include /* We will create this file. */ #define FILE_NAME "tst_floats_1D.nc" #define MAX_TYPE_NAME 6 int main(int argc, char **argv) { printf("\n*** Create some files for testing benchmarks.\n"); #ifdef LARGE_FILE_TESTS printf("*** Creating very large 3D file..."); { #define LARGE_VAR_NAME "Really_Large_Variable" #define LARGE_FILE "tst_large.nc" #define NDIMS3 3 #define D0 400 #define D1 2000 #define D2 2000 int ncid, dimids[NDIMS3], varid; size_t start[NDIMS3], count[NDIMS3]; size_t dim_len[NDIMS3] = {D0, D1, D2}; size_t chunk_sizes[NDIMS3] = {1, D1, D2}; float *data; char file_name[NC_MAX_NAME * 2 + 1]; int d, i; /* Initialize the data to random floats. */ if (!(data = (float *)malloc(D1 * D2 * sizeof(float)))) ERR; for (i = 0; i < D1 * D2; i++) data[i] = (float)rand(); /* User TEMP_LARGE as the directory. */ if (strlen(TEMP_LARGE) + strlen(LARGE_FILE) > NC_MAX_NAME * 2) ERR; sprintf(file_name, "%s/%s", TEMP_LARGE, LARGE_FILE); /* Create file with 3 dims, one variable. */ if (nc_create(file_name, NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, "d0", D0, &dimids[0])) ERR; if (nc_def_dim(ncid, "d1", D1, &dimids[1])) ERR; if (nc_def_dim(ncid, "d2", D2, &dimids[2])) ERR; if (nc_def_var(ncid, LARGE_VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR; #ifndef NOBUG {int status; status = nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunk_sizes); if(status) printf("nc_def_var_chunking fail: %d: %s\n",status,nc_strerror(status)); } #else if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunk_sizes)) ERR; #endif if (nc_enddef(ncid)) ERR; /* Write the data one slice at a time. */ start[0] = 0; count[0] = 1; for (d = 1; d < NDIMS3; d++) { start[d] = 0; count[d] = dim_len[d]; } for ( ; start[0] < D0; (start[0])++) if (nc_put_vara_float(ncid, varid, start, count, (const float *) data)) ERR_RET; /* Close up shop. */ if (nc_close(ncid)) ERR; free(data); } SUMMARIZE_ERR; #endif /* LARGE_FILE_TESTS */ printf("*** Creating a file with floats..."); { #define DIM_NAME "stupidity" #define NUMDIMS 1 #define DIMLEN 10000 int ncid, dimids[NUMDIMS], varid; char var_name[NC_MAX_NAME + 1] = {"Billy-Bob"}; int ndims, nvars, natts, unlimdimid; nc_type xtype; char name_in[NC_MAX_NAME + 1]; size_t len; float data[DIMLEN], data_in[DIMLEN]; int i; for (i = 0; i < DIMLEN; i++) data[i] = ((float)rand() / (float)(RAND_MAX)); /* Create a netCDF netCDF-4/HDF5 format file, with 1 var. */ /* if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;*/ if (nc_create(FILE_NAME, 0, &ncid)) ERR; if (nc_def_dim(ncid, DIM_NAME, DIMLEN, dimids)) ERR; if (nc_def_var(ncid, var_name, NC_FLOAT, NUMDIMS, dimids, &varid)) ERR; if (nc_enddef(ncid)) ERR; if (nc_put_var_float(ncid, varid, data)) ERR; if (nc_close(ncid)) ERR; /* Reopen and check the file. */ if (nc_open(FILE_NAME, 0, &ncid)) ERR; if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR; if (ndims != NUMDIMS || nvars != 1 || natts != 0 || unlimdimid != -1) ERR; if (nc_inq_dimids(ncid, &ndims, dimids, 1)) ERR; if (ndims != 1 || dimids[0] != 0) ERR; if (nc_inq_dim(ncid, 0, name_in, &len)) ERR; if (strcmp(name_in, DIM_NAME) || len != DIMLEN) ERR; if (nc_inq_var(ncid, 0, name_in, &xtype, &ndims, dimids, &natts)) ERR; if (strcmp(name_in, var_name) || xtype != NC_FLOAT || ndims != 1 || dimids[0] != 0 || natts != 0) ERR; if (nc_get_var_float(ncid, 0, data_in)) ERR; for (i = 0; i < DIMLEN; i++) if (data_in[i] != data[i]) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; printf("*** Creating files of various dimensions with various types..."); { #define TOTAL_SIZE 100000 #define MAX_DIMS 6 #define VAR_NAME "Unimaginatively_Named_Variable" #define MAX_TYPES 3 int ncid, dimids[MAX_DIMS], varid; char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1]; char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"}; int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT}; size_t len; float fdata[TOTAL_SIZE]; int idata[TOTAL_SIZE]; short sdata[TOTAL_SIZE]; void *data[MAX_TYPES]; int ndims; int i, d, t; /* Initialize the data to random floats. */ for (i = 0; i < TOTAL_SIZE; i++) { fdata[i] = (float)rand(); idata[i] = rand(); sdata[i] = (short)rand(); } data[0] = fdata; data[1] = idata; data[2] = sdata; /* Do the file creation process below for each type. */ for (t = 0; t < MAX_TYPES; t++) { /* Create MAX_DIMS files, each with different number of * dimensions. */ for (ndims = 1; ndims <= MAX_DIMS; ndims++) { sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims); if (nc_create(file_name, 0, &ncid)) ERR; for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++) { sprintf(dim_name, "dim_%d", d); if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR; } if (nc_def_var(ncid, VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR; if (nc_enddef(ncid)) ERR; if (nc_put_var(ncid, varid, data[t])) ERR; if (nc_close(ncid)) ERR; } /* next ndims. */ } /* next type */ } SUMMARIZE_ERR; printf("*** Creating file like in http://hdfeos.org/workshops/ws06/presentations/Pourmal/HDF5_IO_Perf.pdf..."); { #define XLEN 256 #define YLEN 256 #define ZLEN 1024 #define NDIMS 3 #define E_VAR_NAME "Like_Elenas_Benchmark" #define ELENA_FILE_NAME "tst_elena_int_3D.nc" #define E_TYPE_SIZE 4 int ncid, dimids[NDIMS], varid; int *idata; int i; /* Initialize data to random int between 0 and 255. */ if (!(idata = malloc(XLEN * YLEN * ZLEN * E_TYPE_SIZE))) return NC_ENOMEM; for (i = 0; i < XLEN * YLEN * ZLEN; i++) idata[i] = rand() % 255; /* Create a 3D file with one var. */ if (nc_create(ELENA_FILE_NAME, NC_CLOBBER, &ncid)) ERR; if (nc_def_dim(ncid, "z", ZLEN, &dimids[0])) ERR; if (nc_def_dim(ncid, "y", YLEN, &dimids[1])) ERR; if (nc_def_dim(ncid, "x", XLEN, &dimids[2])) ERR; if (nc_def_var(ncid, E_VAR_NAME, NC_INT, NDIMS, dimids, &varid)) ERR; if (nc_enddef(ncid)) ERR; /* Write the data. */ if (nc_put_var(ncid, varid, idata)) ERR; if (nc_close(ncid)) ERR; free(idata); } SUMMARIZE_ERR; printf("*** Creating super simple file to test non-sequential reads..."); { #define DIM_LEN 10 #define NDIMS1 1 #define S_VAR_NAME "My_Favorite_Numbers_in_order" #define S_FILE_NAME "tst_simple.nc" int ncid, dimids[NDIMS1], varid; int data[DIM_LEN]; int i; /* Initialize data to my favorite numbers. */ for (i = 0; i < DIM_LEN; i++) data[i] = i; /* Create a file with one var. */ if (nc_create(S_FILE_NAME, NC_CLOBBER, &ncid)) ERR; if (nc_def_dim(ncid, "a", DIM_LEN, &dimids[0])) ERR; if (nc_def_var(ncid, S_VAR_NAME, NC_INT, NDIMS1, dimids, &varid)) ERR; if (nc_enddef(ncid)) ERR; /* Write the data. */ if (nc_put_var(ncid, varid, data)) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; printf("*** Creating very simple 3D file..."); { #define SIMPLE_VAR_NAME "Paul_Mau_Dib" #define MAX_TYPES 3 int ncid, dimids[MAX_DIMS], varid; char dim_name[NC_MAX_NAME + 1], file_name[NC_MAX_NAME + 1]; char type_name[MAX_TYPES][MAX_TYPE_NAME + 1] = {"floats", "ints", "shorts"}; int typeid[MAX_TYPES] = {NC_FLOAT, NC_INT, NC_SHORT}; size_t len; float fdata[TOTAL_SIZE]; int idata[TOTAL_SIZE]; short sdata[TOTAL_SIZE]; void *data[MAX_TYPES]; int ndims; int i, d, t; /* Initialize the data to random floats. */ for (i = 0; i < TOTAL_SIZE; i++) { fdata[i] = (float)rand(); idata[i] = rand(); sdata[i] = (short)rand(); } data[0] = fdata; data[1] = idata; data[2] = sdata; /* Do the file creation process below for each type. */ for (t = 0; t < MAX_TYPES; t++) { /* Create MAX_DIMS files, each with different number of * dimensions. */ for (ndims = 1; ndims <= MAX_DIMS; ndims++) { sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims); if (nc_create(file_name, 0, &ncid)) ERR; for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++) { sprintf(dim_name, "dim_%d", d); if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR; } if (nc_def_var(ncid, SIMPLE_VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR; if (nc_enddef(ncid)) ERR; if (nc_put_var(ncid, varid, data[t])) ERR; if (nc_close(ncid)) ERR; } /* next ndims. */ } /* next type */ } SUMMARIZE_ERR; FINAL_RESULTS; }