netcdf-c/nc_test4/tst_chunks2.c

/* This is part of the netCDF package.
   Copyright 2018 University Corporation for Atmospheric Research/Unidata
   See COPYRIGHT file for conditions of use.

   Test netcdf-4 chunking.
*/

#include <nc_tests.h>
#include "err_macros.h"

#define FILE_NAME "tst_chunks2.nc"
#define MAX_WASTE 25.0
#define NUM_RANDOM_TESTS 3
#define NDIMS3 3

/* Calculate the waste of the chunking. A waste of 10% means the
 * chunked data is 10% larger then the unchunked data. */
static int
calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste)
{
   int d;
   float chunked = 1, unchunked = 1;
   size_t *num_chunks;
   size_t chunk_size = 1;

   assert(waste && dimlen && chunksize && ndims);
   if (!(num_chunks = calloc(ndims, sizeof(size_t)))) ERR;

#ifdef PRINT_CHUNK_WASTE_REPORT
   printf("\n");
#endif
   /* Caclulate the total space taken up by the chunked data. */
   for (d = 0; d < ndims; d++)
   {
      /* How many chunks along this dimension are required to hold all the data? */
      for (num_chunks[d] = 0; (num_chunks[d] * chunksize[d]) < (dimlen[d] ? dimlen[d] : 1);
	   num_chunks[d]++)
	 ;
      chunked *= (num_chunks[d] * chunksize[d]);
   }

   /* Calculate the minimum space required for this data
    * (i.e. unchunked) or one record of it. */
   for (d = 0; d < ndims; d++)
      unchunked *= (dimlen[d] ? dimlen[d] : 1);

#ifdef PRINT_CHUNK_WASTE_REPORT
   printf("size for unchunked %g elements; size for chunked %g elements\n",
	  unchunked, chunked);
#endif

   /* Percent of the chunked file that is wasted space. */
   *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0;

#ifdef PRINT_CHUNK_WASTE_REPORT
   printf("\ndimlen\tchunksize\tnum_chunks\n");
#endif
   for (d = 0; d < ndims; d++)
   {
#ifdef PRINT_CHUNK_WASTE_REPORT
      printf("%ld\t%ld\t\t%ld\n", (long int)dimlen[d], (long int)chunksize[d],
	     (long int)num_chunks[d]);
#endif
      chunk_size *= chunksize[d];
   }
#ifdef PRINT_CHUNK_WASTE_REPORT
   printf("size of chunk: %ld elements; wasted space: %2.2f percent\n",
	  (long int)chunk_size, *waste);
#endif

   free(num_chunks);
   return 0;
}

int
main(int argc, char **argv)
{
   printf("\n*** Testing netcdf-4 variable chunking.\n");
   printf("**** testing default chunksizes...");
   {
#define NDIMS3 3
#define NUM_VARS 1
#define Y_NAME "y"
#define X_NAME "x"
#define Z_NAME "z"
#define VAR_NAME_JOE "joe"
#define XDIM_LEN 2
#define YDIM_LEN 5
#define ZDIM_LEN 3000

      int varid, ncid, dims[NDIMS3], dims_in[NDIMS3];
      int ndims, nvars, ngatts, unlimdimid, natts;
      char name_in[NC_MAX_NAME + 1];
      nc_type type_in;
      size_t len_in[NDIMS3];
      int storage = 0;
      size_t chunksizes[NDIMS3];
      float waste = 0;

      /* Create a file with 3D var, turn on chunking, but don't provide chunksizes. */
      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
      if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
      if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
      if (nc_def_dim(ncid, Z_NAME, ZDIM_LEN, &dims[2])) ERR;
      if (nc_def_var(ncid, VAR_NAME_JOE, NC_FLOAT, NDIMS3, dims, &varid)) ERR;
      if (nc_def_var_chunking(ncid, 0, NC_CHUNKED, NULL)) ERR;

      /* Check it out. */
      if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
      if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR;
      if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
      if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 ||
	  dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR;
      if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR;
      if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR;
      if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR;
      if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR;
      if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR;
      if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR;
      if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (nc_close(ncid)) ERR;

      /* Open the file and check again. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
      if (nvars != NUM_VARS || ndims != NDIMS3 || ngatts != 0 || unlimdimid != -1) ERR;
      if (nc_inq_var(ncid, 0, name_in, &type_in, &ndims, dims_in, &natts)) ERR;
      if (strcmp(name_in, VAR_NAME_JOE) || type_in != NC_FLOAT || ndims != NDIMS3 ||
	  dims_in[0] != dims[0] || dims_in[1] != dims[1] || dims_in[2] != dims[2] || natts != 0) ERR;
      if (nc_inq_dim(ncid, 0, name_in, &len_in[0])) ERR;
      if (strcmp(name_in, X_NAME) || len_in[0] != XDIM_LEN) ERR;
      if (nc_inq_dim(ncid, 1, name_in, &len_in[1])) ERR;
      if (strcmp(name_in, Y_NAME) || len_in[1] != YDIM_LEN) ERR;
      if (nc_inq_dim(ncid, 2, name_in, &len_in[2])) ERR;
      if (strcmp(name_in, Z_NAME) || len_in[2] != ZDIM_LEN) ERR;
      if (nc_inq_var_chunking(ncid, 0, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, len_in, chunksizes, &waste)) ERR;
      /*if (waste > MAX_WASTE) ERR;*/
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes some more for a 3D var...");
   {
#define NDIMS3 3
#define VAR_NAME "op-amp"

      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3] = {1, 11, 152750};

      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

      /* Create a few dimensions. */
      for (d = 0; d < NDIMS3; d++)
      {
	 sprintf(dim_name, "dim_%d", d);
	 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
      }

      /* Define a var with these dimensions, and turn on chunking. */
      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
      if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

      /* Check how default chunking worked. */
      if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
/*      if (waste > MAX_WASTE) ERR;*/

      if (nc_close(ncid)) ERR;

      /* Open the file and check. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes even more for a 3D var...");
   {
      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777};

      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

      /* Create a few dimensions. */
      for (d = 0; d < NDIMS3; d++)
      {
	 sprintf(dim_name, "dim_%d", d);
	 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
      }

      /* Define a var with these dimensions, and turn on chunking. */
      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
      if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

      /* Check how default chunking worked. */
      if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
/*      if (waste > MAX_WASTE) ERR;*/

      if (nc_close(ncid)) ERR;

      /* Open the file and check. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes even even more for a 3D var...");
   {
      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3] = {1714636915, 1957747793, 424238335};

      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

      /* Create a few dimensions. */
      for (d = 0; d < NDIMS3; d++)
      {
	 sprintf(dim_name, "dim_%d", d);
	 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
      }

      /* Define a var with these dimensions, and turn on chunking. */
      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
      if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

      /* Check how default chunking worked. */
      if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
/*      if (waste > MAX_WASTE) ERR;*/

      if (nc_close(ncid)) ERR;

      /* Open the file and check. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes some more for a 3D var...");
   {
#define NDIMS3 3
#define VAR_NAME "op-amp"

      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3] = {1967513926, 1365180540, 426};

      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

      /* Create a few dimensions. */
      for (d = 0; d < NDIMS3; d++)
      {
	 sprintf(dim_name, "dim_%d", d);
	 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
      }

      /* Define a var with these dimensions, and turn on chunking. */
      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
      if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

      /* Check how default chunking worked. */
      if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
/*      if (waste > MAX_WASTE) ERR;*/

      if (nc_close(ncid)) ERR;

      /* Open the file and check. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes for very large 3D var...");
   {
#define NDIMS3 3

      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3] = {1804289383, 846930886, 1681692777};

      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

      /* Create a few dimensions. */
      for (d = 0; d < NDIMS3; d++)
      {
	 sprintf(dim_name, "dim_%d", d);
	 if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
      }

      /* Define a var with these dimensions, and turn on chunking. */
      if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
      if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

      /* Check how default chunking worked. */
      if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
      if (storage != NC_CHUNKED) ERR;
      if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
/*      if (waste > MAX_WASTE) ERR;*/

      if (nc_close(ncid)) ERR;

      /* Open the file and check. */
      if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
      if (nc_close(ncid)) ERR;
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes some randomly sized 3D vars...");
   {
#define NDIMS3 3

      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3];
      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d, t;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      for (t = 0; t < NUM_RANDOM_TESTS; t++)
      {
	 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

	 /* Create a few dimensions. */
	 for (d = 0; d < NDIMS3; d++)
	 {
	    dim_len[d] = rand();
	    sprintf(dim_name, "dim_%d", d);
	    if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
	 }

	 /* Define a var with these dimensions, and turn on chunking. */
	 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
	 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

	 /* Check how well default chunking worked. */
	 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
	 if (storage != NC_CHUNKED) ERR;
	 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
	 if (waste > MAX_WASTE) ERR;

	 if (nc_close(ncid)) ERR;
      }
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes some randomly sized 3D vars, with one small dimension...");
   {
      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3];
      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d, t;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      for (t = 0; t < NUM_RANDOM_TESTS; t++)
      {
	 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

	 dim_len[0] = rand();
	 dim_len[1] = rand();
	 dim_len[2] = rand() % 1000;
	 /* Create a few dimensions. */
	 for (d = 0; d < NDIMS3; d++)
	 {
	    sprintf(dim_name, "dim_%d", d);
	    if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
	 }

	 /* Define a var with these dimensions, and turn on chunking. */
	 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
	 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

	 /* Check how well default chunking worked. */
	 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
	 if (storage != NC_CHUNKED) ERR;
	 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
	 if (waste > MAX_WASTE) ERR;

	 if (nc_close(ncid)) ERR;
      }
   }
   SUMMARIZE_ERR;
   printf("**** testing default chunksizes some randomly sized 3D vars, with two small dimensions...");
   {
      int varid, ncid;
      int dimids[NDIMS3];
      size_t dim_len[NDIMS3];
      int storage = 0;
      size_t chunksizes[NDIMS3];
      int d, t;
      char dim_name[NC_MAX_NAME + 1];
      float waste;

      for (t = 0; t < NUM_RANDOM_TESTS; t++)
      {
	 if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;

	 dim_len[0] = rand();
	 dim_len[1] = rand() % 1000;
	 dim_len[2] = rand() % 1000;
	 /* Create a few dimensions. */
	 for (d = 0; d < NDIMS3; d++)
	 {
	    sprintf(dim_name, "dim_%d", d);
	    if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR;
	 }

	 /* Define a var with these dimensions, and turn on chunking. */
	 if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, NDIMS3, dimids, &varid)) ERR;
	 if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR;

	 /* Check how well default chunking worked. */
	 if (nc_inq_var_chunking(ncid, varid, &storage, chunksizes)) ERR;
	 if (storage != NC_CHUNKED) ERR;
	 if (calculate_waste(NDIMS3, dim_len, chunksizes, &waste)) ERR;
	 if (waste > MAX_WASTE) ERR;

	 if (nc_close(ncid)) ERR;
      }
   }
   SUMMARIZE_ERR;
   FINAL_RESULTS;
}