diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a7577778..31f6c3d08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,10 +310,10 @@ ENDIF() ################################ # HDF5 cache variables. -SET(DEFAULT_CHUNK_SIZE 4194304 CACHE STRING "Default Chunk Cache Size.") +SET(DEFAULT_CHUNK_SIZE 16777216 CACHE STRING "Default Chunk Cache Size.") SET(DEFAULT_CHUNKS_IN_CACHE 10 CACHE STRING "Default number of chunks in cache.") -SET(CHUNK_CACHE_SIZE 4194304 CACHE STRING "Default Chunk Cache Size.") -SET(CHUNK_CACHE_NELEMS 1009 CACHE STRING "Default maximum number of elements in cache.") +SET(CHUNK_CACHE_SIZE 16777216 CACHE STRING "Default Chunk Cache Size.") +SET(CHUNK_CACHE_NELEMS 4133 CACHE STRING "Default maximum number of elements in cache.") SET(CHUNK_CACHE_PREEMPTION 0.75 CACHE STRING "Default file chunk cache preemption policy for HDf5 files(a number between 0 and 1, inclusive.") SET(MAX_DEFAULT_CACHE_SIZE 67108864 CACHE STRING "Default maximum cache size.") SET(NETCDF_LIB_NAME "" CACHE STRING "Default name of the netcdf library.") diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ef48f90b1..3974a0fc1 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.4 - TBD +* [Enhancement] Increased default size of cache buffer to 16 MB, from 4 MB. Increased number of slots to 4133. See [https://github.com/Unidata/netcdf-c/issues/1541]. + ## 4.7.3 - November 20, 2019 * [Bug Fix]Fixed an issue where installs from tarballs will not properly compile in parallel environments. diff --git a/configure.ac b/configure.ac index 7df6343ae..4e3e76c8f 100644 --- a/configure.ac +++ b/configure.ac @@ -304,7 +304,7 @@ AC_MSG_CHECKING([whether a default file cache size for HDF5 was specified]) AC_ARG_WITH([chunk-cache-size], [AS_HELP_STRING([--with-chunk-cache-size=], [Specify default file cache chunk size for HDF5 files in bytes.])], - [CHUNK_CACHE_SIZE=$with_chunk_cache_size], [CHUNK_CACHE_SIZE=4194304]) + [CHUNK_CACHE_SIZE=$with_chunk_cache_size], [CHUNK_CACHE_SIZE=16777216]) AC_MSG_RESULT([$CHUNK_CACHE_SIZE]) AC_DEFINE_UNQUOTED([CHUNK_CACHE_SIZE], [$CHUNK_CACHE_SIZE], [default file chunk cache size in bytes.]) @@ -313,7 +313,7 @@ AC_MSG_CHECKING([whether a default file cache maximum number of elements for HDF AC_ARG_WITH([chunk-cache-nelems], [AS_HELP_STRING([--with-chunk-cache-nelems=], [Specify default maximum number of elements in the file chunk cache chunk for HDF5 files (should be prime number).])], - [CHUNK_CACHE_NELEMS=$with_chunk_cache_nelems], [CHUNK_CACHE_NELEMS=1009]) + [CHUNK_CACHE_NELEMS=$with_chunk_cache_nelems], [CHUNK_CACHE_NELEMS=4133]) AC_MSG_RESULT([$CHUNK_CACHE_NELEMS]) AC_DEFINE_UNQUOTED([CHUNK_CACHE_NELEMS], [$CHUNK_CACHE_NELEMS], [default file chunk cache nelems.]) diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index ef4980f2b..c54a131e8 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -803,6 +803,7 @@ INPUT = \ @abs_top_srcdir@/libsrc4/nc4dim.c \ @abs_top_srcdir@/libsrc4/nc4attr.c \ @abs_top_srcdir@/libhdf5/nc4info.c \ + @abs_top_srcdir@/libhdf5/hdf5cache.c \ @abs_top_srcdir@/libsrc4/nc4dispatch.c \ @abs_top_srcdir@/examples/C/simple_xy_wr.c \ @abs_top_srcdir@/examples/C/simple_xy_rd.c \ diff --git a/docs/guide.dox b/docs/guide.dox index 270d929fd..32d878742 100644 --- a/docs/guide.dox +++ b/docs/guide.dox @@ -1637,7 +1637,8 @@ and then verify them in HDF5. (And vice versa). When data are first read or written to a netCDF-4/HDF5 variable, the HDF5 library opens a cache for that variable. The default size of that -cache (settable with the –with-chunk-cache-size at netCDF build time). +cache is 16 MB (settable with the –with-chunk-cache-size at netCDF +build time). For good performance your chunk cache must be larger than one chunk of your data - preferably that it be large enough to hold multiple chunks diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 8029ad660..3b29b0d15 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -1129,6 +1129,12 @@ nc_free_string(size_t len, char **data) until the file is closed. Once re-opened, the variable chunk cache returns to its default value. + Current cache settings for each var may be obtained with + nc_get_var_chunk_cache(). + + Default values for these settings may be changed for the whole file + with nc_set_chunk_cache(). + @param ncid NetCDF or group ID, from a previous call to nc_open(), nc_create(), nc_def_grp(), or associated inquiry functions such as nc_inq_ncid(). @@ -1190,7 +1196,10 @@ nc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, } /** - Get the per-variable chunk cache settings from the HDF5 layer. + Get the per-variable chunk cache settings from the HDF5 + layer. These settings may be changed with nc_set_var_chunk_cache(). + + See nc_set_chunk_cache() for a full discussion of these settings. @param ncid NetCDF or group ID, from a previous call to nc_open(), nc_create(), nc_def_grp(), or associated inquiry functions such as diff --git a/libhdf5/hdf5cache.c b/libhdf5/hdf5cache.c index cbfd17e03..8175eef31 100644 --- a/libhdf5/hdf5cache.c +++ b/libhdf5/hdf5cache.c @@ -2,9 +2,10 @@ * Research. See COPYRIGHT file for copying and redistribution * conditions. */ /** - * @file @internal The netCDF-4 functions which control HDF5 - * caching. These caching controls allow the user to change the cache - * sizes of HDF5 before opening files. + * @file + * The netCDF-4 functions which control HDF5 caching. These caching + * controls allow the user to change the cache sizes of HDF5 before + * opening files. * * @author Ed Hartnett */ @@ -19,16 +20,53 @@ extern size_t nc4_chunk_cache_nelems; extern float nc4_chunk_cache_preemption; /** - * Set chunk cache size. Only affects files opened/created *after* it - * is called. + * Set chunk cache size. Only affects netCDF-4/HDF5 files + * opened/created *after* it is called. * - * @param size Size in bytes to set cache. - * @param nelems Number of elements to hold in cache. - * @param preemption Preemption stragety (between 0 and 1). + * The HDF5 chunk cache for each dataset is used by HDF5 when reading + * and writing files. The size of the chunk cache can be set with this + * function (for all variables in the file) or on a variable basis + * with nc_set_var_chunk_cache(). + * + * Increasing the size of the cache only helps if data access patterns + * support it. If data is read in one sequential pass through the + * file, then the cache will not help much. If data are read from the + * same file multiple times, the chunk cache can improve performance. + * + * The current settings for the file level chunk cache can be obtained + * with nc_get_chunk_cache(). + * + * For more information on HDF5 caching, see + * https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetCache. + * + * @param size Size in bytes to set cache. The default value is 64 MB; + * the default may be changed with configure option + * --with-chunk-cache-size. + * + * @param nelems Number of elements to hold in cache. This is passed + * to the nslots parameter of the HDF5 function H5Pset_cache(). This + * should be a prime number at least ten times larger than the maximum + * number of chunks that are set in the cache. The default value is + * 4133; the default may be set with configure option + * --with-chunk-cache-nelems. + * + * @param preemption Preemption stragety, a float between 0 and 1 + * inclusive and indicates the weighting according to which chunks + * which have been fully read or written are penalized when + * determining which chunks to flush from cache. A value of 0 means + * fully read or written chunks are treated no differently than other + * chunks (the preemption is strictly LRU) while a value of 1 means + * fully read or written chunks are always preempted before other + * chunks. If your application only reads or writes data once, this + * can be safely set to 1. Otherwise, this should be set lower + * depending on how often you re-read or re-write the same data. The + * default value is 0.75; the default may be set with configure option + * --with-chunk-cache-preemption. * * @return ::NC_NOERR No error. * @return ::NC_EINVAL Bad preemption. * @author Ed Hartnett + * @ingroup datasets */ int nc_set_chunk_cache(size_t size, size_t nelems, float preemption) @@ -42,15 +80,19 @@ nc_set_chunk_cache(size_t size, size_t nelems, float preemption) } /** - * Get chunk cache size. Only affects files opened/created *after* it - * is called. + * Get current chunk cache settings. These settings may be changed + * with nc_set_chunk_cache(). * - * @param sizep Pointer that gets size in bytes to set cache. - * @param nelemsp Pointer that gets number of elements to hold in cache. - * @param preemptionp Pointer that gets preemption stragety (between 0 and 1). + * @param sizep Pointer that gets size in bytes to set cache. Ignored + * if NULL. + * @param nelemsp Pointer that gets number of elements to hold in + * cache. Ignored if NULL. + * @param preemptionp Pointer that gets preemption stragety (between 0 + * and 1). Ignored if NULL. * * @return ::NC_NOERR No error. * @author Ed Hartnett + * @ingroup datasets */ int nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) @@ -67,8 +109,10 @@ nc_get_chunk_cache(size_t *sizep, size_t *nelemsp, float *preemptionp) } /** - * @internal Set the chunk cache. Required for fortran to avoid size_t - * issues. + * @internal Set the chunk cache. This is like nc_set_chunk_cache() + * but with integers instead of size_t, and with an integer preemption + * (which is the float preemtion * 100). This was required for fortran + * to avoid size_t issues. * * @param size Cache size. * @param nelems Number of elements. @@ -89,8 +133,10 @@ nc_set_chunk_cache_ints(int size, int nelems, int preemption) } /** - * @internal Get the chunk cache settings. Required for fortran to - * avoid size_t issues. + * @internal Get the chunk cache settings. This is like + * nc_get_chunk_cache() but with integers instead of size_t, and with + * an integer preemption (which is the float preemtion * 100). This + * was required for fortran to avoid size_t issues. * * @param sizep Pointer that gets cache size. * @param nelemsp Pointer that gets number of elements. diff --git a/nc_perf/Makefile.am b/nc_perf/Makefile.am index 1acbc7aa2..d4078038b 100644 --- a/nc_perf/Makefile.am +++ b/nc_perf/Makefile.am @@ -20,7 +20,7 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la check_PROGRAMS = tst_create_files bm_file tst_chunks3 tst_ar4 \ tst_ar4_3d tst_ar4_4d bm_many_objs tst_h_many_atts bm_many_atts \ tst_files2 tst_files3 tst_mem tst_knmi bm_netcdf4_recs tst_wrf_reads \ -tst_attsperf bigmeta openbigmeta +tst_attsperf bigmeta openbigmeta tst_bm_rando bm_file_SOURCES = bm_file.c tst_utils.c bm_netcdf4_recs_SOURCES = bm_netcdf4_recs.c tst_utils.c @@ -33,10 +33,11 @@ tst_ar4_SOURCES = tst_ar4.c tst_utils.c tst_h_many_atts_SOURCES = tst_h_many_atts.c tst_utils.c tst_knmi_SOURCES = tst_knmi.c tst_utils.c tst_wrf_reads_SOURCES = tst_wrf_reads.c tst_utils.c +tst_bm_rando_SOURCES = tst_bm_rando.c tst_utils.c TESTS = tst_ar4_3d tst_create_files tst_files3 tst_mem run_knmi_bm.sh \ tst_wrf_reads tst_attsperf perftest.sh run_tst_chunks.sh \ -run_bm_elena.sh +run_bm_elena.sh tst_bm_rando run_bm_elena.log: tst_create_files.log diff --git a/nc_perf/tst_bm_rando.c b/nc_perf/tst_bm_rando.c new file mode 100644 index 000000000..05dfd216b --- /dev/null +++ b/nc_perf/tst_bm_rando.c @@ -0,0 +1,146 @@ +/* + Copyright 2019, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. + + This program benchmarks random access to chunked data. + + Ed Hartnett 11/25/19 +*/ + +#include +#include +#include +#include /* Extra high precision time info. */ +#include +#include +#include +#include + +/* #define FILE_NAME "gfs.t00z.sfcf024.nc" */ +#define FILE_NAME "tst_bm_rando.nc" +#define MILLION 1000000 +#define NDIM3 3 + +/* These dim lengths taken from the current NOAA GFS surface data. */ +/* #define DIM1_LEN 3072 */ +/* #define DIM2_LEN 1536 */ +#define DIM1_LEN 512 +#define DIM2_LEN 512 + +#define DIM0_NAME "unlimited" +#define DIM1_NAME "x" +#define DIM2_NAME "y" +#define NUM_VAR 10 +#define NUM_REC 100 +#define NUM_TRY 10000 +#define NUM_CACHE_SETTING 3 + +/* Prototype from tst_utils.c. */ +int nc4_timeval_subtract(struct timeval *result, struct timeval *x, + struct timeval *y); +int +main(int argc, char **argv) +{ + printf("Benchmarking random access to file.\n"); + printf("Reading a file randomly...\n"); + { + int ncid; + int dimid[NDIM3]; + int varid[NUM_VAR]; + size_t chunksize[NDIM3] = {1, 512, 512}; + char name[NC_MAX_NAME + 1]; + size_t start[NDIM3] = {0, 0, 0}; + size_t count[NDIM3] = {1, DIM1_LEN, DIM2_LEN}; + float *data; + float *data_in; + size_t size[NUM_CACHE_SETTING] = {4194304, (4194304), (4194304)}; + /* size_t size[NUM_CACHE_SETTING] = {4194304, (4194304 * 4), (4194304 * 16)}; */ + size_t nelems[NUM_CACHE_SETTING] = {1009, 4133, 16141}; + /* size_t nelems[NUM_CACHE_SETTING] = {1009, 1009, 1009}; */ + float preemption[NUM_CACHE_SETTING] = {0.75f, 0.75f, 0.75f}; + + struct timeval start_time, end_time, diff_time; + float read_us; + + int v, d, t, c; + + /* Set random number seed. */ + srand(time(NULL)); + + /* Create the file. */ + if (nc_create(FILE_NAME, NC_CLOBBER|NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, DIM0_NAME, NC_UNLIMITED, &dimid[0])) ERR; + if (nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimid[1])) ERR; + if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimid[2])) ERR; + for (v = 0; v < NUM_VAR; v++) + { + sprintf(name, "var_%d", v); + if (nc_def_var(ncid, name, NC_FLOAT, NDIM3, dimid, &varid[v])) ERR; + if (nc_def_var_chunking(ncid, v, NC_CHUNKED, chunksize)) ERR; + } + if (!(data = malloc(DIM1_LEN * DIM2_LEN * sizeof(float)))) ERR; + for (d = 0; d < DIM1_LEN * DIM2_LEN; d++) + data[d] = d + 1.0f/(float)(rand()%1000); + for (v = 0; v < NUM_VAR; v++) + { + for (start[0] = 0; start[0] < NUM_REC; start[0]++) + { + if (nc_put_vara_float(ncid, v, start, count, data)) ERR; + } + } + if (nc_close(ncid)) ERR; + free(data); + if (!(data_in = malloc(DIM1_LEN * DIM2_LEN * sizeof(float)))) ERR; + + /* nc_set_log_level(3); */ + printf("size\tnelems\tpreemption\tread time(s)\n"); + for (c = 0; c < NUM_CACHE_SETTING; c++) + { + char cmd[NC_MAX_NAME * 2 + 20]; + char file_2[NC_MAX_NAME + 1]; + + if (c) + { + /* Create a copy of file_out. This will defeat any OS + * buffering. */ + sprintf(file_2, "tst_copy_%d_%s", c, FILE_NAME); + sprintf(cmd, "cp %s %s\n", FILE_NAME, file_2); + system(cmd); + } + else + strcpy(file_2, FILE_NAME); + + /* Start timer. */ + if (gettimeofday(&start_time, NULL)) ERR; + + /* Set cache settings. */ + if (nc_set_chunk_cache(size[c], nelems[c], preemption[c])) ERR; + + /* Open the file. */ + if (nc_open(file_2, NC_NOWRITE, &ncid)) ERR; + + /* Read a random record of a random var. */ + for (t = 0; t < NUM_TRY; t++) + { + int var = rand() % NUM_VAR; + start[0] = rand() % NUM_REC; + if (nc_get_vara_float(ncid, var, start, count, data_in)) ERR; + } + + /* Close the file. */ + if (nc_close(ncid)) ERR; + + /* Stop timer. */ + if (gettimeofday(&end_time, NULL)) ERR; + if (nc4_timeval_subtract(&diff_time, &end_time, &start_time)) ERR; + read_us = (int)diff_time.tv_sec + (float)diff_time.tv_usec / MILLION ; + printf("%zu, %ld, %g, %g\n", size[c], nelems[c], preemption[c], read_us); + } + /* Free data storage. */ + free(data_in); + + + } + SUMMARIZE_ERR; + FINAL_RESULTS; +}