From a2e0ef4ea6b8e60ce89cb75eb69bdcbd153dd62a Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 15 Jun 2018 14:23:09 -0500 Subject: [PATCH 01/28] Checking large-sized variables has been moved to NC_enddef() as explained in below comments at lines 491-495. This is most likely caused by git merge failed to delete residue codes. --- libsrc/var.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/libsrc/var.c b/libsrc/var.c index 0e4da4d69..4c26be208 100644 --- a/libsrc/var.c +++ b/libsrc/var.c @@ -484,9 +484,6 @@ NC_var_shape(NC_var *varp, const NC_dimarray *dims) out : - /* No variable size can be > X_INT64_MAX - 3 */ - if (0 == NC_check_vlen(varp, (size_t)X_INT64_MAX-3)) return NC_EVARSIZE; - /* * For CDF-1 and CDF-2 formats, the total number of array elements * cannot exceed 2^32, unless this variable is the last fixed-size From ab2dbc0d84a74f1c19097ecf8703c9a3674eec98 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 15 Jun 2018 15:08:05 -0500 Subject: [PATCH 02/28] Promote NC_var member len from size_t to long long to fix some problems of large-variable tests on 320bit platforms. --- include/nc3internal.h | 6 +++--- libsrc/nc3internal.c | 2 +- libsrc/putget.m4 | 4 ++-- libsrc/v1hpg.c | 4 +++- libsrc/var.c | 7 +++---- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/include/nc3internal.h b/include/nc3internal.h index c41264d20..d5bcd7ad3 100644 --- a/include/nc3internal.h +++ b/include/nc3internal.h @@ -169,7 +169,7 @@ typedef struct NC_var { int *dimids; /* assoc->value */ NC_attrarray attrs; nc_type type; /* the discriminant */ - size_t len; /* the total length originally allocated */ + long long len; /* the total length originally allocated */ off_t begin; /* end xdr */ int no_fill; /* whether fill mode is ON or OFF */ @@ -212,7 +212,7 @@ extern int NC_findvar(const NC_vararray *ncap, const char *name, NC_var **varpp); extern int -NC_check_vlen(NC_var *varp, unsigned long long vlen_max); +NC_check_vlen(NC_var *varp, long long vlen_max); extern int NC_lookupvar(NC3_INFO* ncp, int varid, NC_var **varp); @@ -363,7 +363,7 @@ nc_get_NC(NC3_INFO* ncp); /* Begin defined in putget.c */ extern int -fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno); +fill_NC_var(NC3_INFO* ncp, const NC_var *varp, long long varsize, size_t recno); extern int nc_inq_rec(int ncid, size_t *nrecvars, int *recvarids, size_t *recsizes); diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index fb99798d9..95ef04533 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -713,7 +713,7 @@ NC_check_vlens(NC3_INFO *ncp) /* maximum permitted variable size (or size of one record's worth of a record variable) in bytes. This is different for format 1 and format 2. */ - unsigned long long vlen_max; + long long vlen_max; size_t ii; size_t large_vars_count; size_t rec_vars_count; diff --git a/libsrc/putget.m4 b/libsrc/putget.m4 index 192aeb3b6..baf49e3f5 100644 --- a/libsrc/putget.m4 +++ b/libsrc/putget.m4 @@ -147,7 +147,7 @@ NCFILL(ulonglong, ulonglong, X_SIZEOF_ULONGLONG, NC_FILL_UINT64) xdr_NC_fill() */ int -fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno) +fill_NC_var(NC3_INFO* ncp, const NC_var *varp, long long varsize, size_t recno) { char xfillp[NFILL * X_SIZEOF_DOUBLE]; const size_t step = varp->xsz; @@ -155,7 +155,7 @@ fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno) const size_t xsz = varp->xsz * nelems; NC_attr **attrpp = NULL; off_t offset; - size_t remaining = varsize; + long long remaining = varsize; void *xp; int status = NC_NOERR; diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index 128613851..ee85fe9aa 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1050,7 +1050,9 @@ v1h_get_NC_var(v1hs *gsp, NC_var **varpp) if(status != NC_NOERR) goto unwind_alloc; - status = v1h_get_size_t(gsp, &varp->len); + size_t tmp; + status = v1h_get_size_t(gsp, &tmp); + varp->len = tmp; if(status != NC_NOERR) goto unwind_alloc; diff --git a/libsrc/var.c b/libsrc/var.c index 4c26be208..5b537dcba 100644 --- a/libsrc/var.c +++ b/libsrc/var.c @@ -510,16 +510,15 @@ out : * systems with LFS it should be 2^32 - 4. */ int -NC_check_vlen(NC_var *varp, unsigned long long vlen_max) { - unsigned long long prod=varp->xsz; /* product of xsz and dimensions so far */ - +NC_check_vlen(NC_var *varp, long long vlen_max) { int ii; + long long prod=varp->xsz; /* product of xsz and dimensions so far */ assert(varp != NULL); for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) { if(!varp->shape) return 0; /* Shape is undefined/NULL. */ - if (varp->shape[ii] > (size_t)(vlen_max / prod)) { + if (varp->shape[ii] > vlen_max / prod) { return 0; /* size in bytes won't fit in a 32-bit int */ } prod *= varp->shape[ii]; From 547b7405e7193e7d7e584d5372b7a8160e5e11cd Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 15 Jun 2018 17:30:30 -0500 Subject: [PATCH 03/28] miss another typecast --- libsrc/nc3internal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index 95ef04533..9015e8f97 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -723,7 +723,7 @@ NC_check_vlens(NC3_INFO *ncp) return NC_NOERR; if (fIsSet(ncp->flags,NC_64BIT_DATA)) /* CDF-5 */ - vlen_max = (size_t)X_INT64_MAX - 3; /* "- 3" handles rounded-up size */ + vlen_max = X_INT64_MAX - 3; /* "- 3" handles rounded-up size */ else if (fIsSet(ncp->flags,NC_64BIT_OFFSET) && sizeof(off_t) > 4) /* CDF2 format and LFS */ vlen_max = X_UINT_MAX - 3; /* "- 3" handles rounded-up size */ From 1b6f579aa4ee6b151d7050dea85db549036c7b7d Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sun, 17 Jun 2018 13:16:09 -0500 Subject: [PATCH 04/28] another typecast missed --- libsrc/var.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libsrc/var.c b/libsrc/var.c index 5b537dcba..9254894ed 100644 --- a/libsrc/var.c +++ b/libsrc/var.c @@ -518,7 +518,7 @@ NC_check_vlen(NC_var *varp, long long vlen_max) { for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) { if(!varp->shape) return 0; /* Shape is undefined/NULL. */ - if (varp->shape[ii] > vlen_max / prod) { + if ((long long)varp->shape[ii] > vlen_max / prod) { return 0; /* size in bytes won't fit in a 32-bit int */ } prod *= varp->shape[ii]; From f95d3e33255eaeed84b22718d894fcf5b988669c Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 29 Jun 2018 21:17:07 -0500 Subject: [PATCH 05/28] replace USE_CDF5 with ENABLE_CDF5 --- config.h.cmake.in | 1 - configure.ac | 3 --- libdispatch/dfile.c | 4 ++-- libsrc/nc3internal.c | 6 +++--- nc_test/Makefile.am | 2 +- nc_test/nc_test.c | 2 +- nc_test/t_nc.c | 2 +- nc_test/test_write.m4 | 2 +- nc_test/tst_def_var_fill.c | 2 +- nc_test/tst_global_fillval.c | 4 ++-- nc_test/tst_names.c | 2 +- nc_test/tst_small.c | 4 ++-- ncdump/nccopy.c | 2 +- ncdump/run_ncgen_tests.sh | 4 ++-- ncdump/tst_64bit.sh | 2 +- ncdump/tst_dimsizes.c | 6 +++--- ncdump/tst_formatx3.sh | 10 +++++----- ncdump/tst_lengths.sh | 2 +- ncdump/tst_nccopy3.sh | 2 +- ncgen/main.c | 2 +- 20 files changed, 30 insertions(+), 34 deletions(-) diff --git a/config.h.cmake.in b/config.h.cmake.in index 611ceb7a0..2710361df 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -109,7 +109,6 @@ are set when opening a binary file on Windows. */ /* if true, enable CDF5 Support */ #cmakedefine ENABLE_CDF5 1 -#cmakedefine USE_CDF5 1 /* if true, enable strict null byte header padding. */ #cmakedefine USE_STRICT_NULL_BYTE_HEADER_PADDING 1 diff --git a/configure.ac b/configure.ac index 5864395c4..2bf1b5ac3 100644 --- a/configure.ac +++ b/configure.ac @@ -906,11 +906,8 @@ fi AC_MSG_RESULT($enable_cdf5) if test "x${enable_cdf5}" = xyes; then - AC_DEFINE([USE_CDF5], [1], [if true, enable CDF5 Support]) AC_DEFINE([ENABLE_CDF5], [1], [if true, enable CDF5 Support]) fi - -AM_CONDITIONAL(USE_CDF5, [test x$enable_cdf5 = xyes ]) AM_CONDITIONAL(ENABLE_CDF5, [test x$enable_cdf5 = xyes ]) $SLEEPCMD diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 99c926e5c..98d1681bc 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -2106,7 +2106,7 @@ NC_create(const char *path0, int cmode, size_t initialsz, model = NC_FORMATX_NC4; break; #endif -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 case NC_FORMAT_CDF5: xcmode |= NC_64BIT_DATA; model = NC_FORMATX_NC3; @@ -2322,7 +2322,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp, hdf4built = 1; #endif #endif -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 cdf5built = 1; #endif if(!hdf5built && model == NC_FORMATX_NC4) { diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index fb99798d9..102e2a3ca 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -100,7 +100,7 @@ err: int nc3_cktype(int mode, nc_type type) { -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 if (mode & NC_CDF5) { /* CDF-5 format */ if (type >= NC_BYTE && type < NC_STRING) return NC_NOERR; } else @@ -1167,7 +1167,7 @@ nc_set_default_format(int format, int *old_formatp) return NC_EINVAL; #else if (format != NC_FORMAT_CLASSIC && format != NC_FORMAT_64BIT_OFFSET -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 && format != NC_FORMAT_CDF5 #endif ) @@ -1698,7 +1698,7 @@ NC3_inq_format(int ncid, int *formatp) return NC_NOERR; /* only need to check for netCDF-3 variants, since this is never called for netCDF-4 files */ -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 if (fIsSet(nc3->flags, NC_64BIT_DATA)) *formatp = NC_FORMAT_CDF5; else diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index e92a04dc0..841995c30 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -27,7 +27,7 @@ endif if TEST_PARALLEL4 if USE_PNETCDF -if USE_CDF5 +if ENABLE_CDF5 TESTPROGRAMS += tst_cdf5format endif endif diff --git a/nc_test/nc_test.c b/nc_test/nc_test.c index 5800c339e..9e5490e82 100644 --- a/nc_test/nc_test.c +++ b/nc_test/nc_test.c @@ -134,7 +134,7 @@ main(int argc, char *argv[]) break; case NC_FORMAT_CDF5: -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 nc_set_default_format(NC_FORMAT_CDF5, NULL); fprintf(stderr, "\n\nSwitching to 64-bit data format.\n"); strcpy(testfile, "nc_test_cdf5.nc"); diff --git a/nc_test/t_nc.c b/nc_test/t_nc.c index ec3be5e0e..afddd61d5 100644 --- a/nc_test/t_nc.c +++ b/nc_test/t_nc.c @@ -361,7 +361,7 @@ main(int argc, char *argv[]) MPI_Init(&argc, &argv); cmode |= (NC_PNETCDF); -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 cmode |= (NC_64BIT_DATA); #endif ret = nc_create_par(fname,cmode, MPI_COMM_WORLD, MPI_INFO_NULL, &id); diff --git a/nc_test/test_write.m4 b/nc_test/test_write.m4 index edc303aed..4650e3706 100644 --- a/nc_test/test_write.m4 +++ b/nc_test/test_write.m4 @@ -2424,7 +2424,7 @@ APIFunc(get_file_version)(char *path, int *version) if (strncmp(magic, "CDF", MAGIC_NUM_LEN-1)==0) { if (magic[MAGIC_NUM_LEN-1] == NC_FORMAT_CLASSIC || magic[MAGIC_NUM_LEN-1] == NC_FORMAT_64BIT_OFFSET -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 || magic[MAGIC_NUM_LEN-1] == NC_FORMAT_CDF5 #endif ) diff --git a/nc_test/tst_def_var_fill.c b/nc_test/tst_def_var_fill.c index 96e7f9145..affd3156e 100644 --- a/nc_test/tst_def_var_fill.c +++ b/nc_test/tst_def_var_fill.c @@ -54,7 +54,7 @@ int main(int argc, char** argv) { buf = (int*) malloc(NY*NX * sizeof(int)); for (k=0; k<5; k++) { -#ifndef USE_CDF5 +#ifndef ENABLE_CDF5 if (formats[k] == NC_FORMAT_CDF5) continue; #endif #ifndef USE_NETCDF4 diff --git a/nc_test/tst_global_fillval.c b/nc_test/tst_global_fillval.c index 5058eb7e4..4812d3f72 100644 --- a/nc_test/tst_global_fillval.c +++ b/nc_test/tst_global_fillval.c @@ -37,7 +37,7 @@ main(int argc, char **argv) num_formats += 2; #endif -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 num_formats++; #endif @@ -46,7 +46,7 @@ main(int argc, char **argv) formats[n++] = 0; formats[n++] = NC_64BIT_OFFSET; -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 formats[n++] = NC_64BIT_DATA; #endif #ifdef USE_NETCDF4 diff --git a/nc_test/tst_names.c b/nc_test/tst_names.c index 8127f1b56..5f63c43e4 100644 --- a/nc_test/tst_names.c +++ b/nc_test/tst_names.c @@ -225,7 +225,7 @@ main(int argc, char **argv) NC_FORMAT_CLASSIC , NC_FORMAT_64BIT_OFFSET -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 , NC_FORMAT_CDF5 #endif diff --git a/nc_test/tst_small.c b/nc_test/tst_small.c index 114218f44..60ef9a5c0 100644 --- a/nc_test/tst_small.c +++ b/nc_test/tst_small.c @@ -55,7 +55,7 @@ static int file_create(const char *filename, int cmode, int *ncid) #ifdef USE_PNETCDF if (default_format == NC_FORMAT_CLASSIC || default_format == NC_FORMAT_64BIT_OFFSET -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 || default_format == NC_FORMAT_64BIT_DATA #endif ) @@ -525,7 +525,7 @@ main(int argc, char **argv) printf("Switching to 64-bit offset format.\n"); strcpy(testfile, "tst_small_64bit.nc"); break; -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 case NC_FORMAT_CDF5: nc_set_default_format(NC_FORMAT_CDF5, NULL); printf("Switching to 64-bit data format.\n"); diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index f5f033a3a..5a8901ec8 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -1706,7 +1706,7 @@ copy(char* infile, char* outfile) create_mode |= NC_64BIT_OFFSET; break; case NC_FORMAT_CDF5: -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 create_mode |= NC_64BIT_DATA; break; #else diff --git a/ncdump/run_ncgen_tests.sh b/ncdump/run_ncgen_tests.sh index eb516e12d..752072ae3 100755 --- a/ncdump/run_ncgen_tests.sh +++ b/ncdump/run_ncgen_tests.sh @@ -11,7 +11,7 @@ set -e # This shell script runs the ncdump tests. # get some config.h parameters if test -f ${top_builddir}/config.h ; then - if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then + if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then CDF5=1 else CDF5=0 @@ -51,7 +51,7 @@ echo "*** creating 64-bit offset file c0_64_run_ncgen_tests.nc from c0.cdl..." validateNC c0 "c0_64" -k 64-bit-offset -b -if test "x$USE_CDF5" = x1 ; then +if test "x$ENABLE_CDF5" = x1 ; then echo "*** creating 64-bit data file c5.nc from c5.cdl..." ${NCGEN} -k 64-bit-data -b -o tst_c5_run_ncgen_tests.nc $top_srcdir/ncgen/c5.cdl diff --git a/ncdump/tst_64bit.sh b/ncdump/tst_64bit.sh index 838dabfcd..861b01998 100755 --- a/ncdump/tst_64bit.sh +++ b/ncdump/tst_64bit.sh @@ -8,7 +8,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi # This shell script runs the ncdump tests. # get some config.h parameters if test -f ${top_builddir}/config.h ; then - if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then + if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then CDF5=1 else CDF5=0 diff --git a/ncdump/tst_dimsizes.c b/ncdump/tst_dimsizes.c index edb0eb80d..376a199d8 100644 --- a/ncdump/tst_dimsizes.c +++ b/ncdump/tst_dimsizes.c @@ -12,7 +12,7 @@ #define DIMMAXCLASSIC (NC_MAX_INT - 3) #define DIMMAX64OFFSET (NC_MAX_UINT - 3) -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 #define DIMMAX64DATA (NC_MAX_UINT64 - 3) #endif @@ -66,7 +66,7 @@ main(int argc, char **argv) if(dimsize != DIMMAX64OFFSET) ERR; if ((stat=nc_close(ncid))) ERRSTAT(stat); -#ifdef USE_CDF5 +#ifdef ENABLE_CDF5 if(sizeof(size_t) == 8) { printf("\n*** Writing Max Dimension Size (%llu) For NC_64BIT_DATA\n",DIMMAX64DATA); if ((stat=nc_create(FILE64DATA, NC_CLOBBER | NC_64BIT_DATA, &ncid))) ERRSTAT(stat); @@ -81,7 +81,7 @@ main(int argc, char **argv) if(dimsize != DIMMAX64DATA) ERR; if ((stat=nc_close(ncid))) ERRSTAT(stat); } -#endif /* USE_CDF5 */ +#endif /* ENABLE_CDF5 */ SUMMARIZE_ERR; FINAL_RESULTS; diff --git a/ncdump/tst_formatx3.sh b/ncdump/tst_formatx3.sh index 548b56bf9..ea395e400 100755 --- a/ncdump/tst_formatx3.sh +++ b/ncdump/tst_formatx3.sh @@ -6,10 +6,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi # This shell script runs the ncdump tests. # get some config.h parameters if test -f ${top_builddir}/config.h ; then - if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then - USE_CDF5=1 + if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then + ENABLE_CDF5=1 else - USE_CDF5=0 + ENABLE_CDF5=0 fi else echo "Cannot locate config.h" @@ -42,9 +42,9 @@ ECODE=1 fi -# Only do following test if USE_CDF5 is true. +# Only do following test if ENABLE_CDF5 is true. -if test "x$USE_CDF5" = x1 ; then +if test "x$ENABLE_CDF5" = x1 ; then echo "Test extended format output for a 64-bit CDF-5 classic file" rm -f tmp_tst_formatx3 ${NCGEN} -k5 -b -o ./tst_formatx3.nc $srcdir/ref_tst_small.cdl diff --git a/ncdump/tst_lengths.sh b/ncdump/tst_lengths.sh index dbe5c6810..a21f6a404 100755 --- a/ncdump/tst_lengths.sh +++ b/ncdump/tst_lengths.sh @@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi # get some config.h parameters if test -f ${top_builddir}/config.h ; then - if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then + if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then HAVE_CDF5=1 else HAVE_CDF5=0 diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 1b023e549..449e89686 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -13,7 +13,7 @@ echo "" # get some config.h parameters if test -f ${top_builddir}/config.h ; then - if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then + if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then HAVE_CDF5=1 else HAVE_CDF5=0 diff --git a/ncgen/main.c b/ncgen/main.c index 45c1336bd..8af6909cd 100644 --- a/ncgen/main.c +++ b/ncgen/main.c @@ -512,7 +512,7 @@ main( /* Compute the k_flag (1st pass) using rules in the man page (ncgen.1).*/ -#ifndef USE_CDF5 +#ifndef ENABLE_CDF5 if(k_flag == NC_FORMAT_CDF5) { derror("Output format CDF5 requested, but netcdf was built without cdf5 support."); return 0; From 26f1d9137c97183dab6007cdb6c400d62f0f860a Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sun, 1 Jul 2018 19:42:03 -0500 Subject: [PATCH 06/28] revise #1033 and emulate CMAKE OPTION to use 3rd option AUTO --- CMakeLists.txt | 26 ++++++++++++++++++++------ configure.ac | 16 +++++++++------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc2f72ef3..3ff74c2b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -767,12 +767,6 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4) ENDIF(USE_HDF5 OR ENABLE_NETCDF_4) -# Option to turn on CDF5 support. -OPTION(ENABLE_CDF5 "Enable CDF5 Support." ON) -IF(ENABLE_CDF5) - SET(USE_CDF5 ON CACHE BOOL "") -ENDIF(ENABLE_CDF5) - # Option to Build DAP2+DAP4 Clients OPTION(ENABLE_DAP "Enable DAP2 and DAP4 Client." ON) IF(ENABLE_DAP) @@ -1293,6 +1287,26 @@ CHECK_TYPE_SIZE("_Bool" SIZEOF__BOOL) CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T) +# Check whether to turn on or off CDF5 support. +SET(ENABLE_CDF5 AUTO CACHE STRING "AUTO") +OPTION(ENABLE_CDF5 "Enable CDF5 support" ON) +IF(SIZEOF_SIZE_T EQUAL 4) + IF(ENABLE_CDF5) # enable or auto + STRING(TOUPPER ${ENABLE_CDF5} ENABLE_CDF5) + IF(ENABLE_CDF5 AND NOT ENABLE_CDF5 STREQUAL "AUTO") # explicitly enabled + MESSAGE(FATAL_ERROR "Unable to support CDF5 feature because size_t is less than 8 bytes") + ENDIF(ENABLE_CDF5 AND NOT ENABLE_CDF5 STREQUAL "AUTO") + SET(ENABLE_CDF5 OFF) # cannot support CDF5 + SET(USE_CDF5 OFF CACHE BOOL "") # cannot support CDF5 + ENDIF(ENABLE_CDF5) +ELSE(SIZEOF_SIZE_T EQUAL 4) + IF(ENABLE_CDF5) # explicitly set by user or not set + SET(USE_CDF5 ON CACHE BOOL "") + ELSE(ENABLE_CDF5) # explicitly disabled by user + SET(USE_CDF5 OFF CACHE BOOL "") + ENDIF(ENABLE_CDF5) +ENDIF(SIZEOF_SIZE_T EQUAL 4) + CHECK_TYPE_SIZE("ssize_t" SIZEOF_SSIZE_T) IF(SIZEOF_SSIZE_T) SET(HAVE_SSIZE_T TRUE) diff --git a/configure.ac b/configure.ac index 766eff21f..2bf1b5ac3 100644 --- a/configure.ac +++ b/configure.ac @@ -886,20 +886,22 @@ $SLEEPCMD AC_CHECK_SIZEOF(unsigned long long) # Check whether we want to enable CDF5 support. -AC_MSG_CHECKING([whether CDF5 support should be enabled]) +AC_MSG_CHECKING([whether CDF5 support should be disabled]) AC_ARG_ENABLE([cdf5], - [AS_HELP_STRING([--enable-cdf5], + [AS_HELP_STRING([--disable-cdf5], [build without CDF5 support.])], [enable_cdf5=${enableval}], [enable_cdf5=auto] ) -if test "x${enable_cdf5}" = xyes && test "$ac_cv_sizeof_size_t" -lt "8" ; then - dnl unable to support CDF5, but --enable-cdf5 is explicitly set - AC_MSG_ERROR([Unable to support CDF5 feature because size_t is less than 4 bytes]) -fi if test "$ac_cv_sizeof_size_t" -lt "8" ; then + if test "x${enable_cdf5}" = xyes ; then + dnl unable to support CDF5, but --enable-cdf5 is explicitly set + AC_MSG_ERROR([Unable to support CDF5 feature because size_t is less than 8 bytes]) + fi enable_cdf5=no else - enable_cdf5=yes + if test "x${enable_cdf5}" != xno ; then + enable_cdf5=yes + fi fi AC_MSG_RESULT($enable_cdf5) From 577fcea5069c8504266b50168121bb0ce0dbdf8e Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Thu, 5 Jul 2018 22:54:41 -0500 Subject: [PATCH 07/28] add missing note about CDF-5 format --- docs/FAQ.md | 147 ++++++++++++++++---------- docs/all-error-codes.md | 159 +++++++++++++++++++++------- docs/guide.dox | 224 +++++++++++++++++++++++----------------- docs/tutorial.dox | 29 ++++-- docs/types.dox | 16 +-- include/ncdispatch.h | 2 +- libdispatch/dfile.c | 28 +++-- 7 files changed, 384 insertions(+), 221 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 98d309199..80db23806 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -547,6 +547,7 @@ There are four netCDF format variants: - the classic format - the 64-bit offset format +- the 64-bit data format - the netCDF-4 format - the netCDF-4 classic model format @@ -556,13 +557,13 @@ though these are not usually thought of as formats: CDL and NcML.) The **classic format** was the only format for netCDF data created between 1989 and 2004 by the reference software from Unidata. It is still the default format for new netCDF data files, and the form in -which most netCDF data is stored. +which most netCDF data is stored. This format is also referred as CDF-1 format. In 2004, the **64-bit offset format** variant was added. Nearly identical to netCDF classic format, it allows users to create and access far larger datasets than were possible with the original format. (A 64-bit platform is not required to write or read 64-bit offset netCDF -files.) +files.) This format is also referred as CDF-2 format. In 2008, the **netCDF-4 format** was added to support per-variable compression, multiple unlimited dimensions, more complex data types, and @@ -574,6 +575,14 @@ format**, was added for users who needed the performance benefits of the new format (such as compression) without the complexity of a new programming interface or enhanced data model. +In 2016, the **64-bit data format** variant was added. To support large +variables with more than 4-billion array elements, it replaces most of the +32-bit integers used in the format specification with 64-bit integers. It also +adds support for several new data types including unsigned byte, unsigned +short, unsigned int, signed 64-bit int and unsigned 64-bit int. A 64-bit +platform is required to write or read 64-bit data netCDF files. This format is +also referred as CDF-5 format. + With each additional format variant, the C-based reference software from Unidata has continued to support access to data stored in previous formats transparently, and to also support programs written using @@ -581,15 +590,18 @@ previous programming interfaces. Although strictly speaking, there is no single "netCDF-3 format", that phrase is sometimes used instead of the more cumbersome but correct -"netCDF classic or 64-bit offset format" to describe files created by -the netCDF-3 (or netCDF-1 or netCDF-2) libraries. Similarly "netCDF-4 -format" is sometimes used informally to mean "either the general -netCDF-4 format or the restricted netCDF-4 classic model format". We -will use these shorter phrases in FAQs below when no confusion is -likely. +"netCDF classic CDF-1, 64-bit offset CDF-2, or 64-bit data CDF-5 format" to +describe files created by the netCDF-3 (or netCDF-1 or netCDF-2) libraries. +Similarly "netCDF-4 format" is sometimes used informally to mean "either the +general netCDF-4 format or the restricted netCDF-4 classic model format". We +will use these shorter phrases in FAQs below when no confusion is likely. -A more extensive description of the netCDF formats and a formal -specification of the classic and 64-bit formats is available as a [NASA ESDS community standard](https://earthdata.nasa.gov/sites/default/files/esdswg/spg/rfc/esds-rfc-011/ESDS-RFC-011v2.00.pdf). +A more extensive description of the netCDF formats and a formal specification +of the classic and 64-bit formats is available as a [NASA ESDS community +standard](https://earthdata.nasa.gov/sites/default/files/esdswg/spg/rfc/esds-rfc-011/ESDS-RFC-011v2.00.pdf). + +The 64-bit data CDF-5 format specification is available in +http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html. How can I tell which format a netCDF file uses? {#How-can-I-tell-which-format-a-netCDF-file-uses} ----------------- @@ -598,8 +610,9 @@ How can I tell which format a netCDF file uses? {#How-can-I-tell-which-format-a- The short answer is that under most circumstances, you should not care, if you use version 4.0 or later of the netCDF library to access data in the file. But the difference is indicated in the first four bytes of the -file, which are 'C', 'D', 'F', '\\001' for the classic netCDF format; -'C', 'D', 'F', '\\002' for the 64-bit offset format; or '\\211', 'H', +file, which are 'C', 'D', 'F', '\\001' for the classic netCDF CDF-1 format; +'C', 'D', 'F', '\\002' for the 64-bit offset CDF-2 format; +'C', 'D', 'F', '\\005' for the 64-bit data CDF-5 format; or '\\211', 'H', 'D', 'F' for an HDF5 file, which could be either a netCDF-4 file or a netCDF-4 classic model file. (HDF5 files may also begin with a user-block of 512, 1024, 2048, ... bytes before what is actually an @@ -635,22 +648,37 @@ which will output C D F 002 ~~~~ +~~~~ {.boldcode} + C D F 005 +~~~~ + ~~~~ {.boldcode} 211 H D F ~~~~ -depending on whether foo.nc is a classic, 64-bit offset, or netCDF-4 -file, respectively. This method cannot be used to distinguish between -netCDF-4 and netCDF-4 classic model variants, or between a netCDF-4 file -and a different kind of HDF5 file. +depending on whether foo.nc is a classic CDF-1, 64-bit offset CDF-2, 64-bit +data CDF-5, or netCDF-4 file, respectively. This method cannot be used to +distinguish between netCDF-4 and netCDF-4 classic model variants, or between a +netCDF-4 file and a different kind of HDF5 file. ---------- How many netCDF data models are there? {#How-many-netCDF-data-models-are-there} ----------------- -There are only two netCDF data models, the [classic model](/netcdf/workshops/2008/datamodel/NcClassicModel.html) and the [enhanced model](/netcdf/workshops/2008/netcdf4/Nc4DataModel.html) (also called the netCDF-4 data model). The classic model is the simpler of the two, and is used for all data stored in classic format, 64-bit offset format, or netCDF-4 classic model format. The enhanced model (sometimes also referred to as the netCDF-4 data model) is an extension of the classic model that adds more powerful forms of data representation and -data types at the expense of some additional complexity. Although data represented with the classic model can also be represented using the enhanced model, datasets that use enhanced model features, such as user-defined data types, cannot be represented with the classic model. Use of the enhanced model requires storage in the netCDF-4 format. +There are only two netCDF data models, the [classic +model](/netcdf/workshops/2008/datamodel/NcClassicModel.html) and the [enhanced +model](/netcdf/workshops/2008/netcdf4/Nc4DataModel.html) (also called the +netCDF-4 data model). The classic model is the simpler of the two, and is used +for all data stored in classic CDF-1 format, 64-bit offset CDF-2 format, 64-bit +data CDF-5 format, or netCDF-4 classic model format. The enhanced model +(sometimes also referred to as the netCDF-4 data model) is an extension of the +classic model that adds more powerful forms of data representation and data +types at the expense of some additional complexity. Although data represented +with the classic model can also be represented using the enhanced model, +datasets that use enhanced model features, such as user-defined data types, +cannot be represented with the classic model. Use of the enhanced model +requires storage in the netCDF-4 format. How many releases of the C-based netCDF software are supported? {#How-many-releases-of-the-C-based-netCDF-software-are-supported} ----------------- @@ -664,15 +692,17 @@ the netCDF-4 and netCDF-4 classic model formats, if built using a previously installed HDF5 library and using the "--enable-netcdf-4" configure option. Software built from the netCDF-4.0 release without specifying "--enable-netcdf-4" (the default) was identical to software -built with netCDF-3.6.3. +built with netCDF-3.6.3. Starting from version 4.4.0, netCDF added support +for CDF-5 format. Both netCDF-3 and netCDF-4 C libraries are part of a single software release. The netCDF software may be built to support just the classic -and 64-bit offset formats (the default) or to also support the netCDF-4 -and netCDF-4 classic model formats, if the HDF5-1.8.x library is -installed. Unidata no longer supports a separate netCDF-3-only version -of the software, but instead supports both the classic and enhanced data -models and all four format variants in a single source distribution. +CDF-1 and 64-bit offset CDF-2 formats (the default), 64-bit data CDF-5 format, +or to also support the netCDF-4 and netCDF-4 classic model formats, if the +HDF5-1.8.x library is installed. Unidata no longer supports a separate +netCDF-3-only version of the software, but instead supports both the classic +and enhanced data models and all four format variants in a single source +distribution. This does not indicate any plan to drop support for netCDF-3 or the formats associated with netCDF-3. Support for earlier formats and APIs @@ -681,14 +711,13 @@ will continue with all future versions of netCDF software from Unidata. Should I get netCDF-3 or netCDF-4? {#Should-I-get-netCDF-3-or-netCDF-4} ----------------- - By downloading a current version of netCDF-4, you have the choice to build either -- the default netCDF-3 libraries, which support classic and 64-bit - offset formats, and the classic data model; or +- the default netCDF-3 libraries, which support classic CDF-1, 2, and 5 + formats, and the classic data model; or - the netCDF-4 libraries, which support netCDF-4 and netCDF-4 classic - model formats, as well as classic and 64-bit offset formats, and the + model formats, as well as classic formats, and the enhanced data model. Which version to build depends on how you will use the software. @@ -696,8 +725,8 @@ Which version to build depends on how you will use the software. Installing the simpler netCDF-3 version of the software is recommended if the following situations apply: -- all the data you need to access is available in netCDF classic or - 64-bit offset formats +- all the data you need to access is available in netCDF classic + formats - you are installing netCDF in order to support another software package that uses only netCDF-3 features - you plan to only write data in a form that netCDF-3 software and @@ -845,13 +874,12 @@ How can I convert netCDF-3 files into netCDF-4 files? {#How-can-I-convert-netCDF ----------------- -Every netCDF-3 file can be read or written by a netCDF version 4 -library, so in that respect netCDF-3 files are already netCDF-4 files -and need no conversion. But if you want to convert a classic or 64-bit -offset format file into a netCDF-4 format or netCDF-4 classic model -format file, the easiest way is to use the **nccopy** utility. For example -to convert a classic format file foo3.nc to a netCDF-4 format file -foo4.nc, use: +Every netCDF-3 file can be read or written by a netCDF version 4 library, so in +that respect netCDF-3 files are already netCDF-4 files and need no conversion. +But if you want to convert a classic format file (CDF-1, 2, or 5) into a +netCDF-4 format or netCDF-4 classic model format file, the easiest way is to +use the **nccopy** utility. For example to convert a classic format file +foo3.nc to a netCDF-4 format file foo4.nc, use: ~~~~~~~~~~~~~~~~~~~~~~~~~ {.boldcode} nccopy -k netCDF-4 foo3.nc foo4.nc @@ -872,10 +900,9 @@ utility "ncks" can be used to accomplish the same task, as follows: ~~~~~~~~~~~~~~~~~~~~~~~~ Another method is available for relatively small files, using the **ncdump** -and **ncgen** utilities (built with a netCDF-4 library). Assuming -"small3.nc" is a small classic format or 64-bit offset format netCDF -file, you can create an equivalent netCDF-4 file named -"small4.nc" as follows: +and **ncgen** utilities (built with a netCDF-4 library). Assuming "small3.nc" +is a small classic format netCDF file, you can create an equivalent netCDF-4 +file named "small4.nc" as follows: ~~~~ {.boldcode} ncdump small3.nc > small.cdl @@ -1219,11 +1246,15 @@ reference to the format label at the start of a file. With netCDF version 3.6 and later, a second variant of netCDF format is supported in addition to the classic format. The new variant is referred -to as the *64-bit offset* format, *version 2*, or *CDF2*. The primary +to as the *64-bit offset* format, *version 2*, or *CDF-2*. The primary difference from the classic format is the use of 64-bit file offsets instead of 32-bit offsets, but it also supports larger variable and record sizes. +Starting from version 4.4.0, netCDF added support for CDF-5 format, which +allows multiple large variables with more than 4-billion array elements defined +in the file. This format is only supported on 64-bit machine platforms. + ---------- Do I have to know which netCDF file format variant is used in order to access or modify a netCDF file? {#Do-I-have-to-know-which-netCDF-file-format-variant-is-used-in-order-to-access-or-modify-a-netCDF-file} @@ -1235,10 +1266,10 @@ detect which variant of the format is used for each file when it is opened for reading or writing, so it is not necessary to know which variant of the format is used. The version of the format will be preserved by the library on writing. If you want to modify a classic -format file to use the 64-bit offset format so you can make it much +format file to use the CDF-2 or CDF-5 format so you can make it much larger, you will have to create a new file and copy the data to it. The **nccopy** utility available in version 4.1 can copy a classic file to a -64-bit offset file. +CDF-2 or CDF-5 file. ---------- @@ -1247,13 +1278,13 @@ Will future versions of the netCDF library continue to support accessing files i Yes, the 3.6 library and all planned future versions of the library will -continue to support reading and writing files using the classic (32-bit -offset) format as well as the 64-bit offset format. There is no need to +continue to support reading and writing files using the classic CDF-1 (32-bit +offset), 64-bit offset CDF-2, and 64-bit data CDF-5 format. There is no need to convert existing archives from the classic to the 64-bit offset format. Even netCDF-4, which introduces a third variant of the netCDF format -based on HDF5, continues to support accessing classic format netCDF -files as well as 64-bit offset netCDF files. NetCDF-4 HDF5 files have -even fewer restrictions on size than 64-bit offset netCDF files. +based on HDF5, continues to support accessing classic CDF-1, 2, and 5 format +files. NetCDF-4 HDF5 files have even fewer restrictions on size than CDF-1 and +CDF-2 files. ---------- @@ -1272,17 +1303,17 @@ the classic format. ---------- -How can I tell if a netCDF file uses the classic format or 64-bit offset format? {#How-can-I-tell-if-a-netCDF-file-uses-the-classic-format-or-64-bit-offset-format} +How can I tell if a netCDF file uses the classic format (CDF-1), 64-bit offset format (CDF-2) or 64-bit data format (CDF-5)? {#How-can-I-tell-if-a-netCDF-file-uses-the-classic-format-or-64-bit-offset-format} ----------------- The short answer is that under most circumstances, you should not care, if you use version 3.6.0 or later of the netCDF library. But the difference is indicated in the first four bytes of the file, which are -'C', 'D', 'F', '\\001' for the classic netCDF format and 'C', 'D', 'F', -'\\002' for the 64-bit offset format. On a Unix system, one way to -display the first four bytes of a file, say foo.nc, is to run the -following command: +'C', 'D', 'F', '\\001' for the classic CDF-1 format, 'C', 'D', 'F', +'\\002' for the 64-bit offset CDF-2 format, and 'C', 'D', 'F', '\\005' for the +64-bit data CDF-5 format. On a Unix system, one way to display the first four +bytes of a file, say foo.nc, is to run the following command: ~~~~ {.boldcode} od -An -c -N4 foo.nc @@ -1300,7 +1331,13 @@ or C D F 002 ~~~~ -depending on whether foo.nc is a classic or 64-bit offset netCDF file, +or + +~~~~ {.boldcode} + C D F 005 +~~~~ + +depending on whether foo.nc is a CDF-1, CDF-2, or CDF-5 netCDF file, respectively. With netCDF version 3.6.2 or later, there is an easier way, using the diff --git a/docs/all-error-codes.md b/docs/all-error-codes.md index 3405fc47f..b5b0d5864 100644 --- a/docs/all-error-codes.md +++ b/docs/all-error-codes.md @@ -42,31 +42,7 @@ NetCDF Error Code Listing {#nc-error-codes} #define NC_EVARSIZE (-62) // One or more variable sizes violate format constraints #define NC_EDIMSIZE (-63) // Invalid dimension size #define NC_ETRUNC (-64) // File likely truncated or possibly corrupted -~~~~ - -# NetCDF-4 Error Codes {#nc4-error-codes} - -NetCDF-4 uses all error codes from NetCDF-3 (see section [NetCDF-3 Error -Codes](#NetCDF_002d3-Error-Codes)). The following additional error codes -were added for new errors unique to netCDF-4. - -~~~~ -#define NC_EHDFERR (-101) -#define NC_ECANTREAD (-102) -#define NC_ECANTWRITE (-103) -#define NC_ECANTCREATE (-104) -#define NC_EFILEMETA (-105) -#define NC_EDIMMETA (-106) -#define NC_EATTMETA (-107) -#define NC_EVARMETA (-108) -#define NC_ENOCOMPOUND (-109) -#define NC_EATTEXISTS (-110) -#define NC_ENOTNC4 (-111) // Attempting netcdf-4 operation on netcdf-3 file. -#define NC_ESTRICTNC3 (-112) // Attempting netcdf-4 operation on strict nc3 netcdf-4 file. -#define NC_EBADGRPID (-113) // Bad group id. Bad! -#define NC_EBADTYPEID (-114) // Bad type id. -#define NC_EBADFIELDID (-115) // Bad field id. -#define NC_EUNKNAME (-116) +#define NC_EAXISTYPE (-65) // Unknown axis type ~~~~ # DAP Error Codes {#dap-error-codes} @@ -85,14 +61,127 @@ may occur. #define NC_EDATADDS (-73) // Malformed or inaccessible DATADDS #define NC_EDAPURL (-74) // Malformed DAP URL #define NC_EDAPCONSTRAINT (-75) // Malformed DAP Constraint -#define NC_EDAP (-66) // Generic DAP error -#define NC_ECURL (-67) // Generic libcurl error -#define NC_EIO (-68) // Generic IO error -#define NC_ENODATA (-69) // Attempt to access variable with no data -#define NC_EDAPSVC (-70) // DAP Server side error -#define NC_EDAS (-71) // Malformed or inaccessible DAS -#define NC_EDDS (-72) // Malformed or inaccessible DDS -#define NC_EDATADDS (-73) // Malformed or inaccessible DATADDS -#define NC_EDAPURL (-74) // Malformed DAP URL -#define NC_EDAPCONSTRAINT (-75) // Malformed DAP Constraint +#define NC_ETRANSLATION (-76) // Untranslatable construct +#define NC_EACCESS (-77) // Access Failure +#define NC_EAUTH (-78) // Authorization Failure ~~~~ + +# Misc. additional errors +~~~~ +#define NC_ENOTFOUND (-90) // No such file +#define NC_ECANTREMOVE (-91) // Cannot remove file +#define NC_EINTERNAL (-92) // NetCDF Library Internal Error +~~~~ + +# NetCDF-4 Error Codes {#nc4-error-codes} + +NetCDF-4 uses all error codes from NetCDF-3 (see section [NetCDF-3 Error +Codes](#NetCDF_002d3-Error-Codes)). The following additional error codes +were added for new errors unique to netCDF-4. + +~~~~ +#define NC_EHDFERR (-101) // Error at HDF5 layer. +#define NC_ECANTREAD (-102) // Cannot read. +#define NC_ECANTWRITE (-103) // Cannot write. +#define NC_ECANTCREATE (-104) // Cannot create. +#define NC_EFILEMETA (-105) // Problem with file metadata. +#define NC_EDIMMETA (-106) // Problem with dimension metadata. +#define NC_EATTMETA (-107) // Problem with attribute metadata. +#define NC_EVARMETA (-108) // Problem with variable metadata. +#define NC_ENOCOMPOUND (-109) // Not a compound type. +#define NC_EATTEXISTS (-110) // Attribute already exists. +#define NC_ENOTNC4 (-111) // Attempting netcdf-4 operation on netcdf-3 file. +#define NC_ESTRICTNC3 (-112) // Attempting netcdf-4 operation on strict nc3 netcdf-4 file. +#define NC_ENOTNC3 (-113) // Attempting netcdf-3 operation on netcdf-4 file. +#define NC_ENOPAR (-114) // Parallel operation on file opened for non-parallel access. +#define NC_EPARINIT (-115) // Error initializing for parallel access. +#define NC_EBADGRPID (-116) // Bad group ID. +#define NC_EBADTYPID (-117) // Bad type ID. +#define NC_ETYPDEFINED (-118) // Type has already been defined and may not be edited. +#define NC_EBADFIELD (-119) // Bad field ID. +#define NC_EBADCLASS (-120) // Bad class. +#define NC_EMAPTYPE (-121) // Mapped access for atomic types only. +#define NC_ELATEFILL (-122) // Attempt to define fill value when data already exists. +#define NC_ELATEDEF (-123) // Attempt to define var properties, like deflate, after enddef. +#define NC_EDIMSCALE (-124) // Problem with HDF5 dimscales. +#define NC_ENOGRP (-125) // No group found. +#define NC_ESTORAGE (-126) // Cannot specify both contiguous and chunking. +#define NC_EBADCHUNK (-127) // Bad chunksize. +#define NC_ENOTBUILT (-128) // Attempt to use feature that was not turned on when netCDF was built. +#define NC_EDISKLESS (-129) // Error in using diskless access. +#define NC_ECANTEXTEND (-130) // Attempt to extend dataset during ind. I/O operation. +#define NC_EMPI (-131) // MPI operation failed. +#define NC_EFILTER (-132) // Filter operation failed. +#define NC_ERCFILE (-133) // RC file failure +#define NC_ENULLPAD (-134) // Header Bytes not Null-Byte padded +#define NC_EINMEMORY (-135) // In-memory file error +~~~~ + +# PnetCDF Error Codes {#pnetcdf-error-codes} + +~~~~ +#define NC_ESMALL (-201) // size of MPI_Offset too small for format +#define NC_ENOTINDEP (-202) // Operation not allowed in collective data mode +#define NC_EINDEP (-203) // Operation not allowed in independent data mode +#define NC_EFILE (-204) // Unknown error in file operation +#define NC_EREAD (-205) // Unknown error in reading file +#define NC_EWRITE (-206) // Unknown error in writing to file +#define NC_EOFILE (-207) // file open/creation failed +#define NC_EMULTITYPES (-208) // Multiple etypes used in MPI datatype +#define NC_EIOMISMATCH (-209) // Input/Output data amount mismatch +#define NC_ENEGATIVECNT (-210) // Negative count is specified +#define NC_EUNSPTETYPE (-211) // Unsupported etype in memory MPI datatype +#define NC_EINVAL_REQUEST (-212) // invalid nonblocking request ID +#define NC_EAINT_TOO_SMALL (-213) // MPI_Aint not large enough to hold requested value +#define NC_ENOTSUPPORT (-214) // feature is not yet supported +#define NC_ENULLBUF (-215) // trying to attach a NULL buffer +#define NC_EPREVATTACHBUF (-216) // previous attached buffer is found +#define NC_ENULLABUF (-217) // no attached buffer is found +#define NC_EPENDINGBPUT (-218) // pending bput is found, cannot detach buffer +#define NC_EINSUFFBUF (-219) // attached buffer is too small +#define NC_ENOENT (-220) // File does not exist +#define NC_EINTOVERFLOW (-221) // Overflow when type cast to 4-byte integer +#define NC_ENOTENABLED (-222) // feature is not enabled +#define NC_EBAD_FILE (-223) // Invalid file name (e.g., path name too long) +#define NC_ENO_SPACE (-224) // Not enough space +#define NC_EQUOTA (-225) // Quota exceeded +#define NC_ENULLSTART (-226) // argument start is a NULL pointer +#define NC_ENULLCOUNT (-227) // argument count is a NULL pointer +#define NC_EINVAL_CMODE (-228) // Invalid file create mode +#define NC_ETYPESIZE (-229) // MPI derived data type size error (bigger than the variable size) +#define NC_ETYPE_MISMATCH (-230) // element type of the MPI derived data type mismatches the variable type +#define NC_ETYPESIZE_MISMATCH (-231) // file type size mismatches buffer type size +#define NC_ESTRICTCDF2 (-232) // Attempting CDF-5 operation on CDF-2 file +#define NC_ENOTRECVAR (-233) // Attempting operation only for record variables +#define NC_ENOTFILL (-234) // Attempting to fill a variable when its fill mode is off +#define NC_EINVAL_OMODE (-235) // Invalid file open mode +#define NC_EPENDING (-236) // Pending nonblocking request is found at file close +#define NC_EMAX_REQ (-237) // Size of I/O request exceeds INT_MAX +#define NC_EBADLOG (-238) // Unrecognized log file format +#define NC_EFLUSHED (-239) // Nonblocking request has already been flushed to the PFS. It is too late to cancel +#define NC_EMULTIDEFINE (-250) // NC definitions inconsistent among processes +#define NC_EMULTIDEFINE_OMODE (-251) // inconsistent file open modes among processes +#define NC_EMULTIDEFINE_DIM_NUM (-252) // inconsistent number of dimensions +#define NC_EMULTIDEFINE_DIM_SIZE (-253) // inconsistent size of dimension +#define NC_EMULTIDEFINE_DIM_NAME (-254) // inconsistent dimension names +#define NC_EMULTIDEFINE_VAR_NUM (-255) // inconsistent number of variables +#define NC_EMULTIDEFINE_VAR_NAME (-256) // inconsistent variable name +#define NC_EMULTIDEFINE_VAR_NDIMS (-257) // inconsistent variable number of dimensions +#define NC_EMULTIDEFINE_VAR_DIMIDS (-258) // inconsistent variable dimension IDs +#define NC_EMULTIDEFINE_VAR_TYPE (-259) // inconsistent variable data type +#define NC_EMULTIDEFINE_VAR_LEN (-260) // inconsistent variable size +#define NC_EMULTIDEFINE_NUMRECS (-261) // inconsistent number of records +#define NC_EMULTIDEFINE_VAR_BEGIN (-262) // inconsistent variable file begin offset (internal use) +#define NC_EMULTIDEFINE_ATTR_NUM (-263) // inconsistent number of attributes +#define NC_EMULTIDEFINE_ATTR_SIZE (-264) // inconsistent memory space used by attribute (internal use) +#define NC_EMULTIDEFINE_ATTR_NAME (-265) // inconsistent attribute name +#define NC_EMULTIDEFINE_ATTR_TYPE (-266) // inconsistent attribute type +#define NC_EMULTIDEFINE_ATTR_LEN (-267) // inconsistent attribute length +#define NC_EMULTIDEFINE_ATTR_VAL (-268) // inconsistent attribute value +#define NC_EMULTIDEFINE_FNC_ARGS (-269) // inconsistent function arguments used in collective API +#define NC_EMULTIDEFINE_FILL_MODE (-270) // inconsistent dataset fill mode +#define NC_EMULTIDEFINE_VAR_FILL_MODE (-271) // inconsistent variable fill mode +#define NC_EMULTIDEFINE_VAR_FILL_VALUE (-272) // inconsistent variable fill value +#define NC_EMULTIDEFINE_CMODE (-273) // inconsistent file create modes among processes +~~~~ + diff --git a/docs/guide.dox b/docs/guide.dox index 79c1995d2..2de8484c7 100644 --- a/docs/guide.dox +++ b/docs/guide.dox @@ -115,19 +115,30 @@ is the default format for all versions of netCDF. In version 3.6.0 a new binary format was introduced, 64-bit offset format. Nearly identical to netCDF classic format, it uses 64-bit offsets (hence the name), and allows users to create far larger -datasets. +datasets. This format is also referred as CDF-2, because it bears the signature +string "CDF2" in the file header. After this extension, the classic file +format (i.e. not supporting 64-bit offsets) is now referred as CDF-1. In version 4.0.0 a third binary format was introduced: the HDF5 format. Starting with this version, the netCDF library can use HDF5 files as its base format. (Only HDF5 files created with netCDF-4 can be understood by netCDF-4). -By default, netCDF uses the classic format. To use the 64-bit offset -or netCDF-4/HDF5 format, set the appropriate constant when creating -the file. +Starting from version 4.4.0, netCDF included the support of CDF-5 format. In +order to allows defining large array variables with more than 4-billion +elements, CDF-5 replaces most of the 32-bit integers used to describe metadata +in file header with 64-bit integers. In addition, it supports the following new +external data types: NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, and NC_UINT64. The +CDF-5 format specifications can be found in +(http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html). + +The classic file formats are now referring to the collection of CDF-1, 2 and 5 +formats. By default, netCDF uses the classic format (CDF-1). To use the CDF-2, +CDF-5, or netCDF-4/HDF5 format, set the appropriate constant in the file mode +argument when creating the file. To achieve network-transparency (machine-independence), netCDF classic -and 64-bit offset formats are implemented in terms of an external +formats are implemented in terms of an external representation much like XDR (eXternal Data Representation, see http://www.ietf.org/rfc/rfc1832.txt), a standard for describing and encoding data. This representation provides encoding of data into @@ -137,11 +148,10 @@ be encoded and decoded in a consistent way. The IEEE 754 floating-point standard is used for floating-point data representation. -Descriptions of the overall structure of netCDF classic and 64-bit -offset files are provided later in this manual. See -\ref file_structure_and_performance. +Descriptions of the overall structure of netCDF classic files are provided +later in this manual. See \ref file_structure_and_performance. -The details of the classic and 64-bit offset formats are described in +The details of the CDF-1 and CDF-2 formats are described in an appendix. See \ref netcdf_format. However, users are discouraged from using the format specification to develop independent low-level software for reading and writing netCDF files, because this could lead @@ -149,57 +159,70 @@ to compatibility problems if the format is ever modified. \subsection select_format How to Select the Format -With three different base formats, care must be taken in creating data +With four different base formats, care must be taken in creating data files to choose the correct base format. The format of a netCDF file is determined at create time. When opening an existing netCDF file the netCDF library will transparently detect its format and adjust accordingly. However, -netCDF library versions earlier than 3.6.0 cannot read 64-bit offset -format files, and library versions before 4.0 can't read netCDF-4/HDF5 -files. NetCDF classic format files (even if created by version 3.6.0 +netCDF library versions earlier than 3.6.0 cannot read CDF-2 +format files, library versions before 4.0 can't read netCDF-4/HDF5 +files, and versions before 4.4.0 cannot read CDF-5 files. +NetCDF classic format files (even if created by version 3.6.0 or later) remain compatible with older versions of the netCDF library. -Users are encouraged to use netCDF classic format to distribute data, +Users are encouraged to use netCDF classic CDF-1 format to distribute data, for maximum portability. -To select 64-bit offset or netCDF-4 format files, C programmers should -use flag NC_64BIT_OFFSET or NC_NETCDF4 in function nc_create(). +To select CDF-2, CDF-5 or netCDF-4 format files, C programmers should use flag +NC_64BIT_OFFSET, NC_64BIT_DATA, or NC_NETCDF4 respectively in function +nc_create(). -In Fortran, use flag nf_64bit_offset or nf_format_netcdf4 in function -NF_CREATE. See NF_CREATE. +In Fortran, use flag nf_64bit_offset, nf_64bit_data, or nf_format_netcdf4 in +function NF_CREATE. See NF_CREATE. It is also possible to change the default creation format, to convert a large body of code without changing every create call. C programmers see nc_set_default_format(). Fortran programs see NF_SET_DEFAULT_FORMAT. -\subsection classic_format NetCDF Classic Format +\subsection classic_format NetCDF Classic Format (CDF-1) The original netCDF format is identified using four bytes in the file -header. All files in this format have “CDF\001” at the beginning of -the file. In this documentation this format is referred to as “netCDF -classic format.” +header. All files in this format have "CDF\001" at the beginning of +the file. In this documentation this format is referred to as CDF-1 +format. -NetCDF classic format is identical to the format used by every +NetCDF CDF-1 format is identical to the format used by every previous version of netCDF. It has maximum portability, and is still the default netCDF format. +\subsection netcdf_64bit_offset_format NetCDF 64-bit Offset Format (CDF-2) + For some users, the various 2 GiB format limitations of the classic format become a problem. (see \ref limitations). - -\subsection netcdf_64bit_offset_format NetCDF 64-bit Offset Format - For these users, 64-bit offset format is a natural choice. It greatly eases the size restrictions of netCDF classic files (see \ref limitations). -Files with the 64-bit offsets are identified with a “CDF\002” at the +Files with the 64-bit offsets are identified with a "CDF\002" at the beginning of the file. In this documentation this format is called -“64-bit offset format.” +CDF-2 format. -Since 64-bit offset format was introduced in version 3.6.0, earlier -versions of the netCDF library can't read 64-bit offset files. +Since CDF-2 format was introduced in version 3.6.0, earlier +versions of the netCDF library can't read CDF-2 files. + +\subsection netcdf_64bit_data_format NetCDF 64-bit Data Format (CDF-5) + +To allow large variables with more than 4-billion array elements, +64-bit data format is develop to support such I/O requests. + +Files with the 64-bit data are identified with a "CDF\005" at the +beginning of the file. In this documentation this format is called +CDF-5 format. + +Since CDF-5 format was introduced in version 4.4.0, earlier +versions of the netCDF library can't read CDF-5 files. \subsection netcdf_4_format NetCDF-4 Format @@ -303,17 +326,17 @@ its implementation in software. Some of these limitations have been removed or relaxed in netCDF-4 files, but still apply to netCDF classic and netCDF 64-bit offset files. -Currently, netCDF classic and 64-bit offset formats offer a limited +NetCDF classic CDF-1 and CDF-2 formats offer a limited number of external numeric data types: 8-, 16-, 32-bit integers, or -32- or 64-bit floating-point numbers. (The netCDF-4 format adds 64-bit -integer types and unsigned integer types.) +32- or 64-bit floating-point numbers. The CDF-5 and netCDF-4 formats add 64-bit +integer types and unsigned integer types. With the netCDF-4/HDF5 format, new unsigned integers (of various sizes), 64-bit integers, and the string type allow improved expression of meaning in scientific data. The new VLEN (variable length) and COMPOUND types allow users to organize data in new ways. -With the classic netCDF file format, there are constraints that limit +With the classic CDF-1 file format, there are constraints that limit how a dataset is structured to store more than 2 GiBytes (a GiByte is 2^30 or 1,073,741,824 bytes, as compared to a Gbyte, which is 1,000,000,000 bytes.) of data in a single netCDF dataset. (see \ref @@ -326,7 +349,7 @@ possible to create and access netCDF files larger than 2 GiB on platforms that provide support for such files (see \ref large_file_support). -The new 64-bit offset format allows large files, and makes it easy to +The CDF-2 format allows large files, and makes it easy to create to create fixed variables of about 4 GiB, and record variables of about 4 GiB per record. (see \ref netcdf_64bit_offset_format). However, old netCDF applications will not be able to read the 64-bit offset @@ -338,7 +361,11 @@ and files can be as large as the underlying file system supports. NetCDF-4/HDF5 files are unreadable to the netCDF library before version 4.0. -Another limitation of the classic (and 64-bit offset) model is that +Similarly, CDF-5 format uses 64-bit integers to allow users to define +large variables. CDF-5 files are not unreadable to the netCDF library +before version 4.4.0. + +Another limitation of the classic formats (CDF-1, 2 and 5) is that only one unlimited (changeable) dimension is permitted for each netCDF data set. Multiple variables can share an unlimited dimension, but then they must all grow together. Hence the classic netCDF model does @@ -373,8 +400,8 @@ kinds of data). A data model tailored to capture the shared context among researchers within one discipline may not be appropriate for sharing or combining data from multiple disciplines. -The classic netCDF data model (which is used for classic-format and -64-bit offset format data) does not support nested data structures +The classic netCDF data model (which is used for classic CDF-1, 2 and 5 format +data) does not support nested data structures such as trees, nested arrays, or other recursive structures. Through use of indirection and conventions it is possible to represent some kinds of nested structures, but the result may fall short of the @@ -386,20 +413,15 @@ of types. The VLEN type allows efficient storage of ragged arrays, and the introduction of hierarchical groups allows users new ways to organize data. -Finally, using the netCDF-3 programming interfaces, concurrent access -to a netCDF dataset is limited. One writer and multiple readers may -access data in a single dataset simultaneously, but there is no -support for multiple concurrent writers. - NetCDF-4 supports parallel read/write access to netCDF-4/HDF5 files, using the underlying HDF5 library and parallel read/write access to -classic and 64-bit offset files using the parallel-netcdf library. +classic files using the PnetCDf library. For more information about HDF5, see the HDF5 web site: http://hdfgroup.org/HDF5/. -For more information about parallel-netcdf, see their web site: -http://www.mcs.anl.gov/parallel-netcdf. +For more information about PnetCDF, see their web site: +https://parallel-netcdf.github.io/. \page netcdf_data_set_components The Components of a NetCDF Data Set @@ -448,8 +470,7 @@ and variable may not have the same name within the same group, and similarly for sub-groups of that group.) Groups and user-defined types are only available in files created in -the netCDF-4/HDF5 format. They are not available for classic or 64-bit -offset format files. +the netCDF-4/HDF5 format. They are not available for classic format files. \section dimensions Dimensions @@ -460,8 +481,8 @@ model-run-number. A netCDF dimension has both a name and a length. -A dimension length is an arbitrary positive integer, except that one -dimension in a classic or 64-bit offset netCDF dataset can have the +A dimension length is an arbitrary positive integer, except that only one +dimension in a classic netCDF dataset can have the length UNLIMITED. In a netCDF-4 dataset, any number of unlimited dimensions can be used. @@ -470,7 +491,7 @@ dimension. A variable with an unlimited dimension can grow to any length along that dimension. The unlimited dimension index is like a record number in conventional record-oriented files. -A netCDF classic or 64-bit offset dataset can have at most one +A netCDF classic dataset can have at most one unlimited dimension, but need not have any. If a variable has an unlimited dimension, that dimension must be the most significant (slowest changing) one. Thus any unlimited dimension must be the first @@ -524,15 +545,16 @@ associated attributes, which may be added, deleted or changed after the variable is created. A variable external data type is one of a small set of netCDF -types. In classic and 64-bit offset files, only the original six types +types. In classic CDF-1 and 2 files, only the original six types are available (byte, character, short, int, float, and -double). Variables in netCDF-4 files may also use unsigned short, -unsigned int, 64-bit int, unsigned 64-bit int, or string. Or the user +double). CDF-5 adds unsigned byte, unsigned short, unsigned int, 64-bit int, +and unsigned 64-bit int. In netCDF-4, variables may also use these additional +data types, plus the string data type. Or the user may define a type, as an opaque blob of bytes, as an array of variable length arrays, or as a compound type, which acts like a C struct. (See \ref data_type). -In the CDL notation, classic and 64-bit offset type can be used. They +In the CDL notation, classic data type can be used. They are given the simpler names byte, char, short, int, float, and double. The name real may be used as a synonym for float in the CDL notation. The name long is a deprecated synonym for int. For the exact @@ -694,8 +716,8 @@ attributes, see \ref attribute_conventions. Attributes may be added to a netCDF dataset long after it is first defined, so you don't have to anticipate all potentially useful -attributes. However adding new attributes to an existing classic or -64-bit offset format dataset can incur the same expense as copying the +attributes. However adding new attributes to an existing classic +format dataset can incur the same expense as copying the dataset. For a more extensive discussion see \ref file_structure_and_performance. \section differences_atts_vars Differences between Attributes and Variables @@ -771,7 +793,7 @@ information. \section archival Is NetCDF a Good Archive Format? -NetCDF classic or 64-bit offset formats can be used as a +NetCDF classic formats can be used as a general-purpose archive format for storing arrays. Compression of data is possible with netCDF (e.g., using arrays of eight-bit or 16-bit integers to encode low-resolution floating-point numbers instead of @@ -994,8 +1016,8 @@ servers, full support for the enhanced netCDF-4 data model in the ncgen utility, a new nccopy utility for copying and conversion among netCDF format variants, ability to read some HDF4/HDF5 data archives through the netCDF C or Fortran interfaces, support for parallel I/O -on netCDF classic and 64-bit offset files using the parallel-netcdf -(formerly pnetcdf) library from Argonne/Northwestern, a new nc-config +on netCDF classic files (CDF-1, 2, and 5) using the PnetCDF +library from Argonne/Northwestern, a new nc-config utility to help compile and link programs that use netCDF, inclusion of the UDUNITS library for handling “units” attributes, and inclusion of libcf to assist in creating data compliant with the Climate and @@ -1004,7 +1026,7 @@ Forecast (CF) metadata conventions. In September, 2010, the Netcdf-Java/CDM (Common Data Model) version 4.2 library was declared stable and made available to users. This 100%-Java implementation provided a read-write interface to netCDF-3 -classic and 64-bit offset data, as well as a read-only interface to +classic format files, as well as a read-only interface to netCDF-4 enhanced model data and many other formats of scientific data through a common (CDM) interface. More recent releases support writing netCDF-4 data. The NetCDF-Java library also @@ -1030,7 +1052,7 @@ a different representation from the external type of the variable, a conversion between the internal type and external type will take place when the data is read or written. -Access to data in classic and 64-bit offset format is direct. Access +Access to data in classic formats is direct. Access to netCDF-4 data is buffered by the HDF5 layer. In either case you can access a small subset of data from a large dataset efficiently, without first accessing all the data that precedes it. @@ -1234,8 +1256,8 @@ array section access instead. \section classic_file_parts Parts of a NetCDF Classic File -A netCDF classic or 64-bit offset dataset is stored as a single file -comprising two parts: +A netCDF classic dataset (including CDF-1, 2, and 5 formats) is stored as a +single file comprising two parts: - a header, containing all the information about dimensions, attributes, and variables except for the variable data; - a data part, comprising fixed-size data, containing the data for @@ -1338,8 +1360,8 @@ dimension ids. XDR is a standard for describing and encoding data and a library of functions for external data representation, allowing programmers to -encode data structures in a machine-independent way. Classic or 64-bit -offset netCDF employs an extended form of XDR for representing +encode data structures in a machine-independent way. Classic +netCDF employs an extended form of XDR for representing information in the header part and the data parts. This extended XDR is used to write portable data that can be read on any other machine for which the library has been implemented. @@ -1370,10 +1392,20 @@ limitations. See \ref limitations. In version 3.6.0, netCDF included its first-ever variant of the underlying data format. The new format introduced in 3.6.0 uses 64-bit -file offsets in place of the 32-bit offsets. There are still some +file offsets in place of the 32-bit offsets. The new format is also referred as +CDF-2 format as it bears a signature string of "CDF2" in the file header. +There are still some limits on the sizes of variables, but the new format can create very large datasets. See \ref netcdf_64bit_offset_format. +Starting from version 4.4.0, netCDF included the support of CDF-5 format. In +order to allows defining large array variables with more than 4-billion +elements, CDF-5 replaces most of the 32-bit integers used to describe metadata +with 64-bit integers. In addition, it supports the following new external data +types: NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, and NC_UINT64. The CDF-5 format +specifications can be found in +(http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html). + NetCDF-4 variables and files can be any size supported by the underlying file system. @@ -1387,7 +1419,7 @@ about 1.15e+18 bytes. Note also that all sizes are really 4 bytes less than the ones given below. For example the maximum size of a fixed variable in netCDF 3.6 classic format is really 2 GiB - 4 bytes. -Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4 +Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4 and CDF-5 -------------------------------------------|------------|---------|---------------|---------------------|----------------- Max File Size | 2 GiB | 8 EiB | 8 EiB | 8 EiB | unlimited Max Number of Fixed Vars > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited @@ -1400,14 +1432,14 @@ For more information about the different file formats of netCDF see \section offset_format_limitations NetCDF 64-bit Offset Format Limitations -Although the 64-bit offset format allows the creation of much larger +Although the 64-bit offset format (CDF-2) allows the creation of much larger netCDF files than was possible with the classic format, there are still some restrictions on the size of variables. It's important to note that without Large File Support (LFS) in the operating system, it's impossible to create any file larger than 2 GiBytes. Assuming an operating system with LFS, the following -restrictions apply to the netCDF 64-bit offset format. +restrictions apply to the CDF-2 format. No fixed-size variable can require more than 2^32 - 4 bytes (i.e. 4GiB - 4 bytes, or 4,294,967,292 bytes) of storage for its data, unless it @@ -1415,13 +1447,13 @@ is the last fixed-size variable and there are no record variables. When there are no record variables, the last fixed-size variable can be any size supported by the file system, e.g. terabytes. -A 64-bit offset format netCDF file can have up to 2^32 - 1 fixed sized +A CDF-2 file can have up to 2^32 - 1 fixed sized variables, each under 4GiB in size. If there are no record variables in the file the last fixed variable can be any size. No record variable can require more than 2^32 - 4 bytes of storage for each record's worth of data, unless it is the last record variable. A -64-bit offset format netCDF file can have up to 2^32 - 1 records, of +CDF-2 file can have up to 2^32 - 1 records, of up to 2^32 - 1 variables, as long as the size of one record's data for each record variable except the last is less than 4 GiB - 4. @@ -1432,12 +1464,12 @@ boundaries. There are important constraints on the structure of large netCDF classic files that result from the 32-bit relative offsets that are -part of the netCDF classic file format: +part of the netCDF classic file format (CDF-1): The maximum size of a record in the classic format in versions 3.5.1 and earlier is 2^32 - 4 bytes, or about 4 GiB. In versions 3.6.0 and later, there is no such restriction on total record size for the -classic format or 64-bit offset format. +classic formats. If you don't use the unlimited dimension, only one variable can exceed 2 GiB in size, but it can be as large as the underlying file system @@ -1491,10 +1523,10 @@ the data in a 2.4 Tbyte file might be something like: \section netcdf_3_io The NetCDF-3 I/O Layer -The following discussion applies only to netCDF classic and 64-bit -offset files. For netCDF-4 files, the I/O layer is the HDF5 library. +The following discussion applies only to netCDF classic files (i.e. CDF-1, 2, +and 5 formats). For netCDF-4 files, the I/O layer is the HDF5 library. -For netCDF classic and 64-bit offset files, an I/O layer implemented +For netCDF classic offset files, an I/O layer implemented much like the C standard I/O (stdio) library is used by netCDF to read and write portable data to netCDF datasets. Hence an understanding of the standard I/O library provides answers to many questions about @@ -1519,7 +1551,7 @@ operations can influence I/O performance significantly. Reading data in the same order in which it was written within each record will minimize buffer flushes. -You should not expect netCDF classic or 64-bit offset format data +You should not expect netCDF classic format data access to work with multiple writers having the same file open for writing simultaneously. @@ -1547,12 +1579,12 @@ NetCDF uses the HDF5 parallel programming model for parallel I/O with netCDF-4/HDF5 files. The HDF5 tutorial (http://hdfgroup.org/HDF5//HDF5/Tutor) is a good reference. -For classic and 64-bit offset files, netCDF uses the parallel-netcdf -(formerly pnetcdf) library from Argonne National Labs/Nortwestern -University. For parallel access of classic and 64-bit offset files, -netCDF must be configured with the –with-pnetcdf option at build -time. See the parallel-netcdf site for more information -(http://www.mcs.anl.gov/parallel-netcdf). +For classic files, netCDF uses the PnetCDF library from Argonne National +Labs/Northwestern University. For parallel access of files in classic formats, +netCDF must be configured with the --with-pnetcdf option at build time. See the +PnetCDF site for more information (https://parallel-netcdf.github.io). +Addition information and example programs can be found in +(http://cucis.ece.northwestern.edu/projects/PnetCDF/#InteroperabilityWithNetCDF4) \section interoperability_with_hdf5 Interoperability with HDF5 @@ -2241,7 +2273,7 @@ possible floating-point values would requires '-p 9,17'.) @par -k Show \e kind of netCDF file, that is which format variant the file uses. Other options are ignored if this option is specified. Output will be -one of 'classic'. '64-bit offset', 'netCDF-4', or 'netCDF-4 classic +one of 'classic'. '64-bit offset', '64-bit data', 'netCDF-4', or 'netCDF-4 classic model'. @par -s @@ -2350,7 +2382,7 @@ ncgen(1), netcdf(3) \subsection ncdump_string_note NOTE ON STRING OUTPUT -For classic, 64-bit offset or netCDF-4 classic model data, \b ncdump +For classic, 64-bit offset, 64-bit data, or netCDF-4 classic model data, \b ncdump generates line breaks after embedded newlines in displaying character data. This is not done for netCDF-4 files, because netCDF-4 supports arrays of real strings of varying length. @@ -2374,7 +2406,7 @@ The \b nccopy utility copies an input netCDF file in any supported format variant to an output netCDF file, optionally converting the output to any compatible netCDF format variant, compressing the data, or rechunking the data. For example, if built with the netCDF-3 -library, a netCDF classic file may be copied to a netCDF 64-bit offset +library, a classic CDF-1 file may be copied to a CDF-2 or CDF-5 file, permitting larger variables. If built with the netCDF-4 library, a netCDF classic file may be copied to a netCDF-4 file or to a netCDF-4 classic model file as well, permitting data compression, @@ -2383,7 +2415,7 @@ netCDF-4 features. If no output format is specified, with either \b -k \e kind_name or \e -kind_code, then the output will use the same -format as the input, unless the input is classic or 64-bit offset +format as the input, unless the input is classic format and either chunking or compression is specified, in which case the output will be netCDF-4 classic model format. Attempting some kinds of format conversion will result in an error, if the @@ -2409,7 +2441,8 @@ Use format name to specify the kind of file to be created and, by inference, the data model (i.e. netcdf-3 (classic) or netcdf-4 (enhanced)). The possible arguments are: \n 'nc3' or 'classic' => netCDF classic format \n - 'nc6' or '64-bit offset' => netCDF 64-bit format \n + 'nc6' or '64-bit offset' => netCDF 64-bit offset format \n + 'cdf5' => netCDF 64-bit data format \n 'nc4' or 'netCDF-4' => netCDF-4 format (enhanced data model) \n 'nc7' or 'netCDF-4 classic model' => netCDF-4 classic model format \n @@ -2424,7 +2457,8 @@ Use format numeric code (instead of format name) to specify the kind of file to and, by inference, the data model (i.e. netcdf-3 (classic) versus netcdf-4 (enhanced)). The numeric codes are: \n 3 => netcdf classic format \n - 6 => netCDF 64-bit format \n + 6 => netCDF 64-bit offset format (CDF_2) \n + 5 => netCDF 64-bit data format (CDF-5) \n 4 => netCDF-4 format (enhanced data model) \n 7 => netCDF-4 classic model format \n @@ -2440,7 +2474,7 @@ corresponds to no compression and 9 to maximum compression, with higher levels of compression requiring marginally more time to compress or uncompress than lower levels. Compression achieved may also depend on output chunking parameters. If this option is -specified for a classic format or 64-bit offset format input file, it +specified for a classic format input file, it is not necessary to also specify that the output should be netCDF-4 classic model, as that will be the default. If this option is not specified and the input file has compressed variables, the compression @@ -2618,7 +2652,7 @@ but no general algorithm for computing the optimum number of chunk cache elements has been implemented yet. \par -r -Read netCDF classic or 64-bit offset input file into a diskless netCDF +Read netCDF classic input file into a diskless netCDF file in memory before copying. Requires that input file be small enough to fit into memory. For \b nccopy, this doesn't seem to provide any significant speedup, so may not be a useful option. @@ -2665,9 +2699,9 @@ type, to the output file bar.nc: \code nccopy -d1 foo.nc bar.nc \endcode -If foo.nc was a classic or 64-bit offset netCDF file, bar.nc will be a -netCDF-4 classic model netCDF file, because the classic and 64-bit -offset format variants don't support compression. If foo.nc was a +If foo.nc was a classic netCDF file, bar.nc will be a +netCDF-4 classic model netCDF file, because the classic +formats don't support compression. If foo.nc was a netCDF-4 file with some variables compressed using various deflation levels, the output will also be a netCDF-4 file of the same type, but all the variables, including any uncompressed variables in the input, diff --git a/docs/tutorial.dox b/docs/tutorial.dox index 640204b4f..18536667c 100644 --- a/docs/tutorial.dox +++ b/docs/tutorial.dox @@ -144,9 +144,9 @@ must add support for netCDF-4 advanced features. In addition to the main netCDF-3 C API, there is an additional (older) C API, the netCDF-2 API. This API produces exactly the same files as -the netCDF-3 API - only the API is different. (That is, users can -create either classic format files, the default, or 64-bit offset -files, or netCDF-4/HDF5 files.) +the netCDF-3 API - only the API is different. That is, users can +create either classic CDF-1 format files, the default, 64-bit offset +files (CDF-2), 64-bit data files (CDF-5), or netCDF-4/HDF5 files. The version 2 API was the API before netCDF-3.0 came out. It is still fully supported, however. Programs written to the version 2 API will @@ -262,7 +262,7 @@ opaque types. These features may only be used when working with a netCDF-4/HDF5 -file. Files created in classic or 64-bit offset format cannot support +file. Files created in classic formats cannot support groups or user-defined types (see \ref netcdf_format).

\image html nc4-model.png @@ -797,8 +797,11 @@ with nc_get_vara()/nc_put_vara(). \page parallel_io Parallel I/O with NetCDF-4 -NetCDF-4 provides access to HDF5 parallel I/O features for netCDF-4/HDF5 files. NetCDF classic and 64-bit offset format may not be opened or created for use with HDF5-based parallel I/O. (They may be opened and created, but parallel I/O is not available.) They may be opened or created for use with parallel-netcdf-based parallel I/O. A few functions have been added to the netCDF C API to handle parallel I/O. You must build netCDF-4 properly to take advantage of parallel -features (see \ref build_parallel). +NetCDF-4 provides parallel file access to both classic and netCDF-4/HDF5 files. +The parallel I/O to netCDF-4 files is achieved through the HDF5 library while +the parallel I/O to classic files is through PnetCDF. A few functions have been +added to the netCDF C API to handle parallel I/O. You must build netCDF-4 +properly to take advantage of parallel features (see \ref build_parallel). The nc_open_par() and nc_create_par() functions are used to create/open a netCDF file with parallel access. @@ -807,9 +810,13 @@ create/open a netCDF file with parallel access. \section collective_independent Collective/Independent Access -Parallel file access is either collective (all processors must -participate) or independent (any processor may access the data without -waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access(). +Parallel file access is either collective (all processors must participate) or +independent (any processor may access the data without waiting for others). All +netCDF metadata writing operations are collective. That is, all creation of +groups, types, variables, dimensions, or attributes. Data reads and writes +(e.g. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent, the +default) or collective. To make writes to a variable collective, call +nc_var_par_access(). \page tutorial_ncids Numbering of NetCDF IDs @@ -846,7 +853,7 @@ may change the next time the file is opened. To construct a netCDF file you need to: - create the file - Specify the name, optionally the format: classic -(the default) or 64bit-offset. +(the default), 64bit-offset, or 64-bit data. - define metadata - Specify the names and types of dimensions, data variables, and attributes. - write data - Write arrays of data from program variables to the @@ -914,7 +921,7 @@ attributes, and the unlimited dimid. num attributes. - nc_inq_dimid() Find dimension ID from its name. - nc_inq_varid() Find variable ID from its name. -- nc_inq_format() Find file format: classic or 64-bit offset +- nc_inq_format() Find file format: classic CDF-1, 64-bit offset CDF-2, or 64-bit data CDF-5 - nc_inq_libvers() Find the netCDF library version. \page accessing_subsets Reading and Writing Subsets of Data diff --git a/docs/types.dox b/docs/types.dox index c09c17a6a..229b598b4 100644 --- a/docs/types.dox +++ b/docs/types.dox @@ -12,8 +12,8 @@ a user-defined data type (see \ref user_defined_types). The atomic external types supported by the netCDF interface are: - ::NC_BYTE 8-bit signed integer -- ::NC_UBYTE 8-bit unsigned integer -- ::NC_CHAR 8-bit character byte +- ::NC_UBYTE 8-bit unsigned integer * +- ::NC_CHAR 8-bit character - ::NC_SHORT 16-bit signed integer - ::NC_USHORT 16-bit unsigned integer * - ::NC_INT (or ::NC_LONG) 32-bit signed integer @@ -24,7 +24,7 @@ The atomic external types supported by the netCDF interface are: - ::NC_DOUBLE 64-bit floating point - ::NC_STRING variable length character string + -\remark * These types are available only for CDF5 (NC_CDF5) and netCDF-4 format (NC_NETCDF4) files. All the unsigned ints (except \ref NC_CHAR) and the 64-bit ints are for CDF5 or netCDF-4 files only. +\remark * These types are available only for CDF5 (NC_CDF5) and netCDF-4 format (NC_NETCDF4) files. All the unsigned ints and the 64-bit ints are for CDF5 or netCDF-4 files only. \remark + These types are available only for netCDF-4 (NC_NETCDF4) files. These types were chosen to provide a reasonably wide range of @@ -73,7 +73,7 @@ avoid such precision loss, check the external types of the variables you access to make sure you use an internal type that has adequate precision. -The names for the primitive external data types (byte, char, short, +The names for the primitive external data types (char, byte, ubyte, short, ushort, int, uint, int64, uint64, float or real, double, string) are reserved words in CDL, so the names of variables, dimensions, and attributes must not be type names. @@ -85,16 +85,16 @@ converted into other numeric types, it is interpreted as signed. For the correspondence between netCDF external data types and the data types of a language see \ref variables. -\section classic_structures Data Structures in Classic and 64-bit Offset Files +\section classic_structures Data Structures in Classic Files The only kind of data structure directly supported by the netCDF -classic (and 64-bit offset) abstraction is a collection of named +classic abstraction, i.e. CDF-1, 2, and 5 formats, is a collection of named arrays with attached vector attributes. NetCDF is not particularly well-suited for storing linked lists, trees, sparse matrices, ragged arrays or other kinds of data structures requiring pointers. It is possible to build other kinds of data structures in netCDF -classic or 64-bit offset formats, from sets of arrays by adopting +classic formats, from sets of arrays by adopting various conventions regarding the use of data in one array as pointers into another array. The netCDF library won't provide much help or hindrance with constructing such data structures, but netCDF provides @@ -116,7 +116,7 @@ arrays. See below.) \endcode As another example, netCDF variables may be grouped within a netCDF -classic or 64-bit offset dataset by defining attributes that list the +classic dataset by defining attributes that list the names of the variables in each group, separated by a conventional delimiter such as a space or comma. Using a naming convention for attribute names for such groupings permits any number of named groups diff --git a/include/ncdispatch.h b/include/ncdispatch.h index 8c65bc7cf..bb6d40aec 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -108,7 +108,7 @@ typedef struct NC_MPI_INFO { /* Define known dispatch tables and initializers */ /*Forward*/ -typedef struct NC_Dispatch NC_Dispatch; +// typedef struct NC_Dispatch NC_Dispatch; extern int NCDISPATCH_initialize(void); extern int NCDISPATCH_finalize(void); diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 99c926e5c..e6f416701 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -372,7 +372,7 @@ and attributes. NC_NOCLOBBER (do not overwrite existing file), NC_SHARE (limit write caching - netcdf classic files only), NC_64BIT_OFFSET (create 64-bit offset file), - NC_64BIT_DATA (Alias NC_CDF5) (create CDF-5 file), + NC_64BIT_DATA (alias NC_CDF5) (create CDF-5 file), NC_NETCDF4 (create netCDF-4/HDF5 file), NC_CLASSIC_MODEL (enforce netCDF classic mode on netCDF-4/HDF5 files), NC_DISKLESS (store data in memory), @@ -408,14 +408,10 @@ file, instead of a netCDF classic format file. The 64-bit offset format imposes far fewer restrictions on very large (i.e. over 2 GB) data files. See Large File Support. -Setting NC_64BIT_DATA (Alias NC_CDF5) causes netCDF to create a CDF-5 +Setting NC_64BIT_DATA (alias NC_CDF5) causes netCDF to create a CDF-5 file format that supports large files (i.e. over 2GB) and large variables (over 2B array elements.). See Large File Support. -Note that the flag NC_PNETCDF also exists as the combination of -NC_CDF5 or'd with NC_MPIIO to indicate that the pnetcdf library -should be used. - A zero value (defined for convenience as NC_CLOBBER) specifies the default behavior: overwrite any existing dataset with the same file name and buffer and cache accesses for efficiency. The dataset will be @@ -425,7 +421,7 @@ Setting NC_NETCDF4 causes netCDF to create a HDF5/NetCDF-4 file. Setting NC_CLASSIC_MODEL causes netCDF to enforce the classic data model in this file. (This only has effect for netCDF-4/HDF5 files, as -classic and 64-bit offset files always use the classic model.) When +CDF-1, 2 and 5 files always use the classic model.) When used with NC_NETCDF4, this flag ensures that the resulting netCDF-4/HDF5 file may never contain any new constructs from the enhanced data model. That is, it cannot contain groups, user defined @@ -580,7 +576,7 @@ nc_create(const char *path, int cmode, int *ncidp) * @param initialsz On some systems, and with custom I/O layers, it * may be advantageous to set the size of the output file at creation * time. This parameter sets the initial size of the file at creation - * time. This only applies to classic and 64-bit offset files. The + * time. This only applies to classic CDF-1, 2, and 5 files. The * special value NC_SIZEHINT_DEFAULT (which is the value 0), lets the * netcdf library choose a suitable initial size. * @param chunksizehintp A pointer to the chunk size hint, which @@ -597,7 +593,7 @@ nc_create(const char *path, int cmode, int *ncidp) * call to discover the system pagesize, we just set default bufrsize * to 8192. The bufrsize is a property of a given open netcdf * descriptor ncid, it is not a persistent property of the netcdf - * dataset. This only applies to classic and 64-bit offset files. + * dataset. This only applies to classic files. * @param ncidp Pointer to location where returned netCDF ID is to be * stored. * @@ -704,7 +700,7 @@ nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp) * ignored for other files. * @param basepe Deprecated parameter from the Cray days. * @param chunksizehintp A pointer to the chunk size hint. This only - * applies to classic and 64-bit offset files. + * applies to classic files. * @param ncidp Pointer that gets ncid. * * @return ::NC_NOERR No error. @@ -723,7 +719,7 @@ nc__create_mp(const char *path, int cmode, size_t initialsz, * * This function opens an existing netCDF dataset for access. It * determines the underlying file format automatically. Use the same - * call to open a netCDF classic, 64-bit offset, or netCDF-4 file. + * call to open a netCDF classic or netCDF-4 file. * * @param path File name for netCDF dataset to be opened. When DAP * support is enabled, then the path may be an OPeNDAP URL rather than @@ -746,7 +742,7 @@ nc__create_mp(const char *path, int cmode, size_t initialsz, * renaming dimensions, variables, and attributes, or deleting * attributes.) * - * The NC_SHARE flag is only used for netCDF classic and 64-bit offset + * The NC_SHARE flag is only used for netCDF classic * files. It is appropriate when one process may be writing the * dataset and one or more other processes reading the dataset * concurrently; it means that dataset accesses are not buffered and @@ -847,7 +843,7 @@ file path. access) and NC_SHARE as in nc_open(). \param chunksizehintp A size hint for the classic library. Only -applies to classic and 64-bit offset files. See below for more +applies to classic files. See below for more information. \param ncidp Pointer to location where returned netCDF ID is to be @@ -1033,7 +1029,7 @@ nc_open_memio(const char* path, int mode, NC_memio* params, int* ncidp) * @param mode Open mode. * @param basepe Deprecated parameter from the Cray days. * @param chunksizehintp A pointer to the chunk size hint. This only - * applies to classic and 64-bit offset files. + * applies to classic files. * @param ncidp Pointer to location where returned netCDF ID is to be * stored. * @@ -2005,11 +2001,11 @@ check_create_mode(int mode) * @param path0 The file name of the new netCDF dataset. * @param cmode The creation mode flag, the same as in nc_create(). * @param initialsz This parameter sets the initial size of the file - * at creation time. This only applies to classic and 64-bit offset + * at creation time. This only applies to classic * files. * @param basepe Deprecated parameter from the Cray days. * @param chunksizehintp A pointer to the chunk size hint. This only - * applies to classic and 64-bit offset files. + * applies to classic files. * @param useparallel Non-zero if parallel I/O is to be used on this * file. * @param parameters Pointer to MPI comm and info. From 43173957d0af712911077fcea317821cc5f2448c Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Fri, 6 Jul 2018 08:04:11 -0400 Subject: [PATCH 08/28] Eliminate compiler warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `typedef struct NC_Dispatch NC_Dispatch;` declaration is in both netcdf.h and in ncdispatch.h. Even though it is the same declaration, there are compilers which see this as an error and the compilation will fail: ``` In file included from /netcdf-c/include/nc4dispatch.h:15, from /netcdf-c/libhdf5/hdf5attr.c:16: /netcdf-c/include/ncdispatch.h:111: error: redefinition of typedef €`NC_Dispatch` /netcdf/netcdf-c/include/netcdf.h:515: note: previous declaration of `NC_Dispatch` was here ``` This is with a very old version of gcc-4.4.7 which isn't used much anymore, but this is the default gcc compiler on RHEL6 so it might be worth fixing. --- include/ncdispatch.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/ncdispatch.h b/include/ncdispatch.h index 8c65bc7cf..c76fe2782 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -107,9 +107,6 @@ typedef struct NC_MPI_INFO { /* Define known dispatch tables and initializers */ -/*Forward*/ -typedef struct NC_Dispatch NC_Dispatch; - extern int NCDISPATCH_initialize(void); extern int NCDISPATCH_finalize(void); From 1eb2504043031ab9cd9b07fb1f4f767eda49c7a0 Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Tue, 10 Jul 2018 08:12:04 -0400 Subject: [PATCH 09/28] Fix missing ERR in tst_parallel5.c Missing the ERR macro for a few tests in tst_parallel5.c --- nc_test4/tst_parallel5.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nc_test4/tst_parallel5.c b/nc_test4/tst_parallel5.c index 51650ca8c..253570a42 100644 --- a/nc_test4/tst_parallel5.c +++ b/nc_test4/tst_parallel5.c @@ -178,10 +178,10 @@ main(int argc, char **argv) /* Create a compound type. */ if (nc_def_compound(ncid, sizeof(struct crew), COMPOUND_NAME, &typeid)) ERR; - if (nc_insert_array_compound(ncid, typeid, "name", NC_COMPOUND_OFFSET(struct crew, name), NC_CHAR, 1, &dim_size)); - if (nc_insert_array_compound(ncid, typeid, "description", NC_COMPOUND_OFFSET(struct crew, description), NC_CHAR, 1, &dim_size)); - if (nc_insert_array_compound(ncid, typeid, "origin", NC_COMPOUND_OFFSET(struct crew, origin), NC_CHAR, 1, &dim_size)); - if (nc_insert_compound(ncid, typeid, "age", NC_COMPOUND_OFFSET(struct crew, age), NC_INT)); + if (nc_insert_array_compound(ncid, typeid, "name", NC_COMPOUND_OFFSET(struct crew, name), NC_CHAR, 1, &dim_size)) ERR; + if (nc_insert_array_compound(ncid, typeid, "description", NC_COMPOUND_OFFSET(struct crew, description), NC_CHAR, 1, &dim_size)) ERR; + if (nc_insert_array_compound(ncid, typeid, "origin", NC_COMPOUND_OFFSET(struct crew, origin), NC_CHAR, 1, &dim_size)) ERR; + if (nc_insert_compound(ncid, typeid, "age", NC_COMPOUND_OFFSET(struct crew, age), NC_INT)) ERR; /* Create one var. */ if (nc_def_var(ncid, COMPOUND_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR; From 786c5a8f2ec11a750f746e6eeb7b2a5d97d547c3 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 Jul 2018 07:05:21 -0600 Subject: [PATCH 10/28] moved hdf5 specific header stuff to hdf5internal.h --- include/Makefile.am | 6 ++--- include/hdf5internal.h | 55 ++++++++++++++++++++++++++++++++++++++++++ include/nc4internal.h | 45 ++++++---------------------------- include/ncdimscale.h | 2 ++ libhdf5/hdf5attr.c | 1 + libhdf5/hdf5dim.c | 1 + libhdf5/hdf5file.c | 1 + libhdf5/hdf5grp.c | 1 + libhdf5/hdf5internal.c | 1 + libhdf5/hdf5type.c | 1 + libhdf5/nc4hdf.c | 1 + libhdf5/nc4info.c | 1 + libsrc4/nc4attr.c | 1 + libsrc4/nc4var.c | 1 + nc_test4/tst_atts.c | 1 + 15 files changed, 79 insertions(+), 40 deletions(-) create mode 100644 include/hdf5internal.h diff --git a/include/Makefile.am b/include/Makefile.am index dceffa5dc..af0059951 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -17,9 +17,9 @@ include_HEADERS += netcdf_mem.h noinst_HEADERS = nc_logging.h nc_tests.h fbits.h nc.h nclist.h \ ncuri.h ncutf8.h ncdispatch.h ncdimscale.h netcdf_f.h err_macros.h \ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \ -nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h \ -ncauth.h ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h \ -ncexternl.h ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h +nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ +ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ +ncwinpath.h ncfilter.h ncindex.h hdf4dispatch.h hdf5internal.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h new file mode 100644 index 000000000..128f4ecce --- /dev/null +++ b/include/hdf5internal.h @@ -0,0 +1,55 @@ +/* Copyright 2005-2018 University Corporation for Atmospheric + Research/Unidata. */ +/** + * @file This header file contains macros, types, and prototypes for + * the HDF5 code in libhdf5. This header should not be included in + * code outside libhdf5. + * + * @author Ed Hartnett +*/ + +#ifndef _HDF5INTERNAL_ +#define _HDF5INTERNAL_ + +#include "config.h" +#include "nc4internal.h" +#include +#include "ncdimscale.h" + +#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) + +/* These have to do with creating chuncked datasets in HDF5. */ +#define NC_HDF5_UNLIMITED_DIMSIZE (0) +#define NC_HDF5_CHUNKSIZE_FACTOR (10) +#define NC_HDF5_MIN_CHUNK_SIZE (2) + +#define NC_EMPTY_SCALE "NC_EMPTY_SCALE" + +/* This is an attribute I had to add to handle multidimensional + * coordinate variables. */ +#define COORDINATES "_Netcdf4Coordinates" +#define COORDINATES_LEN (NC_MAX_NAME * 5) + +/* This is used when the user defines a non-coordinate variable with + * same name as a dimension. */ +#define NON_COORD_PREPEND "_nc4_non_coord_" + +/* An attribute in the HDF5 root group of this name means that the + * file must follow strict netCDF classic format rules. */ +#define NC3_STRICT_ATT_NAME "_nc3_strict" + +/* If this attribute is present on a dimscale variable, use the value + * as the netCDF dimid. */ +#define NC_DIMID_ATT_NAME "_Netcdf4Dimid" + +/** This is the name of the class HDF5 dimension scale attribute. */ +#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS" + +/** This is the name of the name HDF5 dimension scale attribute. */ +#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME" + +/* These functions do HDF5 things. */ +int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); +int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); + +#endif /* _HDF5INTERNAL_ */ diff --git a/include/nc4internal.h b/include/nc4internal.h index e9aea2c40..6dbefd438 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -1,8 +1,10 @@ /* Copyright 2005-2018 University Corporation for Atmospheric Research/Unidata. */ /** - * @file This header file contains the definitions of structs used to - * hold netCDF file metadata in memory. + * @file This header file contains macros, types and prototypes used + * to build and manipulate the netCDF metadata model. + * + * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher */ #ifndef _NC4INTERNAL_ @@ -14,7 +16,6 @@ #include #include #include -#include #include "ncdimscale.h" #include "nc_logging.h" @@ -37,7 +38,7 @@ typedef enum {GET, PUT} NC_PG_T; typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; -#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) +/* #define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) */ #define NC_V2_ERR (-1) /* The name of the root group. */ @@ -74,36 +75,6 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; #define X_DOUBLE_MAX 1.7976931348623157e+308 #define X_DOUBLE_MIN (-X_DOUBLE_MAX) -/* These have to do with creating chuncked datasets in HDF5. */ -#define NC_HDF5_UNLIMITED_DIMSIZE (0) -#define NC_HDF5_CHUNKSIZE_FACTOR (10) -#define NC_HDF5_MIN_CHUNK_SIZE (2) - -#define NC_EMPTY_SCALE "NC_EMPTY_SCALE" - -/* This is an attribute I had to add to handle multidimensional - * coordinate variables. */ -#define COORDINATES "_Netcdf4Coordinates" -#define COORDINATES_LEN (NC_MAX_NAME * 5) - -/* This is used when the user defines a non-coordinate variable with - * same name as a dimension. */ -#define NON_COORD_PREPEND "_nc4_non_coord_" - -/* An attribute in the HDF5 root group of this name means that the - * file must follow strict netCDF classic format rules. */ -#define NC3_STRICT_ATT_NAME "_nc3_strict" - -/* If this attribute is present on a dimscale variable, use the value - * as the netCDF dimid. */ -#define NC_DIMID_ATT_NAME "_Netcdf4Dimid" - -/** This is the name of the class HDF5 dimension scale attribute. */ -#define HDF5_DIMSCALE_CLASS_ATT_NAME "CLASS" - -/** This is the name of the name HDF5 dimension scale attribute. */ -#define HDF5_DIMSCALE_NAME_ATT_NAME "NAME" - /** This is the number of netCDF atomic types. */ #define NUM_ATOMIC_TYPES (NC_MAX_ATOMIC_TYPE + 1) @@ -352,8 +323,8 @@ int nc4_convert_type(const void *src, void *dest, const nc_type src_type, const void *fill_value, int strict_nc3); /* These functions do HDF5 things. */ -int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); -int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); +/* int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); */ +/* int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); */ int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim); int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset); int nc4_put_vars(NC *nc, int ncid, int varid, const size_t *startp, @@ -521,4 +492,4 @@ extern int NC4_hdf5get_libversion(unsigned*,unsigned*,unsigned*);/*libsrc4/nc4hd extern int NC4_hdf5get_superblock(struct NC_FILE_INFO*, int*);/*libsrc4/nc4hdf.c*/ extern int NC4_isnetcdf4(struct NC_FILE_INFO*); /*libsrc4/nc4hdf.c*/ -#endif /* _NETCDF4_ */ +#endif /* _NC4INTERNAL_ */ diff --git a/include/ncdimscale.h b/include/ncdimscale.h index 3738bb165..cd88d3ec3 100644 --- a/include/ncdimscale.h +++ b/include/ncdimscale.h @@ -8,6 +8,8 @@ #ifndef _NCDIMSCALE_H_ #define _NCDIMSCALE_H_ +#include + typedef struct hdf5_objid { unsigned long fileno[2]; /* file number */ diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 2ffbd6398..9ed2d7f87 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -13,6 +13,7 @@ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "ncdispatch.h" diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index ec109a904..b052985c3 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -12,6 +12,7 @@ */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index c85b3d42f..c235f30ac 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -15,6 +15,7 @@ #include /* netcdf functions sometimes return system errors */ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "netcdf_mem.h" #ifdef USE_HDF4 diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index 17f39f074..dd0eae1d5 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -12,6 +12,7 @@ */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 8828687d3..92175db6a 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -15,6 +15,7 @@ */ #include "config.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc.h" /* from libsrc */ #include "ncdispatch.h" /* from libdispatch */ #include "ncutf8.h" diff --git a/libhdf5/hdf5type.c b/libhdf5/hdf5type.c index 9a113e4bd..c86a283a3 100644 --- a/libhdf5/hdf5type.c +++ b/libhdf5/hdf5type.c @@ -12,6 +12,7 @@ * @author Ed Hartnett */ #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" /** diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 3c220a60a..dc3d8f984 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -15,6 +15,7 @@ #include "config.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include #include diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index f754aa2d5..b2286bc0c 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -12,6 +12,7 @@ #include #include "netcdf.h" #include "nc4internal.h" +#include "hdf5internal.h" #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */ diff --git a/libsrc4/nc4attr.c b/libsrc4/nc4attr.c index 6a2090141..fe4fd03e8 100644 --- a/libsrc4/nc4attr.c +++ b/libsrc4/nc4attr.c @@ -18,6 +18,7 @@ #include "nc.h" #include "nc4internal.h" +#include "hdf5internal.h" #include "nc4dispatch.h" #include "ncdispatch.h" diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index c0d95d862..137e3023f 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -12,6 +12,7 @@ #include #include "nc4dispatch.h" +#include "hdf5internal.h" #include /* Szip options. */ diff --git a/nc_test4/tst_atts.c b/nc_test4/tst_atts.c index 4090dc2e8..4487ebf11 100644 --- a/nc_test4/tst_atts.c +++ b/nc_test4/tst_atts.c @@ -11,6 +11,7 @@ #include #include "err_macros.h" #include "nc4internal.h" +#include "hdf5internal.h" /* The data file we will create. */ #define FILE_NAME "tst_atts.nc" From 4484c4896474213f350b552359f4f2a94b18223b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 Jul 2018 07:06:40 -0600 Subject: [PATCH 11/28] clean up --- include/nc4internal.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 6dbefd438..9b9626b7f 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -38,7 +38,6 @@ typedef enum {GET, PUT} NC_PG_T; typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT; -/* #define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) */ #define NC_V2_ERR (-1) /* The name of the root group. */ From e00b2ee5574e0e0e481c96311b2dcd1d0954799c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 Jul 2018 07:07:23 -0600 Subject: [PATCH 12/28] clean up --- include/nc4internal.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/nc4internal.h b/include/nc4internal.h index 9b9626b7f..b6fd866e1 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -322,8 +322,6 @@ int nc4_convert_type(const void *src, void *dest, const nc_type src_type, const void *fill_value, int strict_nc3); /* These functions do HDF5 things. */ -/* int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); */ -/* int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid); */ int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim); int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset); int nc4_put_vars(NC *nc, int ncid, int varid, const size_t *startp, From 3a08d85e13686003ded9b45a0acb42ca1da289cb Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 12 Jul 2018 07:08:27 -0600 Subject: [PATCH 13/28] clean up --- nc_test4/tst_atts.c | 1 - 1 file changed, 1 deletion(-) diff --git a/nc_test4/tst_atts.c b/nc_test4/tst_atts.c index 4487ebf11..44e94808a 100644 --- a/nc_test4/tst_atts.c +++ b/nc_test4/tst_atts.c @@ -10,7 +10,6 @@ #include #include #include "err_macros.h" -#include "nc4internal.h" #include "hdf5internal.h" /* The data file we will create. */ From 2dd0ac63198c5978bc9a9022a6efd35ea7a9f63e Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 04:50:15 -0600 Subject: [PATCH 14/28] clean up --- include/hdf5internal.h | 4 +++- libhdf5/hdf5attr.c | 5 +---- libhdf5/hdf5dim.c | 3 +-- libhdf5/hdf5file.c | 9 --------- libhdf5/hdf5grp.c | 3 +-- libhdf5/hdf5internal.c | 6 +----- libhdf5/hdf5type.c | 4 ++-- libhdf5/nc4hdf.c | 3 --- libhdf5/nc4info.c | 6 +----- 9 files changed, 10 insertions(+), 33 deletions(-) diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 128f4ecce..1573ca6b1 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -12,9 +12,11 @@ #define _HDF5INTERNAL_ #include "config.h" -#include "nc4internal.h" #include +#include +#include "nc4internal.h" #include "ncdimscale.h" +#include "nc4dispatch.h" #define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10) diff --git a/libhdf5/hdf5attr.c b/libhdf5/hdf5attr.c index 9ed2d7f87..aa60865d6 100644 --- a/libhdf5/hdf5attr.c +++ b/libhdf5/hdf5attr.c @@ -11,11 +11,8 @@ * @author Ed Hartnett */ -#include "nc.h" -#include "nc4internal.h" +#include "config.h" #include "hdf5internal.h" -#include "nc4dispatch.h" -#include "ncdispatch.h" int nc4typelen(nc_type type); diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index b052985c3..fe8d95911 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -11,9 +11,8 @@ * @author Ed Hartnett */ -#include "nc4internal.h" +#include "config.h" #include "hdf5internal.h" -#include "nc4dispatch.h" /** * @internal Dimensions are defined in attributes attached to the diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index 51423d29e..1d816062a 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -12,16 +12,7 @@ */ #include "config.h" -#include /* netcdf functions sometimes return system errors */ -#include "nc.h" -#include "nc4internal.h" #include "hdf5internal.h" -#include "nc4dispatch.h" -#include "netcdf_mem.h" -#ifdef USE_HDF4 -#include -#endif -#include extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); diff --git a/libhdf5/hdf5grp.c b/libhdf5/hdf5grp.c index dd0eae1d5..cc967d354 100644 --- a/libhdf5/hdf5grp.c +++ b/libhdf5/hdf5grp.c @@ -11,9 +11,8 @@ * @author Ed Hartnett */ -#include "nc4internal.h" +#include "config.h" #include "hdf5internal.h" -#include "nc4dispatch.h" /** * @internal Create a group. Its ncid is returned in the new_ncid diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index 92175db6a..c0dbeedb0 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -13,13 +13,9 @@ * * @author Ed Hartnett */ + #include "config.h" -#include "nc4internal.h" #include "hdf5internal.h" -#include "nc.h" /* from libsrc */ -#include "ncdispatch.h" /* from libdispatch */ -#include "ncutf8.h" -#include "H5DSpublic.h" #undef DEBUGH5 diff --git a/libhdf5/hdf5type.c b/libhdf5/hdf5type.c index c86a283a3..e955721c6 100644 --- a/libhdf5/hdf5type.c +++ b/libhdf5/hdf5type.c @@ -11,9 +11,9 @@ * * @author Ed Hartnett */ -#include "nc4internal.h" + +#include "config.h" #include "hdf5internal.h" -#include "nc4dispatch.h" /** * @internal Determine if two types are equal. diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index dc3d8f984..0f20e1057 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -14,10 +14,7 @@ */ #include "config.h" -#include "nc4internal.h" #include "hdf5internal.h" -#include "nc4dispatch.h" -#include #include #ifdef HAVE_INTTYPES_H diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index b2286bc0c..866fe450d 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -6,12 +6,8 @@ * and redistribution conditions. * @author Dennis Heimbigner */ + #include "config.h" -#include -#include -#include -#include "netcdf.h" -#include "nc4internal.h" #include "hdf5internal.h" #define HDF5_MAX_NAME 1024 /**< HDF5 max name. */ From 7fdf22f2a79b9bbaf8072f727092b3200adb89a7 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 06:23:16 -0600 Subject: [PATCH 15/28] fixing parens --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0afe01f7f..f578edebd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -535,7 +535,7 @@ IF(ENABLE_LOGGING) SET(LOGGING ON) ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) -IF(ENABLE_NETCDF_$ AND (NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING) +IF(ENABLE_NETCDF_$ AND ((NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING)) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) ENDIF() From 987ce47f5e2b917ba960d386a0c150ffefa28888 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 06:24:57 -0600 Subject: [PATCH 16/28] removed duplicate handling of logging --- CMakeLists.txt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0afe01f7f..ce6b6c3e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -381,14 +381,6 @@ OPTION(ENABLE_MMAP "Use MMAP." ON) # Option to use examples. OPTION(ENABLE_EXAMPLES "Build Examples" ON) - -# Option Logging, only valid for netcdf4. -OPTION(ENABLE_LOGGING "Enable Logging." OFF) -IF(ENABLE_LOGGING) - ADD_DEFINITIONS(-DLOGGING) - SET(LOGGING ON) -ENDIF() - # Option to automatically build netcdf-fortran. IF(NOT MSVC) OPTION(ENABLE_REMOTE_FORTRAN_BOOTSTRAP "Download and build netcdf-fortran automatically (EXPERIMENTAL)." OFF) From 03de993ce5b260148532ecb3bb6ad604e92a09b0 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 06:31:44 -0600 Subject: [PATCH 17/28] attempting to fix ENABLE_SET_LOG_LEVEL problem with cmake build --- config.h.cmake.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.h.cmake.in b/config.h.cmake.in index 611ceb7a0..9313fc7cd 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -367,6 +367,9 @@ are set when opening a binary file on Windows. */ /* If true, turn on logging. */ #cmakedefine LOGGING 1 +/* If true, define nc_set_log_level. */ +#cmakedefine ENABLE_SET_LOG_LEVEL 1 + /* max size of the default per-var chunk cache. */ #cmakedefine MAX_DEFAULT_CACHE_SIZE ${MAX_DEFAULT_CACHE_SIZE} From 82d0302fa9852dee14bac66c8cc70f1648e10d0a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 06:39:15 -0600 Subject: [PATCH 18/28] attempting to fix ENABLE_SET_LOG_LEVEL problem with cmake build --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index ce6b6c3e0..18bf82c15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -529,6 +529,7 @@ ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) IF(ENABLE_NETCDF_$ AND (NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) + SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() # Option to allow for strict null file padding. From cdfa6abad813a1c6523b5bf649883ff44f844a3b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 08:34:00 -0600 Subject: [PATCH 19/28] attempting to fix ENABLE_SET_LOG_LEVEL problem with cmake build --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 18bf82c15..a3ec745ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -531,6 +531,8 @@ IF(ENABLE_NETCDF_$ AND (NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENA ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() +ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) +SET(ENABLE_SET_LOG_LEVEL ON) # Option to allow for strict null file padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information From 966fec740d4a293a54166aaf10c89f859e77171c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 08:39:23 -0600 Subject: [PATCH 20/28] attempting to fix ENABLE_SET_LOG_LEVEL problem with cmake build --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a3ec745ce..9afd3610f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,14 +525,13 @@ IF(ENABLE_LOGGING) ADD_DEFINITIONS(-DLOGGING) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) SET(LOGGING ON) + SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) -IF(ENABLE_NETCDF_$ AND (NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING) +IF(NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() -ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) -SET(ENABLE_SET_LOG_LEVEL ON) # Option to allow for strict null file padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information From 9ac15aac468fc9de84867e84ecc50feb32efae1a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 11:45:26 -0600 Subject: [PATCH 21/28] attempting to fix ENABLE_SET_LOG_LEVEL problem with cmake build --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9afd3610f..5d59b024c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,7 @@ IF(ENABLE_LOGGING) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) -IF(NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) +IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() From c4196ce5ec0435139f991869a98a62253abd7e6d Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 11:52:03 -0600 Subject: [PATCH 22/28] moving definition of nc_set_log_level() to nothing --- include/nc_logging.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/nc_logging.h b/include/nc_logging.h index 6c17152fb..74ad4dc9a 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -52,12 +52,13 @@ void nc_log_hdf5(void); #define BAIL_QUIET BAIL +#endif /* LOGGING */ + #ifndef ENABLE_SET_LOG_LEVEL +/* Define away any calls to nc_set_log_level(), if its not enabled. */ #define nc_set_log_level(e) #endif /* ENABLE_SET_LOG_LEVEL */ -#endif /* LOGGING */ - /* To log an error message (if 'LOGGING' is defined), set retval, and jump to exit. */ #define BAIL(e) \ do { \ From 6a27bbcec6c416dc17c8edbba5a874c11db9f46b Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 12:09:40 -0600 Subject: [PATCH 23/28] moving definition of nc_set_log_level() to nothing --- include/nc_logging.h | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/nc_logging.h b/include/nc_logging.h index 74ad4dc9a..ec6dfae15 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -42,6 +42,10 @@ void nc_log_hdf5(void); #else /* LOGGING */ /* These definitions will be used unless LOGGING is defined. */ +#ifndef ENABLE_SET_LOG_LEVEL +/* Define away any calls to nc_set_log_level(), if its not enabled. */ +#define nc_set_log_level(e) +#endif /* ENABLE_SET_LOG_LEVEL */ #define LOG(e) @@ -54,11 +58,6 @@ void nc_log_hdf5(void); #endif /* LOGGING */ -#ifndef ENABLE_SET_LOG_LEVEL -/* Define away any calls to nc_set_log_level(), if its not enabled. */ -#define nc_set_log_level(e) -#endif /* ENABLE_SET_LOG_LEVEL */ - /* To log an error message (if 'LOGGING' is defined), set retval, and jump to exit. */ #define BAIL(e) \ do { \ From 413f8c288660ee7b2123c9f2b2638490d16c1c26 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 13:36:03 -0600 Subject: [PATCH 24/28] moving definition of nc_set_log_level() to nothing --- include/nc_logging.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/include/nc_logging.h b/include/nc_logging.h index ec6dfae15..c4735b71f 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -42,11 +42,6 @@ void nc_log_hdf5(void); #else /* LOGGING */ /* These definitions will be used unless LOGGING is defined. */ -#ifndef ENABLE_SET_LOG_LEVEL -/* Define away any calls to nc_set_log_level(), if its not enabled. */ -#define nc_set_log_level(e) -#endif /* ENABLE_SET_LOG_LEVEL */ - #define LOG(e) #define BAIL2(e) \ @@ -56,6 +51,13 @@ void nc_log_hdf5(void); #define BAIL_QUIET BAIL +#ifdef USE_NETCDF_4 +#ifndef ENABLE_SET_LOG_LEVEL +/* Define away any calls to nc_set_log_level(), if its not enabled. */ +#define nc_set_log_level(e) +#endif /* ENABLE_SET_LOG_LEVEL */ +#endif + #endif /* LOGGING */ /* To log an error message (if 'LOGGING' is defined), set retval, and jump to exit. */ From 52d7985aec2e9c0f453cddadcb0e1126fcc7e212 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 13:41:36 -0600 Subject: [PATCH 25/28] moving definition of nc_set_log_level() to nothing --- include/nc_logging.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/nc_logging.h b/include/nc_logging.h index c4735b71f..27628d42d 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -51,12 +51,10 @@ void nc_log_hdf5(void); #define BAIL_QUIET BAIL -#ifdef USE_NETCDF_4 #ifndef ENABLE_SET_LOG_LEVEL /* Define away any calls to nc_set_log_level(), if its not enabled. */ #define nc_set_log_level(e) #endif /* ENABLE_SET_LOG_LEVEL */ -#endif #endif /* LOGGING */ From 99c4f82f2e2a0ea2393f27eeed8d45e463296ef9 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 13:47:18 -0600 Subject: [PATCH 26/28] only define nc_set_log_level() to nothing for netcdf-4 builds --- include/nc_logging.h | 2 ++ ncdump/tst_dimsizes.c | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/nc_logging.h b/include/nc_logging.h index 27628d42d..c4735b71f 100644 --- a/include/nc_logging.h +++ b/include/nc_logging.h @@ -51,10 +51,12 @@ void nc_log_hdf5(void); #define BAIL_QUIET BAIL +#ifdef USE_NETCDF_4 #ifndef ENABLE_SET_LOG_LEVEL /* Define away any calls to nc_set_log_level(), if its not enabled. */ #define nc_set_log_level(e) #endif /* ENABLE_SET_LOG_LEVEL */ +#endif #endif /* LOGGING */ diff --git a/ncdump/tst_dimsizes.c b/ncdump/tst_dimsizes.c index edb0eb80d..eee6c340e 100644 --- a/ncdump/tst_dimsizes.c +++ b/ncdump/tst_dimsizes.c @@ -34,7 +34,6 @@ main(int argc, char **argv) size_t dimsize; int dimid; int stat = NC_NOERR; - nc_set_log_level(5); printf("\n*** Testing Max Dimension Sizes\n"); printf("\n|size_t|=%lu\n",(unsigned long)sizeof(size_t)); From a8faab2f93595e57d3ddba9e3e1c864235e8cffe Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 16 Jul 2018 15:38:45 -0600 Subject: [PATCH 27/28] nc_set_log_level only available for netcdf-4 builds --- nc_test/tst_inmemory.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nc_test/tst_inmemory.c b/nc_test/tst_inmemory.c index 9f979be8e..c91fd4458 100644 --- a/nc_test/tst_inmemory.c +++ b/nc_test/tst_inmemory.c @@ -605,9 +605,8 @@ main(int argc, char **argv) NC_memio filedata3; NC_memio filedata4; - nc_set_log_level(0); - #ifdef USE_NETCDF4 + nc_set_log_level(0); H5Eprint1(stderr); #endif From 0deb4635d5ba522558e8bf283061a5aa3f6de234 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Fri, 20 Jul 2018 10:24:58 -0600 Subject: [PATCH 28/28] fixed master --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f578edebd..0afe01f7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -535,7 +535,7 @@ IF(ENABLE_LOGGING) SET(LOGGING ON) ENDIF() OPTION(ENABLE_SET_LOG_LEVEL_FUNC "Enable definition of nc_set_log_level()." ON) -IF(ENABLE_NETCDF_$ AND ((NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING)) +IF(ENABLE_NETCDF_$ AND (NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) OR ENABLE_LOGGING) ADD_DEFINITIONS(-DENABLE_SET_LOG_LEVEL) ENDIF()