From ac503dd5657efe53583a7bd657eb7ba2c7cdb2d1 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 8 Aug 2017 13:17:51 -0600 Subject: [PATCH 1/9] Updated release notes in support of https://github.com/Unidata/netcdf-c/pull/450 --- RELEASE_NOTES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 57c0ca3ae..89805d092 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,8 +5,14 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. +## 4.5.1 - TBD + + + ## 4.5.0 - TBD +* [Bug Fix] Corrected a potential stack buffer overflow. See [GitHub #450](https://github.com/Unidata/netcdf-c/pull/450) for more information. + ### 4.5.0-rc2 - August 7, 2017 * [Bug Fix] Addressed an issue with how cmake was implementing large file support on 32-bit systems. See [GitHub #385](https://github.com/Unidata/netcdf-c/issues/385) for more information. From 24ca95645c2530040fa9b654f1f94b31b8c26dc7 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 8 Aug 2017 13:18:53 -0600 Subject: [PATCH 2/9] Applied fix for a stack buffer overflow into 4.5.0 release branch. --- libsrc4/nc4internal.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index fe6e2f6a9..89526d840 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -102,6 +102,12 @@ nc4_check_name(const char *name, char *norm_name) retval = nc_utf8_normalize((const unsigned char *)name,(unsigned char**)&temp); if(retval != NC_NOERR) return retval; + + if(strlen(temp) > NC_MAX_NAME) { + free(temp); + return NC_EMAXNAME; + } + strcpy(norm_name, temp); free(temp); @@ -1603,4 +1609,3 @@ NC4_show_metadata(int ncid) #endif /*LOGGING*/ return retval; } - From 7992242a39993a138f9c98653e519e93b1d0d46b Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Tue, 8 Aug 2017 16:54:38 -0500 Subject: [PATCH 3/9] Call nc_close, which calls MPI_File_close, to avoid MPI wraning messages on MPI objects are still allocated. --- nc_test/tst_formatx_pnetcdf.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nc_test/tst_formatx_pnetcdf.c b/nc_test/tst_formatx_pnetcdf.c index b11699f3e..99fa5c517 100644 --- a/nc_test/tst_formatx_pnetcdf.c +++ b/nc_test/tst_formatx_pnetcdf.c @@ -66,6 +66,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } + if (nc_close(ncid)) ERR; /* test CDF-2 file format */ cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_OFFSET; @@ -84,6 +85,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } + if (nc_close(ncid)) ERR; /* test CDF-5 file format */ cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_DATA; @@ -102,8 +104,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } - - if (nc_abort(ncid)) ERR; + if (nc_close(ncid)) ERR; fn_exit: MPI_Finalize(); From b50508479ab37c8e413318cea1a81636b8188a53 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 8 Aug 2017 16:11:13 -0600 Subject: [PATCH 4/9] Added a new product to the static software html page. --- docs/static-pages/software.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/static-pages/software.html b/docs/static-pages/software.html index 719889237..2190b7ee1 100644 --- a/docs/static-pages/software.html +++ b/docs/static-pages/software.html @@ -300,6 +300,9 @@
  • ArcGIS Pro - Space Time Pattern Mining Toolbox
    • +
  • + AgriMetSoft - Netcdf-Extractor +
  • ASA ViewNcDap
    • @@ -2994,6 +2997,13 @@ or using ECMWF reanalysis on a reduced grid

    +

    AgriMetSoft Netcdf-Extractor

    + There are different software packages or various codes that may be used for manipulating or displaying NetCDF data. Some of them such as MATLAB programming language needs to enough information about provide codes in MATLAB as well as to know run codes and statements. Furthermore, user should have to pay for MATLAB programming and install it in her/his system. Also, another of them are related to other programming languages such as R, NCL (NCAR Command Language), and etc. It has been mentioned before that working by many of this methods to extract NetCDF files are so rough and complicated for users. Therefore, it is essential that users can extract their NC files in a user-friendly tool without any concerns. The NetCDF Extractor software is an easy tool for all users to extract their aim region from the main files. This tool is flexible to run for various datasets such as CMIP5 models, AgMERRA datasets, Aphrodite, CRU, CORDEX, and etc. Finally, this tool is NETCDF EXTRACTOR, NETCDF FILE VIEWER, and NETCDF CONVERTER. NetCDF-Extractor website. +

    + + +

    +

    ASA ViewNcDap

    Applied Science Associates, Inc. has made the ASA View NC/Dap From bd33076424e8416af8b5fd569b9e19e33931f37d Mon Sep 17 00:00:00 2001 From: Lars Pastewka Date: Wed, 9 Aug 2017 21:47:08 +0200 Subject: [PATCH 5/9] Fixed computation of recsize for NC_64BIT_DATA files. --- libsrc/v1hpg.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index e464c091f..07c322ce5 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1220,7 +1220,8 @@ NC_computeshapes(NC3_INFO* ncp) if(first_rec == NULL) first_rec = *vpp; if((*vpp)->len == UINT32_MAX && - fIsSet(ncp->flags, NC_64BIT_OFFSET)) /* Flag for large last record */ + (fIsSet(ncp->flags, NC_64BIT_OFFSET) || + fIsSet(ncp->flags, NC_64BIT_DATA))) /* Flag for large last record */ ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz; else ncp->recsize += (*vpp)->len; From 5f0e76b281e55925e179808ab866ea0666c66b49 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 9 Aug 2017 14:12:24 -0600 Subject: [PATCH 6/9] Updated release notes. --- RELEASE_NOTES.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 89805d092..e64bbea32 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -5,12 +5,9 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. -## 4.5.1 - TBD - - - ## 4.5.0 - TBD +* [Bug Fix] Corrected an issue where `NC_64BIT_DATA` files were being read incorrectly by ncdump, despite the data having been written correctly. See [GitHub #457](https://github.com/Unidata/netcdf-c/issues/457) for more information. * [Bug Fix] Corrected a potential stack buffer overflow. See [GitHub #450](https://github.com/Unidata/netcdf-c/pull/450) for more information. ### 4.5.0-rc2 - August 7, 2017 From 96c8e190e7df135792b04136fcb6195b38e85a45 Mon Sep 17 00:00:00 2001 From: edhartnett Date: Thu, 10 Aug 2017 12:44:57 -0600 Subject: [PATCH 7/9] fixed table and link in Large File section --- docs/guide.dox | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/docs/guide.dox b/docs/guide.dox index 0a8dcf637..b3a8c3838 100644 --- a/docs/guide.dox +++ b/docs/guide.dox @@ -1373,20 +1373,16 @@ about 1.15e+18 bytes. Note also that all sizes are really 4 bytes less than the ones given below. For example the maximum size of a fixed variable in netCDF 3.6 classic format is really 2 GiB - 4 bytes. -Limit No LFS v3.5 v3.6/classic v3.6/64-bit offset v4.0/netCDF-4 +Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4 +-------------------------------------------|------------|---------|---------------|---------------------|----------------- +Max File Size | 2 GiB | 8 EiB | 8 EiB | 8 EiB | unlimited +Max Number of Fixed Vars > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited +Max Record Vars w/ Rec Size > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited +Max Size of Fixed/Record Size of Record Var| 2 GiB | 2 GiB | 2 GiB | 4 GiB | unlimited +Max Record Size | 2 GiB/nrecs| 4 GiB | 8 EiB/nrecs | 8 EiB/nrecs | unlimited -Max File Size 2 GiB 8 EiB 8 EiB 8 EiB ?? - -Max Number of Fixed Vars > 2 GiB 0 1 (last) 1 (last) 2^32 ?? - -Max Record Vars w/ Rec Size > 2 GiB 0 1 (last) 1 (last) 2^32 ?? - -Max Size of Fixed/Record Size of Record Var 2 GiB 2 GiB 2 GiB 4 GiB ?? - -Max Record Size 2 GiB/nrecs 4 GiB 8 EiB/nrecs 8 EiB/nrecs ?? - -For more information about the different file formats of netCDF See -Which Format. +For more information about the different file formats of netCDF see +\ref select_format "How to Select the Format". \section offset_format_limitations NetCDF 64-bit Offset Format Limitations From 85e9aaf368a92d152e6294317a9014cf18097728 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 11 Aug 2017 18:18:11 -0600 Subject: [PATCH 8/9] Wiring in a large test to check against a regression for the issue described in https://github.com/Unidata/netcdf-c/pull/457 --- nc_test/CMakeLists.txt | 2 +- nc_test/Makefile.am | 5 +- nc_test/tst_large_cdf5.c | 179 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 nc_test/tst_large_cdf5.c diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt index 209bacdd0..a081aa319 100644 --- a/nc_test/CMakeLists.txt +++ b/nc_test/CMakeLists.txt @@ -51,7 +51,7 @@ IF(USE_PNETCDF) ENDIF() IF(LARGE_FILE_TESTS) - SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large) + SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large tst_large_cdf5) IF(NOT MSVC) SET(TESTS ${TESTS} large_files) ENDIF() diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index 8e03ff31f..87d074e9e 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -16,7 +16,8 @@ tst_*.nc t_nc.nc large_files.nc quick_large_files.nc \ tst_diskless.nc tst_diskless2.nc \ tst_diskless3.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \ tst_diskless4.cdl tst_diskless4.nc tst_formatx.nc nc_test_cdf5.nc \ -unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc +unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc \ +tst_large_cdf5.nc check_PROGRAMS = @@ -45,7 +46,7 @@ AM_CPPFLAGS += -I$(top_builddir)/liblib -I$(top_builddir)/include -I$(top_srcdir # If the user asked for large file tests, then add them. if LARGE_FILE_TESTS TESTPROGRAMS += quick_large_files tst_big_var6 tst_big_var2 \ -tst_big_rvar tst_big_var tst_large large_files +tst_big_rvar tst_big_var tst_large large_files tst_large_cdf5 endif # LARGE_FILE_TESTS if BUILD_BENCHMARKS diff --git a/nc_test/tst_large_cdf5.c b/nc_test/tst_large_cdf5.c new file mode 100644 index 000000000..c45e47e29 --- /dev/null +++ b/nc_test/tst_large_cdf5.c @@ -0,0 +1,179 @@ +/* + Copyright 2017, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. + + This program tests the large file bug in netCDF 3.6.2, + creating byte and short variables larger than 4 GiB. + + This program is based on tst_large.c, and tests against + CDF5. + + $Id: tst_large.c,v 1.16 2010/05/18 20:20:01 russ Exp $ +*/ +#include +#include +#include +#include +#include +#include "err_macros.h" +#include + +/* Test with both classic and 64-bit offset files. If netcdf-4 is + * included, test with both netCDF-4 format variants also. */ +#ifdef USE_NETCDF4 +#define NUM_FORMATS (4) +#else +#define NUM_FORMATS (2) +#endif + +#define NUMDIMS 2 /* rank of each variable in tests */ +#define DIM1 2048 +#define DIM2 2097153 /* DIM1*DIM2*sizeof(char) > 2**32 */ +#define DIM3 1024 +#define DIM4 2097153 /* DIM3*DIM4*sizeof(short) > 2**32 */ + +/* + * In netCDF-3.6.2, a divide by zero occurs on 32-bit platforms when + * creating a variable for which the product of dimensions is exactly + * 2**32. Check that this bug has been fixed. + */ +static int +test_big_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + int cflag = NC_CLOBBER; + nc_type type = NC_BYTE; + size_t index[NUMDIMS]; + signed char nval = 99; + int nval_in; + + /* Define the file with one large variable. */ + if (nc_create(testfile, cflag|NC_64BIT_DATA, &ncid)) ERR; + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim2", DIM2 - 1, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", type, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + + /* Write one datum, near the end of the variable. */ + index[0] = DIM1 - 1; + index[1] = DIM2 - 2; + if (nc_put_var1_schar(ncid, varid, index, &nval)) ERR; + if (nc_close(ncid)) ERR; + + /* Reopen the file and check that datum. */ + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_int(ncid, varid, index, &nval_in)) ERR; + if (nval != nval_in) + ERR; + if (nc_close(ncid)) ERR; + return 0; +} + +static int +test_large_byte_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + size_t index[NUMDIMS] = {0, 0}; + signed char vals[DIM2]; + signed char char_val_in; + size_t start[NUMDIMS], count[NUMDIMS]; + int j; + + if (nc_create(testfile, NC_CLOBBER|NC_64BIT_DATA, &ncid)) ERR; + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim2", DIM2, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", NC_BYTE, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + + for (j = 0; j < DIM2; j++) { + vals[j] = 9 * (j + 11); /* note vals[j] is 99 when j==0 */ + } + start[1] = 0; + count[0] = 1; + count[1] = DIM2; + for (start[0] = 0; start[0] < DIM1; start[0]++) { + if (nc_put_vara_schar(ncid, varid, start, count, vals)) + { + ERR; + break; + } + } + + if (nc_close(ncid)) ERR; + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_schar(ncid, varid, index, &char_val_in)) ERR; + if (char_val_in != 99) /* see above, the value written when start[0]==0, j==0 */ + ERR; + if (nc_close(ncid)) ERR; + return 0; +} + +static int +test_large_short_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + int int_val_in, int_val_out = 99; + size_t index[2]; + int cflag = NC_CLOBBER; + + if (nc_create(testfile, cflag|NC_64BIT_DATA, &ncid)) ERR; + if (nc_def_dim(ncid, "dim3", DIM3, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim4", DIM4, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", NC_SHORT, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + index[0] = 0; + index[1] = 1; + if (nc_put_var1_int(ncid, varid, index, &int_val_out)) ERR; + if (nc_close(ncid)) ERR; + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR; + if (int_val_in != int_val_out) + ERR; +#ifndef NOFILL + index[0] = 1; + index[1] = 2; + if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR; + if (int_val_in != NC_FILL_SHORT) + ERR; +#endif + if (nc_close(ncid)) ERR; + return 0; +} + +#define FILE_NAME "tst_large_cdf5.nc" + +int +main(int argc, char **argv) { + int i; + char testfile[NC_MAX_NAME + 1]; + + sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + + printf("\n*** Testing fix for 3.6.2 large file bug in %s.\n", + testfile); + /* Go thru formats and run all tests for each of two (for netCDF-3 + * only builds), or 4 (for netCDF-4 builds) different formats. */ + for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) + { + nc_set_default_format(i, NULL); + + printf("*** testing format %d with a variable with 2**32 values...", i); + test_big_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a byte variable with > 2**32 values...", i); + test_large_byte_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a short variable with > 2**32 values...", i); + test_large_short_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + } + + FINAL_RESULTS; +} From e006ea5e0786565abbcddffa86ac736ab950fa43 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 11 Aug 2017 18:36:44 -0600 Subject: [PATCH 9/9] Fixed a simple mistake in test organization. --- nc_test/tst_large_cdf5.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/nc_test/tst_large_cdf5.c b/nc_test/tst_large_cdf5.c index c45e47e29..ef8681f0b 100644 --- a/nc_test/tst_large_cdf5.c +++ b/nc_test/tst_large_cdf5.c @@ -47,7 +47,7 @@ test_big_var(const char *testfile) { int nval_in; /* Define the file with one large variable. */ - if (nc_create(testfile, cflag|NC_64BIT_DATA, &ncid)) ERR; + if (nc_create(testfile, cflag, &ncid)) ERR; if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; if (nc_def_dim(ncid, "dim2", DIM2 - 1, &dimids[1])) ERR; @@ -79,7 +79,7 @@ test_large_byte_var(const char *testfile) { size_t start[NUMDIMS], count[NUMDIMS]; int j; - if (nc_create(testfile, NC_CLOBBER|NC_64BIT_DATA, &ncid)) ERR; + if (nc_create(testfile, NC_CLOBBER, &ncid)) ERR; if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; if (nc_def_dim(ncid, "dim2", DIM2, &dimids[1])) ERR; @@ -117,7 +117,7 @@ test_large_short_var(const char *testfile) { size_t index[2]; int cflag = NC_CLOBBER; - if (nc_create(testfile, cflag|NC_64BIT_DATA, &ncid)) ERR; + if (nc_create(testfile, cflag, &ncid)) ERR; if (nc_def_dim(ncid, "dim3", DIM3, &dimids[0])) ERR; if (nc_def_dim(ncid, "dim4", DIM4, &dimids[1])) ERR; if (nc_def_var(ncid, "var", NC_SHORT, NUMDIMS, dimids, &varid)) ERR; @@ -175,5 +175,27 @@ main(int argc, char **argv) { SUMMARIZE_ERR; } + /* Test CDF5 */ + { + i = NC_64BIT_DATA; + nc_set_default_format(i, NULL); + + printf("*** testing format %d with a variable with 2**32 values...", i); + test_big_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a byte variable with > 2**32 values...", i); + test_large_byte_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a short variable with > 2**32 values...", i); + test_large_short_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + } + FINAL_RESULTS; }