diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 57c0ca3ae..e64bbea32 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release ## 4.5.0 - TBD +* [Bug Fix] Corrected an issue where `NC_64BIT_DATA` files were being read incorrectly by ncdump, despite the data having been written correctly. See [GitHub #457](https://github.com/Unidata/netcdf-c/issues/457) for more information. +* [Bug Fix] Corrected a potential stack buffer overflow. See [GitHub #450](https://github.com/Unidata/netcdf-c/pull/450) for more information. + ### 4.5.0-rc2 - August 7, 2017 * [Bug Fix] Addressed an issue with how cmake was implementing large file support on 32-bit systems. See [GitHub #385](https://github.com/Unidata/netcdf-c/issues/385) for more information. diff --git a/docs/guide.dox b/docs/guide.dox index 0a8dcf637..b3a8c3838 100644 --- a/docs/guide.dox +++ b/docs/guide.dox @@ -1373,20 +1373,16 @@ about 1.15e+18 bytes. Note also that all sizes are really 4 bytes less than the ones given below. For example the maximum size of a fixed variable in netCDF 3.6 classic format is really 2 GiB - 4 bytes. -Limit No LFS v3.5 v3.6/classic v3.6/64-bit offset v4.0/netCDF-4 +Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4 +-------------------------------------------|------------|---------|---------------|---------------------|----------------- +Max File Size | 2 GiB | 8 EiB | 8 EiB | 8 EiB | unlimited +Max Number of Fixed Vars > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited +Max Record Vars w/ Rec Size > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited +Max Size of Fixed/Record Size of Record Var| 2 GiB | 2 GiB | 2 GiB | 4 GiB | unlimited +Max Record Size | 2 GiB/nrecs| 4 GiB | 8 EiB/nrecs | 8 EiB/nrecs | unlimited -Max File Size 2 GiB 8 EiB 8 EiB 8 EiB ?? - -Max Number of Fixed Vars > 2 GiB 0 1 (last) 1 (last) 2^32 ?? - -Max Record Vars w/ Rec Size > 2 GiB 0 1 (last) 1 (last) 2^32 ?? - -Max Size of Fixed/Record Size of Record Var 2 GiB 2 GiB 2 GiB 4 GiB ?? - -Max Record Size 2 GiB/nrecs 4 GiB 8 EiB/nrecs 8 EiB/nrecs ?? - -For more information about the different file formats of netCDF See -Which Format. +For more information about the different file formats of netCDF see +\ref select_format "How to Select the Format". \section offset_format_limitations NetCDF 64-bit Offset Format Limitations diff --git a/docs/static-pages/software.html b/docs/static-pages/software.html index 719889237..2190b7ee1 100644 --- a/docs/static-pages/software.html +++ b/docs/static-pages/software.html @@ -300,6 +300,9 @@
  • ArcGIS Pro - Space Time Pattern Mining Toolbox
    • +
  • + AgriMetSoft - Netcdf-Extractor +
  • ASA ViewNcDap
    • @@ -2994,6 +2997,13 @@ or using ECMWF reanalysis on a reduced grid

    +

    AgriMetSoft Netcdf-Extractor

    + There are different software packages or various codes that may be used for manipulating or displaying NetCDF data. Some of them such as MATLAB programming language needs to enough information about provide codes in MATLAB as well as to know run codes and statements. Furthermore, user should have to pay for MATLAB programming and install it in her/his system. Also, another of them are related to other programming languages such as R, NCL (NCAR Command Language), and etc. It has been mentioned before that working by many of this methods to extract NetCDF files are so rough and complicated for users. Therefore, it is essential that users can extract their NC files in a user-friendly tool without any concerns. The NetCDF Extractor software is an easy tool for all users to extract their aim region from the main files. This tool is flexible to run for various datasets such as CMIP5 models, AgMERRA datasets, Aphrodite, CRU, CORDEX, and etc. Finally, this tool is NETCDF EXTRACTOR, NETCDF FILE VIEWER, and NETCDF CONVERTER. NetCDF-Extractor website. +

    + + +

    +

    ASA ViewNcDap

    Applied Science Associates, Inc. has made the ASA View NC/Dap diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index e464c091f..07c322ce5 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1220,7 +1220,8 @@ NC_computeshapes(NC3_INFO* ncp) if(first_rec == NULL) first_rec = *vpp; if((*vpp)->len == UINT32_MAX && - fIsSet(ncp->flags, NC_64BIT_OFFSET)) /* Flag for large last record */ + (fIsSet(ncp->flags, NC_64BIT_OFFSET) || + fIsSet(ncp->flags, NC_64BIT_DATA))) /* Flag for large last record */ ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz; else ncp->recsize += (*vpp)->len; diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 6ad753799..89526d840 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -102,11 +102,12 @@ nc4_check_name(const char *name, char *norm_name) retval = nc_utf8_normalize((const unsigned char *)name,(unsigned char**)&temp); if(retval != NC_NOERR) return retval; - if( strlen(temp) > NC_MAX_NAME ) - { - free(temp); - return NC_EMAXNAME; + + if(strlen(temp) > NC_MAX_NAME) { + free(temp); + return NC_EMAXNAME; } + strcpy(norm_name, temp); free(temp); @@ -1608,4 +1609,3 @@ NC4_show_metadata(int ncid) #endif /*LOGGING*/ return retval; } - diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt index 209bacdd0..a081aa319 100644 --- a/nc_test/CMakeLists.txt +++ b/nc_test/CMakeLists.txt @@ -51,7 +51,7 @@ IF(USE_PNETCDF) ENDIF() IF(LARGE_FILE_TESTS) - SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large) + SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large tst_large_cdf5) IF(NOT MSVC) SET(TESTS ${TESTS} large_files) ENDIF() diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index 8e03ff31f..87d074e9e 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -16,7 +16,8 @@ tst_*.nc t_nc.nc large_files.nc quick_large_files.nc \ tst_diskless.nc tst_diskless2.nc \ tst_diskless3.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \ tst_diskless4.cdl tst_diskless4.nc tst_formatx.nc nc_test_cdf5.nc \ -unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc +unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc \ +tst_large_cdf5.nc check_PROGRAMS = @@ -45,7 +46,7 @@ AM_CPPFLAGS += -I$(top_builddir)/liblib -I$(top_builddir)/include -I$(top_srcdir # If the user asked for large file tests, then add them. if LARGE_FILE_TESTS TESTPROGRAMS += quick_large_files tst_big_var6 tst_big_var2 \ -tst_big_rvar tst_big_var tst_large large_files +tst_big_rvar tst_big_var tst_large large_files tst_large_cdf5 endif # LARGE_FILE_TESTS if BUILD_BENCHMARKS diff --git a/nc_test/tst_formatx_pnetcdf.c b/nc_test/tst_formatx_pnetcdf.c index b11699f3e..99fa5c517 100644 --- a/nc_test/tst_formatx_pnetcdf.c +++ b/nc_test/tst_formatx_pnetcdf.c @@ -66,6 +66,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } + if (nc_close(ncid)) ERR; /* test CDF-2 file format */ cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_OFFSET; @@ -84,6 +85,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } + if (nc_close(ncid)) ERR; /* test CDF-5 file format */ cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_DATA; @@ -102,8 +104,7 @@ int main(int argc, char* argv[]) ecode = 1; ERR; } - - if (nc_abort(ncid)) ERR; + if (nc_close(ncid)) ERR; fn_exit: MPI_Finalize(); diff --git a/nc_test/tst_large_cdf5.c b/nc_test/tst_large_cdf5.c new file mode 100644 index 000000000..ef8681f0b --- /dev/null +++ b/nc_test/tst_large_cdf5.c @@ -0,0 +1,201 @@ +/* + Copyright 2017, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. + + This program tests the large file bug in netCDF 3.6.2, + creating byte and short variables larger than 4 GiB. + + This program is based on tst_large.c, and tests against + CDF5. + + $Id: tst_large.c,v 1.16 2010/05/18 20:20:01 russ Exp $ +*/ +#include +#include +#include +#include +#include +#include "err_macros.h" +#include + +/* Test with both classic and 64-bit offset files. If netcdf-4 is + * included, test with both netCDF-4 format variants also. */ +#ifdef USE_NETCDF4 +#define NUM_FORMATS (4) +#else +#define NUM_FORMATS (2) +#endif + +#define NUMDIMS 2 /* rank of each variable in tests */ +#define DIM1 2048 +#define DIM2 2097153 /* DIM1*DIM2*sizeof(char) > 2**32 */ +#define DIM3 1024 +#define DIM4 2097153 /* DIM3*DIM4*sizeof(short) > 2**32 */ + +/* + * In netCDF-3.6.2, a divide by zero occurs on 32-bit platforms when + * creating a variable for which the product of dimensions is exactly + * 2**32. Check that this bug has been fixed. + */ +static int +test_big_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + int cflag = NC_CLOBBER; + nc_type type = NC_BYTE; + size_t index[NUMDIMS]; + signed char nval = 99; + int nval_in; + + /* Define the file with one large variable. */ + if (nc_create(testfile, cflag, &ncid)) ERR; + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim2", DIM2 - 1, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", type, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + + /* Write one datum, near the end of the variable. */ + index[0] = DIM1 - 1; + index[1] = DIM2 - 2; + if (nc_put_var1_schar(ncid, varid, index, &nval)) ERR; + if (nc_close(ncid)) ERR; + + /* Reopen the file and check that datum. */ + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_int(ncid, varid, index, &nval_in)) ERR; + if (nval != nval_in) + ERR; + if (nc_close(ncid)) ERR; + return 0; +} + +static int +test_large_byte_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + size_t index[NUMDIMS] = {0, 0}; + signed char vals[DIM2]; + signed char char_val_in; + size_t start[NUMDIMS], count[NUMDIMS]; + int j; + + if (nc_create(testfile, NC_CLOBBER, &ncid)) ERR; + if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; + if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim2", DIM2, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", NC_BYTE, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + + for (j = 0; j < DIM2; j++) { + vals[j] = 9 * (j + 11); /* note vals[j] is 99 when j==0 */ + } + start[1] = 0; + count[0] = 1; + count[1] = DIM2; + for (start[0] = 0; start[0] < DIM1; start[0]++) { + if (nc_put_vara_schar(ncid, varid, start, count, vals)) + { + ERR; + break; + } + } + + if (nc_close(ncid)) ERR; + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_schar(ncid, varid, index, &char_val_in)) ERR; + if (char_val_in != 99) /* see above, the value written when start[0]==0, j==0 */ + ERR; + if (nc_close(ncid)) ERR; + return 0; +} + +static int +test_large_short_var(const char *testfile) { + int ncid, varid, dimids[NUMDIMS]; + int int_val_in, int_val_out = 99; + size_t index[2]; + int cflag = NC_CLOBBER; + + if (nc_create(testfile, cflag, &ncid)) ERR; + if (nc_def_dim(ncid, "dim3", DIM3, &dimids[0])) ERR; + if (nc_def_dim(ncid, "dim4", DIM4, &dimids[1])) ERR; + if (nc_def_var(ncid, "var", NC_SHORT, NUMDIMS, dimids, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + index[0] = 0; + index[1] = 1; + if (nc_put_var1_int(ncid, varid, index, &int_val_out)) ERR; + if (nc_close(ncid)) ERR; + if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR; + if (nc_inq_varid(ncid, "var", &varid)) ERR; + if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR; + if (int_val_in != int_val_out) + ERR; +#ifndef NOFILL + index[0] = 1; + index[1] = 2; + if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR; + if (int_val_in != NC_FILL_SHORT) + ERR; +#endif + if (nc_close(ncid)) ERR; + return 0; +} + +#define FILE_NAME "tst_large_cdf5.nc" + +int +main(int argc, char **argv) { + int i; + char testfile[NC_MAX_NAME + 1]; + + sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + + printf("\n*** Testing fix for 3.6.2 large file bug in %s.\n", + testfile); + /* Go thru formats and run all tests for each of two (for netCDF-3 + * only builds), or 4 (for netCDF-4 builds) different formats. */ + for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) + { + nc_set_default_format(i, NULL); + + printf("*** testing format %d with a variable with 2**32 values...", i); + test_big_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a byte variable with > 2**32 values...", i); + test_large_byte_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a short variable with > 2**32 values...", i); + test_large_short_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + } + + /* Test CDF5 */ + { + i = NC_64BIT_DATA; + nc_set_default_format(i, NULL); + + printf("*** testing format %d with a variable with 2**32 values...", i); + test_big_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a byte variable with > 2**32 values...", i); + test_large_byte_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + printf("*** testing format %d with a short variable with > 2**32 values...", i); + test_large_short_var(testfile); + (void) remove(testfile); + SUMMARIZE_ERR; + + } + + FINAL_RESULTS; +}