Merge branch 'v4.5.0-release-branch'

RELEASE_NOTES.md

@@ -7,6 +7,9 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.5.0 - TBD
 
+* [Bug Fix] Corrected an issue where `NC_64BIT_DATA` files were being read incorrectly by ncdump, despite the data having been written correctly.  See [GitHub #457](https://github.com/Unidata/netcdf-c/issues/457) for more information.
+* [Bug Fix] Corrected a potential stack buffer overflow.  See [GitHub #450](https://github.com/Unidata/netcdf-c/pull/450) for more information.
+
 ### 4.5.0-rc2 - August 7, 2017
 
 * [Bug Fix] Addressed an issue with how cmake was implementing large file support on 32-bit systems. See [GitHub #385](https://github.com/Unidata/netcdf-c/issues/385) for more information.

docs/guide.dox

@@ -1373,20 +1373,16 @@ about 1.15e+18 bytes. Note also that all sizes are really 4 bytes less
 than the ones given below. For example the maximum size of a fixed
 variable in netCDF 3.6 classic format is really 2 GiB - 4 bytes.
 
-Limit 	No LFS 	v3.5 	v3.6/classic 	v3.6/64-bit offset 	v4.0/netCDF-4
+Limits                                     | No LFS     |  v3.5   |  v3.6/classic |  v3.6/64-bit offset |    v4.0/netCDF-4
+-------------------------------------------|------------|---------|---------------|---------------------|-----------------
+Max File Size                              | 2 GiB      | 8 EiB   |  8 EiB        |  8 EiB              | unlimited
+Max Number of Fixed Vars > 2 GiB           |     0      | 1 (last)|  1 (last)     |  2^32               | unlimited
+Max Record Vars w/ Rec Size > 2 GiB        |     0      | 1 (last)|  1 (last)     |  2^32               | unlimited
+Max Size of Fixed/Record Size of Record Var| 2 GiB      | 2 GiB   |  2 GiB        |  4 GiB              | unlimited
+Max Record Size                            | 2 GiB/nrecs| 4 GiB   |  8 EiB/nrecs  |  8 EiB/nrecs        | unlimited
 
-Max File Size 	2 GiB 	8 EiB 	8 EiB 	8 EiB 	??
-
-Max Number of Fixed Vars > 2 GiB 	0 	1 (last) 	1 (last) 	2^32 	??
-
-Max Record Vars w/ Rec Size > 2 GiB 	0 	1 (last) 	1 (last) 	2^32 	??
-
-Max Size of Fixed/Record Size of Record Var 	2 GiB 	2 GiB 	2 GiB 	4 GiB 	??
-
-Max Record Size 	2 GiB/nrecs 	4 GiB 	8 EiB/nrecs 	8 EiB/nrecs 	??
-
-For more information about the different file formats of netCDF See
-Which Format.
+For more information about the different file formats of netCDF see
+\ref select_format "How to Select the Format".
 
 \section offset_format_limitations NetCDF 64-bit Offset Format Limitations
 

docs/static-pages/software.html

@@ -300,6 +300,9 @@
       <li>
         <a href="#ArcGIS">ArcGIS Pro - Space Time Pattern Mining Toolbox
       </li>
+      <li>
+        <a href="#Agrimetsoft"> AgriMetSoft - Netcdf-Extractor
+      </li>
       <li>
         <a href="#ViewNcDap" >ASA ViewNcDap</a>
       </li>
@@ -2994,6 +2997,13 @@ or using ECMWF reanalysis on a reduced grid
     </p>
     <p></p>
 
+    <h2><a id="Agrimetsoft" name="Agrimetsoft">AgriMetSoft Netcdf-Extractor</a></h2>
+      There are different software packages or various codes that may be used for manipulating or displaying NetCDF data. Some of them such as MATLAB programming language needs to enough information about provide codes in MATLAB as well as to know run codes and statements. Furthermore, user should have to pay for MATLAB programming and install it in her/his system. Also, another of them are related to other programming languages such as R, NCL (NCAR Command Language), and etc. It has been mentioned before that working by many of this methods to extract NetCDF files are so rough and complicated for users. Therefore, it is essential that users can extract their NC files in a user-friendly tool without any concerns. The NetCDF Extractor software is an easy tool for all users to extract their aim region from the main files. This tool is flexible to run for various datasets such as CMIP5 models, AgMERRA datasets, Aphrodite, CRU, CORDEX, and etc. Finally, this tool is NETCDF EXTRACTOR, NETCDF FILE VIEWER, and NETCDF CONVERTER. <a href="http://www.agrimetsoft.com/Netcdf-Extractor.aspx">NetCDF-Extractor website.</A>
+    <p>
+
+
+    <p></p>
+
     <h2><a id="ViewNcDap" name="ViewNcDap">ASA ViewNcDap</a></h2>
     <p>
       Applied Science Associates, Inc. has made the ASA View NC/Dap

libsrc/v1hpg.c

@@ -1220,7 +1220,8 @@ NC_computeshapes(NC3_INFO* ncp)
 	  		if(first_rec == NULL)
 				first_rec = *vpp;
 			if((*vpp)->len == UINT32_MAX &&
-                           fIsSet(ncp->flags, NC_64BIT_OFFSET)) /* Flag for large last record */
+                           (fIsSet(ncp->flags, NC_64BIT_OFFSET) ||
+                            fIsSet(ncp->flags, NC_64BIT_DATA))) /* Flag for large last record */
                             ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz;
 			else
 			    ncp->recsize += (*vpp)->len;

libsrc4/nc4internal.c

@@ -102,11 +102,12 @@ nc4_check_name(const char *name, char *norm_name)
    retval = nc_utf8_normalize((const unsigned char *)name,(unsigned char**)&temp);
    if(retval != NC_NOERR)
       return retval;
-   if( strlen(temp) > NC_MAX_NAME )
-   {
-       free(temp);
-       return NC_EMAXNAME;
+
+   if(strlen(temp) > NC_MAX_NAME) {
+     free(temp);
+     return NC_EMAXNAME;
    }
+
    strcpy(norm_name, temp);
    free(temp);
 
@@ -1608,4 +1609,3 @@ NC4_show_metadata(int ncid)
 #endif /*LOGGING*/
    return retval;
 }
-

nc_test/CMakeLists.txt

@@ -51,7 +51,7 @@ IF(USE_PNETCDF)
 ENDIF()
 
 IF(LARGE_FILE_TESTS)
-  SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large)
+  SET(TESTS ${TESTS} quick_large_files tst_big_var6 tst_big_var2 tst_big_rvar tst_big_var tst_large tst_large_cdf5)
   IF(NOT MSVC)
     SET(TESTS ${TESTS} large_files)
   ENDIF()

nc_test/Makefile.am

@@ -16,7 +16,8 @@ tst_*.nc t_nc.nc large_files.nc quick_large_files.nc \
 tst_diskless.nc tst_diskless2.nc \
 tst_diskless3.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \
 tst_diskless4.cdl tst_diskless4.nc tst_formatx.nc nc_test_cdf5.nc \
-unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc
+unlim.nc tst_inq_type.nc tst_elatefill.nc tst_global_fillval.nc \
+tst_large_cdf5.nc
 
 check_PROGRAMS =
 
@@ -45,7 +46,7 @@ AM_CPPFLAGS += -I$(top_builddir)/liblib -I$(top_builddir)/include -I$(top_srcdir
 # If the user asked for large file tests, then add them.
 if LARGE_FILE_TESTS
 TESTPROGRAMS += quick_large_files tst_big_var6 tst_big_var2	\
-tst_big_rvar tst_big_var tst_large large_files
+tst_big_rvar tst_big_var tst_large large_files tst_large_cdf5
 endif # LARGE_FILE_TESTS
 
 if BUILD_BENCHMARKS

nc_test/tst_formatx_pnetcdf.c

@@ -66,6 +66,7 @@ int main(int argc, char* argv[])
 	ecode = 1;
 	ERR;
     }
+    if (nc_close(ncid)) ERR;
 
     /* test CDF-2 file format */
     cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_OFFSET;
@@ -84,6 +85,7 @@ int main(int argc, char* argv[])
 	ecode = 1;
 	ERR;
     }
+    if (nc_close(ncid)) ERR;
 
     /* test CDF-5 file format */
     cmode = NC_PNETCDF | NC_CLOBBER | NC_64BIT_DATA;
@@ -102,8 +104,7 @@ int main(int argc, char* argv[])
 	ecode = 1;
 	ERR;
     }
-
-    if (nc_abort(ncid)) ERR;
+    if (nc_close(ncid)) ERR;
 
 fn_exit:
     MPI_Finalize();

nc_test/tst_large_cdf5.c

@@ -0,0 +1,201 @@
+/*
+  Copyright 2017, UCAR/Unidata
+  See COPYRIGHT file for copying and redistribution conditions.
+
+  This program tests the large file bug in netCDF 3.6.2,
+  creating byte and short variables larger than 4 GiB.
+
+  This program is based on tst_large.c, and tests against
+  CDF5.
+
+  $Id: tst_large.c,v 1.16 2010/05/18 20:20:01 russ Exp $
+*/
+#include <config.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <nc_tests.h>
+#include "err_macros.h"
+#include <netcdf.h>
+
+/* Test with both classic and 64-bit offset files. If netcdf-4 is
+ * included, test with both netCDF-4 format variants also. */
+#ifdef USE_NETCDF4
+#define NUM_FORMATS (4)
+#else
+#define NUM_FORMATS (2)
+#endif
+
+#define NUMDIMS 2		/* rank of each variable in tests */
+#define DIM1 2048
+#define DIM2 2097153		/* DIM1*DIM2*sizeof(char)   > 2**32 */
+#define DIM3 1024
+#define DIM4 2097153		/* DIM3*DIM4*sizeof(short)  > 2**32  */
+
+/*
+ * In netCDF-3.6.2, a divide by zero occurs on 32-bit platforms when
+ * creating a variable for which the product of dimensions is exactly
+ * 2**32.  Check that this bug has been fixed.
+ */
+static int
+test_big_var(const char *testfile) {
+    int ncid, varid, dimids[NUMDIMS];
+    int cflag = NC_CLOBBER;
+    nc_type type = NC_BYTE;
+    size_t index[NUMDIMS];
+    signed char nval = 99;
+    int nval_in;
+
+    /* Define the file with one large variable. */
+    if (nc_create(testfile, cflag, &ncid)) ERR;
+    if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR;
+    if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR;
+    if (nc_def_dim(ncid, "dim2", DIM2 - 1, &dimids[1])) ERR;
+    if (nc_def_var(ncid, "var", type, NUMDIMS, dimids, &varid)) ERR;
+    if (nc_enddef(ncid)) ERR;
+
+    /* Write one datum, near the end of the variable. */
+    index[0] = DIM1 - 1;
+    index[1] = DIM2 - 2;
+    if (nc_put_var1_schar(ncid, varid, index, &nval)) ERR;
+    if (nc_close(ncid)) ERR;
+
+    /* Reopen the file and check that datum. */
+    if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR;
+    if (nc_inq_varid(ncid, "var", &varid)) ERR;
+    if (nc_get_var1_int(ncid, varid, index, &nval_in)) ERR;
+    if (nval != nval_in)
+	ERR;
+    if (nc_close(ncid)) ERR;
+    return 0;
+}
+
+static int
+test_large_byte_var(const char *testfile) {
+    int ncid, varid, dimids[NUMDIMS];
+    size_t index[NUMDIMS] = {0, 0};
+    signed char vals[DIM2];
+    signed char char_val_in;
+    size_t start[NUMDIMS], count[NUMDIMS];
+    int j;
+
+    if (nc_create(testfile, NC_CLOBBER, &ncid)) ERR;
+    if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR;
+    if (nc_def_dim(ncid, "dim1", DIM1, &dimids[0])) ERR;
+    if (nc_def_dim(ncid, "dim2", DIM2, &dimids[1])) ERR;
+    if (nc_def_var(ncid, "var", NC_BYTE, NUMDIMS, dimids, &varid)) ERR;
+    if (nc_enddef(ncid)) ERR;
+
+    for (j = 0; j < DIM2; j++) {
+       vals[j] = 9 * (j + 11); /* note vals[j] is 99 when j==0 */
+    }
+    start[1] = 0;
+    count[0] = 1;
+    count[1] = DIM2;
+    for (start[0] = 0; start[0] < DIM1; start[0]++) {
+	if (nc_put_vara_schar(ncid, varid, start, count, vals))
+	{
+	    ERR;
+	    break;
+	}
+    }
+
+    if (nc_close(ncid)) ERR;
+    if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR;
+    if (nc_inq_varid(ncid, "var", &varid)) ERR;
+    if (nc_get_var1_schar(ncid, varid, index, &char_val_in)) ERR;
+    if (char_val_in != 99)	/* see above, the value written when start[0]==0, j==0 */
+	ERR;
+    if (nc_close(ncid)) ERR;
+    return 0;
+}
+
+static int
+test_large_short_var(const char *testfile) {
+    int ncid, varid, dimids[NUMDIMS];
+    int int_val_in, int_val_out = 99;
+    size_t index[2];
+    int cflag = NC_CLOBBER;
+
+    if (nc_create(testfile, cflag, &ncid)) ERR;
+    if (nc_def_dim(ncid, "dim3", DIM3, &dimids[0])) ERR;
+    if (nc_def_dim(ncid, "dim4", DIM4, &dimids[1])) ERR;
+    if (nc_def_var(ncid, "var", NC_SHORT, NUMDIMS, dimids, &varid)) ERR;
+    if (nc_enddef(ncid)) ERR;
+    index[0] = 0;
+    index[1] = 1;
+    if (nc_put_var1_int(ncid, varid, index, &int_val_out)) ERR;
+    if (nc_close(ncid)) ERR;
+    if (nc_open(testfile, NC_NOWRITE, &ncid)) ERR;
+    if (nc_inq_varid(ncid, "var", &varid)) ERR;
+    if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR;
+    if (int_val_in != int_val_out)
+	ERR;
+#ifndef NOFILL
+    index[0] = 1;
+    index[1] = 2;
+    if (nc_get_var1_int(ncid, varid, index, &int_val_in)) ERR;
+    if (int_val_in != NC_FILL_SHORT)
+	ERR;
+#endif
+    if (nc_close(ncid)) ERR;
+    return 0;
+}
+
+#define FILE_NAME "tst_large_cdf5.nc"
+
+int
+main(int argc, char **argv) {
+    int i;
+    char testfile[NC_MAX_NAME + 1];
+
+    sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME);
+
+    printf("\n*** Testing fix for 3.6.2 large file bug in %s.\n",
+	   testfile);
+   /* Go thru formats and run all tests for each of two (for netCDF-3
+    * only builds), or 4 (for netCDF-4 builds) different formats. */
+    for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++)
+    {
+       nc_set_default_format(i, NULL);
+
+       printf("*** testing format %d with a variable with 2**32 values...", i);
+       test_big_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+
+       printf("*** testing format %d with a byte variable with > 2**32 values...", i);
+       test_large_byte_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+
+       printf("*** testing format %d with a short variable with > 2**32 values...", i);
+       test_large_short_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+    }
+
+    /* Test CDF5 */
+    {
+      i = NC_64BIT_DATA;
+      nc_set_default_format(i, NULL);
+
+       printf("*** testing format %d with a variable with 2**32 values...", i);
+       test_big_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+
+       printf("*** testing format %d with a byte variable with > 2**32 values...", i);
+       test_large_byte_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+
+       printf("*** testing format %d with a short variable with > 2**32 values...", i);
+       test_large_short_var(testfile);
+       (void) remove(testfile);
+       SUMMARIZE_ERR;
+
+    }
+
+    FINAL_RESULTS;
+}