Merge branch 'master' into ejh_udf

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.7.0 - TBD
 
+* [Enhancement] Improved the performance of the nc_get/put_vars operations by using the equivalent slab capabilities of hdf5. Result is a significant speedup of these operations.  See [GitHub #1001](https://github.com/Unidata/netcdf-c/pull/1001) for more information.
 * [Enhancement] Expanded the capabilities of `NC_INMEMORY` to support writing and accessing the final modified memory.  See [GitHub #879](https://github.com/Unidata/netcdf-c/pull/879) for more information.
 * [Enhancement] Made CDF5 support enabled by default.  See [Github #931](https://github.com/Unidata/netcdf-c/issues/931) for more information.
 * [Bug Fix] Corrected a number of memory issues identified in `ncgen`.  See [GitHub #558 for more information](https://github.com/Unidata/netcdf-c/pull/558).

include/nc4dispatch.h

@@ -147,6 +147,16 @@ NC4_get_vara(int ncid, int varid,
 	     const size_t *start, const size_t *count,
              void *value, nc_type);
 
+extern int
+NC4_put_vars(int ncid, int varid,
+   	     const size_t *start, const size_t *count, const ptrdiff_t* stride,
+             const void *value, nc_type);
+
+extern int
+NC4_get_vars(int ncid, int varid,
+	     const size_t *start, const size_t *count, const ptrdiff_t* stride,
+             void *value, nc_type);
+
 /* End _var */
 
 /* netCDF4 API only */

include/nc4internal.h

@@ -105,7 +105,7 @@ typedef enum {NCNAT, NCVAR, NCDIM, NCATT, NCTYP, NCFLD, NCGRP} NC_SORT;
 #define HDF5_DIMSCALE_NAME_ATT_NAME "NAME"
 
 /** This is the number of netCDF atomic types. */
-#define NUM_ATOMIC_TYPES (NC_MAX_ATOMIC_TYPE + 1) 
+#define NUM_ATOMIC_TYPES (NC_MAX_ATOMIC_TYPE + 1)
 
 /* Boolean type, to make the code easier to read */
 typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
@@ -351,7 +351,8 @@ int nc4_get_typelen_mem(NC_HDF5_FILE_INFO_T *h5, nc_type xtype, size_t *len);
 int nc4_convert_type(const void *src, void *dest,
 		     const nc_type src_type, const nc_type dest_type,
 		     const size_t len, int *range_error,
-		     const void *fill_value, int strict_nc3);
+		     const void *fill_value, int strict_nc3, int src_long,
+		     int dest_long);
 
 /* These functions do HDF5 things. */
 int rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
@@ -359,9 +360,15 @@ int rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid);
 int delete_existing_dimscale_dataset(NC_GRP_INFO_T *grp, int dimid, NC_DIM_INFO_T *dim);
 int nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset);
 int nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
-		 const size_t *countp, nc_type xtype, void *op);
+		 const size_t *countp, nc_type xtype, int is_long, void *op);
 int nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
-		 const size_t *countp, nc_type xtype, void *op);
+		 const size_t *countp, nc_type xtype, int is_long, void *op);
+int nc4_put_vars(NC *nc, int ncid, int varid, const size_t *startp,
+		 const size_t *countp, const ptrdiff_t* stridep,
+		 nc_type xtype, int is_long, void *op);
+int nc4_get_vars(NC *nc, int ncid, int varid, const size_t *startp,
+		 const size_t *countp, const ptrdiff_t* stridep,
+		 nc_type xtype, int is_long, void *op);
 int nc4_rec_match_dimscales(NC_GRP_INFO_T *grp);
 int nc4_rec_detect_need_to_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp);
 int nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order);

libhdf4/hdf4var.c

@@ -96,7 +96,7 @@ NC_HDF4_get_vara(int ncid, int varid, const size_t *startp,
    if (var->type_info->hdr.id != memtype)
    {
       if ((retval = nc4_convert_type(data, ip, var->type_info->hdr.id, memtype, nelem,
-                                     &range_error, NULL, 0)))
+                                     &range_error, NULL, 0, 0, 0)))
          return retval;
       free(data);
       if (range_error)

libhdf5/hdf5attr.c

@@ -581,7 +581,7 @@ NC4_put_att(int ncid, int varid, const char *name, nc_type file_type,
             /* Data types are like religions, in that one can convert.  */
             if ((retval = nc4_convert_type(data, att->data, mem_type, file_type,
                                            len, &range_error, NULL,
-                                           (h5->cmode & NC_CLASSIC_MODEL))))
+                                           (h5->cmode & NC_CLASSIC_MODEL), 0, 0)))
                BAIL(retval);
          }
       }

libsrc4/nc4attr.c

@@ -205,7 +205,7 @@ nc4_get_att(int ncid, int varid, const char *name, nc_type *xtype,
       need_to_convert++;
       if ((retval = nc4_convert_type(att->data, bufr, att->nc_typeid,
                                      mem_type, (size_t)att->len, &range_error,
-                                     NULL, (h5->cmode & NC_CLASSIC_MODEL))))
+                                     NULL, (h5->cmode & NC_CLASSIC_MODEL), 0, 0)))
          BAIL(retval);
 
       /* For strict netcdf-3 rules, ignore erange errors between UBYTE

libsrc4/nc4dispatch.c

@@ -64,8 +64,8 @@ NC4_inq_varid,
 NC4_rename_var,
 NC4_get_vara,
 NC4_put_vara,
-NCDEFAULT_get_vars,
-NCDEFAULT_put_vars,
+NC4_get_vars,
+NC4_put_vars,
 NCDEFAULT_get_varm,
 NCDEFAULT_put_varm,
 

libsrc4/nc4var.c

@@ -1673,7 +1673,7 @@ NC4_put_vara(int ncid, int varid, const size_t *startp,
    if (!(nc = nc4_find_nc_file(ncid, NULL)))
       return NC_EBADID;
 
-   return nc4_put_vara(nc, ncid, varid, startp, countp, memtype, (void *)op);
+   return nc4_put_vara(nc, ncid, varid, startp, countp, memtype, 0, (void *)op);
 }
 
 /**
@@ -1705,5 +1705,90 @@ NC4_get_vara(int ncid, int varid, const size_t *startp,
       return NC_EBADID;
 
    /* Get the data. */
-   return nc4_get_vara(nc, ncid, varid, startp, countp, memtype, (void *)ip);
+   return nc4_get_vara(nc, ncid, varid, startp, countp, memtype,
+                       0, (void *)ip);
+}
+
+
+/* Provide a temporary hook
+to choose old NCDEFAULT methods vs new versions
+of get/put vars
+*/
+/* Temporary flag to choose default vs not put/get vars */
+static int defaultvars = 0;
+
+void
+nc4_set_default_vars(int tf)
+{
+    defaultvars = (tf ? 1 : 0);
+}
+
+/**
+ * @internal Write an array of data to a variable. This is called by
+ * nc_put_vars() and other nc_put_vars_* functions, for netCDF-4
+ * files.
+ * 
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param startp Array of start indices.
+ * @param countp Array of counts.
+ * @param stridep Array of strides.
+ * @param op pointer that gets the data.
+ * @param memtype The type of these data in memory.
+ *
+ * @returns ::NC_NOERR for success
+ * @author Dennis Heimbigner
+ */
+int
+NC4_put_vars(int ncid, int varid, const size_t *startp,
+             const size_t *countp, const ptrdiff_t* stridep,
+	     const void *op, int memtype)
+{
+   NC *nc;
+
+   if(defaultvars)
+	return NCDEFAULT_put_vars(ncid,varid,startp,countp,stridep,op,memtype);
+
+   if (!(nc = nc4_find_nc_file(ncid, NULL)))
+      return NC_EBADID;
+
+   return nc4_put_vars(nc, ncid, varid, startp, countp, stridep, memtype, 0, (void *)op);
+}
+
+/**
+ * Read an array of values. This is called by nc_get_vars() for
+ * netCDF-4 files, as well as all the other nc_get_vars_*
+ * functions.
+ *
+ * @param ncid File ID.
+ * @param varid Variable ID.
+ * @param startp Array of start indices.
+ * @param countp Array of counts.
+ * @param stridep Array of strides.
+ * @param ip pointer that gets the data.
+ * @param memtype The type of these data after it is read into memory.
+
+ * @returns ::NC_NOERR for success
+ * @author Dennis Heimbigner
+ */
+int
+NC4_get_vars(int ncid, int varid, const size_t *startp,
+             const size_t *countp, const ptrdiff_t *stridep,
+	     void *ip, int memtype)
+{
+   NC *nc;
+   NC_HDF5_FILE_INFO_T* h5;
+
+   if(defaultvars)
+	return NCDEFAULT_get_vars(ncid,varid,startp,countp,stridep,ip,memtype);
+
+   LOG((2, "%s: ncid 0x%x varid %d memtype %d", __func__, ncid, varid,
+        memtype));
+
+   if (!(nc = nc4_find_nc_file(ncid, &h5)))
+      return NC_EBADID;
+
+   /* Get the data. */
+   return nc4_get_vars(nc, ncid, varid, startp, countp, stridep, memtype,
+                       0, (void *)ip);
 }

nc_test/Make0

@@ -6,6 +6,8 @@ H510=10
 
 #ARGS=f03_1820.nc
 
+SRC=test_put.c test_get.c test_read.c test_write.c util.c error.c
+
 #CMD=valgrind --leak-check=full
 #CMD=export NETCDF_LOG_LEVEL=5 ;gdb --args
 CMD=gdb --args

nc_test4/Make0

@@ -1,10 +1,10 @@
 # Test c output
-T=t_dap3a
+T=tst_varsperf
 
 #SRC=hdf5plugins/H5Zmisc.c
 
 #CMD=valgrind --leak-check=full
-CMD=gdb --args
+#CMD=gdb --args
 
 #FILTER=H5Zmisc
 #FILTEROBJ=hdf5plugins/${FILTER}.o
@@ -31,7 +31,7 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
 
 all:: cmp
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
-	${CMD} ./t
+	${CMD} ./t ${ARGS}
 
 cmp::
 	export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \

nc_test4/Makefile.am

@@ -30,7 +30,8 @@ t_type cdm_sea_soundings tst_camrun tst_vl tst_atts1 tst_atts2		\
 tst_vars2 tst_files5 tst_files6 tst_sync tst_h_scalar tst_rename	\
 tst_rename2 tst_h5_endians tst_atts_string_rewrite			\
 tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_filterparser	\
-tst_bug324 tst_types tst_atts3 tst_put_vars tst_elatefill tst_udf
+tst_bug324 tst_types tst_atts3 tst_put_vars tst_elatefill tst_udf tst_varsperf
+
 
 # Temporary I hope
 if !ISCYGWIN
@@ -138,15 +139,20 @@ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5	\
 ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl unfiltered.cdl	\
 ref_bzip2.c findplugin.in perftest.sh
 
-CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc	\
-bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt			\
-radar_3d_compression.txt radar_2d_compression.txt			\
-radar_3d_chunking.txt tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl	\
-tst_*.nc tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl		\
-tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.*	\
-usi_01.* thetau_01.* tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp	\
-ref_grp_rename.cdl foo1.nc test.nc testszip.nc test.h5 szip_dump.cdl	\
-perftest.txt bigmeta.nc
+
+CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc   \
+bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt                  \
+radar_3d_compression.txt radar_2d_compression.txt                       \
+radar_3d_chunking.txt tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl    \
+tst_*.nc tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl            \
+tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.*        \
+usi_01.* thetau_01.* tst_*.nc tst_*.h5 tst_grp_rename.cdl               \
+tst_grp_rename.nc tst_grp_rename.dmp ref_grp_rename.cdl foo1.nc         \
+tst_interops2.h4 tst_h5_endians.nc tst_h4_lendian.h4 test.nc            \
+tst_atts_string_rewrite.nc tst_empty_vlen_unlim.nc                      \
+tst_empty_vlen_lim.nc tst_parallel4_simplerw_coll.nc                    \
+tst_fill_attr_vanish.nc tst_rehash.nc testszip.nc test.h5               \
+szip_dump.cdl perftest.txt bigmeta.nc bigvars.nc
 
 DISTCLEANFILES = findplugin.sh
 

nc_test4/tst_varsperf.c

@@ -0,0 +1,149 @@
+/*
+Create a netcdf-4 file with
+a large variable for the purpose
+of doing performance test on the
+new NC4_get/put_vars functions.
+
+WARNING: do not attempt to run this
+under windows because of the use
+of gettimeofday().
+
+*/
+
+#include "config.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/time.h>
+#include <assert.h>
+#include "netcdf.h"
+
+
+#define FILE "bigvars.nc"
+#define VAR "bigvar"
+#define NDIMS 2
+#define DIM0 "d0"
+#define DIM1 "d1"
+#define DIMSIZE0 1024
+#define DIMSIZE1 1024
+#define TOTALSIZE (DIMSIZE0*DIMSIZE1)
+
+#define CHECK(expr) assert((expr) == NC_NOERR)
+
+static int data[TOTALSIZE];
+static int read[TOTALSIZE];
+
+extern void nc4_set_default_vars(int);
+
+void
+buildfile(void)
+{
+    int ncid, varid;
+    int dimids[NDIMS];
+    int* p;
+    int index;
+
+    CHECK(nc_create(FILE, NC_NETCDF4, &ncid));
+
+    CHECK(nc_def_dim(ncid,DIM0,(size_t)DIMSIZE0,&dimids[0]));
+    CHECK(nc_def_dim(ncid,DIM1,(size_t)DIMSIZE1,&dimids[1]));
+    CHECK(nc_def_var(ncid,VAR,NC_INT,NDIMS,dimids,&varid));
+    
+    CHECK(nc_enddef(ncid));
+
+    for(p=data,index=0;index<TOTALSIZE;index++)
+	*p++ = index;	
+
+    CHECK(nc_put_var_int(ncid,varid,data));
+
+    CHECK(nc_close(ncid));
+}
+
+long long
+readfile(void)
+{
+    int ncid, varid;
+    int ndims, i;
+    int dimids[NDIMS];
+    size_t dimsizes[NDIMS];
+    int vardims[NDIMS];
+    size_t start[NDIMS];
+    size_t count[NDIMS];
+    ptrdiff_t stride[NDIMS];
+    struct timeval starttime, endtime;
+    long long delta;
+    long long startt, endt;
+
+    memset(&starttime,0,sizeof(starttime));
+    memset(&endtime,0,sizeof(endtime));
+
+    /* re-open to read */
+    CHECK(nc_open(FILE, NC_NETCDF4, &ncid));
+
+    CHECK(nc_inq_dimid(ncid,DIM0,&dimids[0]));
+    CHECK(nc_inq_dimid(ncid,DIM1,&dimids[1]));
+    CHECK(nc_inq_dim(ncid,dimids[0],NULL,&dimsizes[0]));
+    CHECK(nc_inq_dim(ncid,dimids[1],NULL,&dimsizes[1]));
+    assert(dimsizes[0] == DIMSIZE0);
+    assert(dimsizes[1] == DIMSIZE1);
+
+    CHECK(nc_inq_varid(ncid,VAR,&varid));
+    CHECK(nc_inq_varndims(ncid,varid,&ndims));
+    assert(ndims == NDIMS);
+    CHECK(nc_inq_vardimid(ncid,varid,vardims));
+    for(i=0;i<NDIMS;i++)
+        assert(vardims[i] == dimids[i]);
+
+    /* Do the timed read */
+    for(i=0;i<NDIMS;i++) {
+	start[i] = 0;
+	count[i] = dimsizes[i]/2;
+	stride[i] = 2;
+    }
+
+    memset(read,0,sizeof(read));
+    gettimeofday(&starttime,NULL);
+    CHECK(nc_get_vars(ncid,varid,start,count,stride,read));
+    gettimeofday(&endtime,NULL);
+
+    CHECK(nc_close(ncid));
+
+    /* Verify read -- Note: NDIMS dependent */
+    {
+	int d0, d1;
+	i = 0;
+	for(d0=0;d0<DIMSIZE0;d0+=stride[0]) {
+            for(d1=0;d1<DIMSIZE1;d1+=stride[1]) {
+		size_t dataindex = (d0 * DIMSIZE0) + d1;
+		size_t readindex = i;
+		assert(data[dataindex] == read[readindex]);
+		i++;
+	    }	
+	}
+    }
+
+    /* Compute the time delta */
+    startt = (1000000*starttime.tv_sec) + starttime.tv_usec;
+    endt = (1000000*endtime.tv_sec) + endtime.tv_usec;
+    delta = endt - startt;
+    return delta;
+}
+
+
+int
+main(int argc, char **argv)
+{
+    long long defaultdelta, nc4delta, factor;
+
+    buildfile();
+    nc4_set_default_vars(1);
+    defaultdelta = readfile();
+    nc4_set_default_vars(0);
+    nc4delta = readfile();
+
+    /* Print results to the millisec */
+    factor = defaultdelta / nc4delta;    
+    printf("NCDEFAULT time=%lld NC4 time=%lld Speedup=%lld\n",
+		defaultdelta, nc4delta, factor);
+    return 0;
+}

ncgen/semantics.c

@@ -449,7 +449,7 @@ processeconstrefsR(Datalist* data)
 {
     NCConstant* con = NULL;
     int i;
-    for(i=0,con=data->data;i<data->alloc;i++,con++) {
+    for(i=0,con=data->data;i<data->length;i++,con++) {
 	if(con->nctype == NC_COMPOUND) {
 	    /* Iterate over the sublists */
 	    processeconstrefsR(con->value.compoundv);
@@ -467,6 +467,7 @@ fixeconstref(NCConstant* con)
     Symbol* refsym = con->value.enumv;
     List* grpmatches;
 
+    
     /* Locate all possible matching enum constant definitions */
     List* candidates = findecmatches(refsym->name);
     if(candidates == NULL) {