From 3e00967879355e34e943277270cd5644397858eb Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Thu, 19 Dec 2019 09:19:23 -0700
Subject: [PATCH 01/28] allow parallel writes to use zlib

---
 libhdf5/hdf5var.c    | 5 ++++-
 nc_test4/Makefile.am | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 78f0cad89..0ffc2f544 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -659,8 +659,11 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     assert(var && var->hdr.id == varid);
 
     /* Can't turn on parallel and deflate/fletcher32/szip/shuffle (for now). */
+    /* if (h5->parallel == NC_TRUE) */
+    /*     if (deflate || fletcher32 || shuffle) */
+    /*         return NC_EINVAL; */
     if (h5->parallel == NC_TRUE)
-        if (deflate || fletcher32 || shuffle)
+        if (fletcher32)
             return NC_EINVAL;
 
     /* If the HDF5 dataset has already been created, then it is too
diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am
index 455677880..c29071fe9 100644
--- a/nc_test4/Makefile.am
+++ b/nc_test4/Makefile.am
@@ -84,7 +84,7 @@ endif # BUILD_UTILITIES
 if TEST_PARALLEL4
 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3		\
 tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r	\
-tst_mode
+tst_mode tst_parallel_zlib
 TESTS += run_par_test.sh
 endif
 

From cde58d23c1fb8c5291b3d4a8346e10a6bca0641e Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Thu, 19 Dec 2019 09:20:20 -0700
Subject: [PATCH 02/28] allowing parallel write of gzipped data, plus added
 test

---
 nc_test4/CMakeLists.txt      |   1 +
 nc_test4/tst_parallel_zlib.c | 167 +++++++++++++++++++++++++++++++++++
 2 files changed, 168 insertions(+)
 create mode 100644 nc_test4/tst_parallel_zlib.c

diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt
index fd2a16339..524312681 100644
--- a/nc_test4/CMakeLists.txt
+++ b/nc_test4/CMakeLists.txt
@@ -85,6 +85,7 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_parallel3)
   build_bin_test(tst_parallel4)
   build_bin_test(tst_parallel5)
+  build_bin_test(tst_parallel_zlib)
   build_bin_test(tst_nc4perf)
   build_bin_test(tst_mode)
   build_bin_test(tst_simplerw_coll_r)
diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
new file mode 100644
index 000000000..8d53f59d4
--- /dev/null
+++ b/nc_test4/tst_parallel_zlib.c
@@ -0,0 +1,167 @@
+/*
+Copyright 2018, UCAR/Unidata
+See COPYRIGHT file for copying and redistribution conditions.
+
+This program tests netcdf-4 parallel I/O using zlib compression while
+writing.
+
+Ed Hartnett, 2019/12/19
+*/
+
+/* Defining USE_MPE causes the MPE trace library to be used (and you
+ * must also relink with -llmpe -lmpe). This causes clog2 output to be
+ * written, which can be converted to slog2 (by the program
+ * clog2TOslog2) and then used in the analysis program jumpshot. */
+/*#define USE_MPE 1*/
+
+#include <nc_tests.h>
+#include "err_macros.h"
+#include <mpi.h>
+#ifdef USE_MPE
+#include <mpe.h>
+#endif /* USE_MPE */
+
+#define FILE "tst_parallel_zlib.nc"
+#define NDIMS 3
+#define DIMSIZE 24
+#define QTR_DATA (DIMSIZE * DIMSIZE / 4)
+#define NUM_PROC 4
+#define NUM_SLABS 10
+
+int
+main(int argc, char **argv)
+{
+    /* MPI stuff. */
+    int mpi_namelen;
+    char mpi_name[MPI_MAX_PROCESSOR_NAME];
+    int mpi_size, mpi_rank;
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    /* Netcdf-4 stuff. */
+    int ncid, v1id, dimids[NDIMS];
+    size_t start[NDIMS], count[NDIMS];
+
+    int i, res;
+    int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
+    char file_name[NC_MAX_NAME + 1];
+
+#ifdef USE_MPE
+    int s_init, e_init, s_define, e_define, s_write, e_write, s_close, e_close;
+#endif /* USE_MPE */
+
+    /* Initialize MPI. */
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+    MPI_Get_processor_name(mpi_name, &mpi_namelen);
+    /*printf("mpi_name: %s size: %d rank: %d\n", mpi_name,
+      mpi_size, mpi_rank);*/
+
+#ifdef USE_MPE
+    MPE_Init_log();
+    s_init = MPE_Log_get_event_number();
+    e_init = MPE_Log_get_event_number();
+    s_define = MPE_Log_get_event_number();
+    e_define = MPE_Log_get_event_number();
+    s_write = MPE_Log_get_event_number();
+    e_write = MPE_Log_get_event_number();
+    s_close = MPE_Log_get_event_number();
+    e_close = MPE_Log_get_event_number();
+    MPE_Describe_state(s_init, e_init, "Init", "red");
+    MPE_Describe_state(s_define, e_define, "Define", "yellow");
+    MPE_Describe_state(s_write, e_write, "Write", "green");
+    MPE_Describe_state(s_close, e_close, "Close", "purple");
+    MPE_Start_log();
+    MPE_Log_event(s_init, 0, "start init");
+#endif /* USE_MPE */
+
+    if (mpi_rank == 0)
+    {
+       printf("\n*** Testing parallel writes with zlib.\n");
+       printf("*** testing simple write with zlib...");
+    }
+
+    /* Create phony data. We're going to write a 24x24 array of ints,
+       in 4 sets of 144. */
+    for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
+       slab_data[i] = mpi_rank;
+
+#ifdef USE_MPE
+    MPE_Log_event(e_init, 0, "end init");
+    MPE_Log_event(s_define, 0, "start define file");
+#endif /* USE_MPE */
+
+    /* Create a parallel netcdf-4 file. */
+    /*nc_set_log_level(3);*/
+    /* sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); */
+    sprintf(file_name, "%s", FILE);
+    if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR;
+
+    /* Create three dimensions. */
+    if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
+    if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
+    if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
+
+    /* Create one var. */
+    if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
+    if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
+
+    /* Write metadata to file. */
+    if ((res = nc_enddef(ncid))) ERR;
+
+#ifdef USE_MPE
+    MPE_Log_event(e_define, 0, "end define file");
+    if (mpi_rank)
+       sleep(mpi_rank);
+#endif /* USE_MPE */
+
+    /* Set up slab for this process. */
+    start[0] = mpi_rank * DIMSIZE/mpi_size;
+    start[1] = 0;
+    count[0] = DIMSIZE/mpi_size;
+    count[1] = DIMSIZE;
+    count[2] = 1;
+    /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
+      mpi_rank, start[0], start[1], count[0], count[1]);*/
+
+    if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
+/*    if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
+
+    for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
+    {
+#ifdef USE_MPE
+       MPE_Log_event(s_write, 0, "start write slab");
+#endif /* USE_MPE */
+
+       /* Write slabs of phoney data. */
+       if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
+#ifdef USE_MPE
+       MPE_Log_event(e_write, 0, "end write file");
+#endif /* USE_MPE */
+    }
+
+#ifdef USE_MPE
+    MPE_Log_event(s_close, 0, "start close file");
+#endif /* USE_MPE */
+
+    /* Close the netcdf file. */
+    if ((res = nc_close(ncid)))	ERR;
+
+#ifdef USE_MPE
+    MPE_Log_event(e_close, 0, "end close file");
+#endif /* USE_MPE */
+
+    /* Delete this large file. */
+    /* remove(file_name); */
+
+    /* Shut down MPI. */
+    MPI_Finalize();
+
+    if (mpi_rank == 0)
+    {
+       SUMMARIZE_ERR;
+       FINAL_RESULTS;
+    }
+    return 0;
+}

From 2a415dba7cfb2410a7df88304b71301ec4aa446b Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 05:22:53 -0700
Subject: [PATCH 03/28] stop running tst_mode test

---
 nc_test4/run_par_test.sh.in |  7 +++-
 nc_test4/tst_mode.c         | 76 ++++++++++++++++++-------------------
 2 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in
index 2bd12fff4..cd789f237 100644
--- a/nc_test4/run_par_test.sh.in
+++ b/nc_test4/run_par_test.sh.in
@@ -8,8 +8,6 @@
 set -x
 set -e
 echo
-echo "Testing MPI parallel I/O with various other mode flags..."
-@MPIEXEC@ -n 1 ./tst_mode
 echo
 echo "Testing MPI parallel I/O without netCDF..."
 @MPIEXEC@ -n 4 ./tst_mpi_parallel
@@ -42,3 +40,8 @@ echo "Parallel I/O test for Collective I/O, contributed by HDF Group."
 @MPIEXEC@ -n 1 ./tst_simplerw_coll_r
 @MPIEXEC@ -n 2 ./tst_simplerw_coll_r
 @MPIEXEC@ -n 4 ./tst_simplerw_coll_r
+
+echo
+echo "Parallel I/O test with zlib."
+@MPIEXEC@ -n 1 ./tst_parallel_zlib
+@MPIEXEC@ -n 4 ./tst_parallel_zlib
diff --git a/nc_test4/tst_mode.c b/nc_test4/tst_mode.c
index 67996ed4d..03d0f6c0d 100644
--- a/nc_test4/tst_mode.c
+++ b/nc_test4/tst_mode.c
@@ -1,16 +1,16 @@
 /*! \file
 
-Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
-2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
-2015, 2016, 2017, 2018
-University Corporation for Atmospheric Research/Unidata.
+  Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
+  2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
+  2015, 2016, 2017, 2018
+  University Corporation for Atmospheric Research/Unidata.
 
-See \ref copyright file for more info.
+  See \ref copyright file for more info.
 
 
- * Test some illegal mode combinations
- *
- */
+  * Test some illegal mode combinations
+  *
+  */
 
 #include "nc_tests.h"
 #include "err_macros.h"
@@ -21,37 +21,37 @@ See \ref copyright file for more info.
 int
 main(int argc, char** argv)
 {
-   int ncid,varid;
-   int retval;
+    int ncid,varid;
+    int retval;
 
-   printf("\n*** Testing illegal mode combinations\n");
+    printf("\n*** Testing illegal mode combinations\n");
 
-   MPI_Init(&argc,&argv);
+    MPI_Init(&argc,&argv);
 
-   printf("*** Testing create + MPIO + fletcher32\n");
-   if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
-   if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
-   retval = nc_def_var_fletcher32(ncid,varid,NC_FLETCHER32);
-   if(retval != NC_EINVAL) ERR;
-   if ((retval = nc_abort(ncid)))
-{
-fprintf(stderr,"XXX: err=%d\n",retval);
-fflush(stderr);
-ERR;
-}
-
-   printf("*** Testing create + MPIO + deflation\n");
-   if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
-   if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
-   retval = nc_def_var_deflate(ncid,varid, NC_NOSHUFFLE, 1, 1);
-   if(retval != NC_EINVAL) ERR;
-   if ((retval = nc_abort(ncid))) {
-	fprintf(stderr,"XXX: nc_abort: %d\n",retval); fflush(stderr);
-	ERR;
-   }
-
-   MPI_Finalize();
-
-   SUMMARIZE_ERR;
-   FINAL_RESULTS;
+    printf("*** Testing create + MPIO + fletcher32\n");
+    if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
+    if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
+    retval = nc_def_var_fletcher32(ncid,varid,NC_FLETCHER32);
+    if(retval != NC_EINVAL) ERR;
+    if ((retval = nc_abort(ncid)))
+    {
+        fprintf(stderr,"XXX: err=%d\n",retval);
+        fflush(stderr);
+        ERR;
+    }
+
+    printf("*** Testing create + MPIO + deflation\n");
+    if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
+    if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
+    retval = nc_def_var_deflate(ncid,varid, NC_NOSHUFFLE, 1, 1);
+    if(retval != NC_EINVAL) ERR;
+    if ((retval = nc_abort(ncid))) {
+        fprintf(stderr,"XXX: nc_abort: %d\n",retval); fflush(stderr);
+        ERR;
+    }
+
+    MPI_Finalize();
+
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
 }

From d534b1298a0d80240e257ea31626f0dca8127c95 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 05:28:20 -0700
Subject: [PATCH 04/28] adding another zlib parallel I/O test

---
 libhdf5/hdf5var.c             |  6 +--
 nc_test4/Makefile.am          |  2 +-
 nc_test4/tst_parallel_zlib2.c | 99 +++++++++++++++++++++++++++++++++++
 3 files changed, 103 insertions(+), 4 deletions(-)
 create mode 100644 nc_test4/tst_parallel_zlib2.c

diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 0ffc2f544..69735e14d 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -662,9 +662,9 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     /* if (h5->parallel == NC_TRUE) */
     /*     if (deflate || fletcher32 || shuffle) */
     /*         return NC_EINVAL; */
-    if (h5->parallel == NC_TRUE)
-        if (fletcher32)
-            return NC_EINVAL;
+    /* if (h5->parallel == NC_TRUE) */
+    /*     if (fletcher32) */
+    /*         return NC_EINVAL; */
 
     /* If the HDF5 dataset has already been created, then it is too
      * late to set all the extra stuff. */
diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am
index c29071fe9..da4c93b46 100644
--- a/nc_test4/Makefile.am
+++ b/nc_test4/Makefile.am
@@ -84,7 +84,7 @@ endif # BUILD_UTILITIES
 if TEST_PARALLEL4
 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3		\
 tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r	\
-tst_mode tst_parallel_zlib
+tst_mode tst_parallel_zlib tst_parallel_zlib2
 TESTS += run_par_test.sh
 endif
 
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
new file mode 100644
index 000000000..999a20cf2
--- /dev/null
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -0,0 +1,99 @@
+/* Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and
+redistribution conditions.
+
+This program tests netcdf-4 parallel I/O using zlib compression while
+writing.
+
+Ed Hartnett, 12/19/2019
+*/
+
+#include <nc_tests.h>
+#include "err_macros.h"
+#include <mpi.h>
+
+#define FILE "tst_parallel_zlib2.nc"
+#define NDIMS 3
+#define DIMSIZE 24
+#define QTR_DATA (DIMSIZE * DIMSIZE / 4)
+#define NUM_PROC 4
+#define NUM_SLABS 10
+
+int
+main(int argc, char **argv)
+{
+    /* MPI stuff. */
+    int mpi_size, mpi_rank;
+    MPI_Comm comm = MPI_COMM_WORLD;
+    MPI_Info info = MPI_INFO_NULL;
+
+    /* Netcdf-4 stuff. */
+    int ncid, v1id, dimids[NDIMS];
+    size_t start[NDIMS], count[NDIMS];
+
+    int i, res;
+    int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
+    char file_name[NC_MAX_NAME + 1];
+
+    /* Initialize MPI. */
+    MPI_Init(&argc, &argv);
+    MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
+    MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
+
+    if (mpi_rank == 0)
+    {
+       printf("\n*** Testing parallel writes with zlib some more.\n");
+       printf("*** testing simple write with zlib...");
+    }
+
+    /* Create phony data. We're going to write a 24x24 array of ints,
+       in 4 sets of 144. */
+    for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
+       slab_data[i] = mpi_rank;
+
+    /* Create a parallel netcdf-4 file. */
+    /*nc_set_log_level(3);*/
+    /* sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); */
+    sprintf(file_name, "%s", FILE);
+    if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR;
+
+    /* Create three dimensions. */
+    if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
+    if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
+    if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
+
+    /* Create one var. */
+    if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
+    if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
+
+    /* Write metadata to file. */
+    if ((res = nc_enddef(ncid))) ERR;
+
+    /* Set up slab for this process. */
+    start[0] = mpi_rank * DIMSIZE/mpi_size;
+    start[1] = 0;
+    count[0] = DIMSIZE/mpi_size;
+    count[1] = DIMSIZE;
+    count[2] = 1;
+    /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
+      mpi_rank, start[0], start[1], count[0], count[1]);*/
+
+    if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
+/*    if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
+
+    /* Write slabs of phoney data. */
+    for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
+       if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
+
+    /* Close the netcdf file. */
+    if ((res = nc_close(ncid)))	ERR;
+
+    /* Shut down MPI. */
+    MPI_Finalize();
+
+    if (mpi_rank == 0)
+    {
+       SUMMARIZE_ERR;
+       FINAL_RESULTS;
+    }
+    return 0;
+}

From 6952eb779b313dda0538596a58e5c44b186483a0 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 05:36:09 -0700
Subject: [PATCH 05/28] documentation updates

---
 libdispatch/dvar.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c
index bafa5389c..1b9c94bb2 100644
--- a/libdispatch/dvar.c
+++ b/libdispatch/dvar.c
@@ -312,7 +312,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
 }
 
 /**
-   Set the compression settings for a netCDF-4/HDF5 variable.
+   Set the zlib compression settings for a netCDF-4/HDF5 variable.
 
    This function must be called after nc_def_var and before nc_enddef
    or any functions which writes data to the file.
@@ -324,15 +324,18 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
 
    If this function is called on a scalar variable, it is ignored.
 
+   @note Parallel I/O reads work with compressed data. Parallel I/O
+   writes work with compressed data starting in netcdf-c-4.7.4 and
+   later release, using hdf5-1.10.2 and later releases.
+
    @param ncid NetCDF or group ID, from a previous call to nc_open(),
    nc_create(), nc_def_grp(), or associated inquiry functions such as
    nc_inq_ncid().
    @param varid Variable ID
    @param shuffle True to turn on the shuffle filter. The shuffle
-   filter can assist with the compression of integer data by changing
-   the byte order in the data stream. It makes no sense to use the
-   shuffle filter without setting a deflate level, or to use shuffle
-   on non-integer data.
+   filter can assist with the compression of data by changing the byte
+   order in the data stream. It makes no sense to use the shuffle
+   filter without setting a deflate level.
    @param deflate True to turn on deflation for this variable.
    @param deflate_level A number between 0 (no compression) and 9
    (maximum compression).
@@ -347,11 +350,8 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
    @return ::NC_ELATEDEF Too late to change settings for this variable.
    @return ::NC_ENOTINDEFINE Not in define mode.
    @return ::NC_EPERM File is read only.
-   @return ::NC_EMAXDIMS Classic model file exceeds ::NC_MAX_VAR_DIMS.
    @return ::NC_ESTRICTNC3 Attempting to create netCDF-4 type var in
    classic model file
-   @return ::NC_EBADTYPE Bad type.
-   @return ::NC_ENOMEM Out of memory.
    @return ::NC_EHDFERR Error returned by HDF5 layer.
    @return ::NC_EINVAL Invalid input. Deflate can't be set unless
    variable storage is NC_CHUNK.

From f86c0fb8f9bb9b3393fa2db3cea7e93d8036eb34 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 05:54:21 -0700
Subject: [PATCH 06/28] now check that HDF5 version supports parallel zlib

---
 configure.ac      | 11 +++++++++--
 libhdf5/hdf5var.c | 14 +++++++-------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/configure.ac b/configure.ac
index 584b32efd..72acfaf04 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1038,7 +1038,7 @@ if test "x$enable_hdf5" = xyes; then
 
    # H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
    # Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.
-   AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops H5Z_SZIP])
+   AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops H5Z_SZIP H5DOread_chunk])
 
    # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
    if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
@@ -1053,7 +1053,14 @@ if test "x$enable_hdf5" = xyes; then
    AC_MSG_CHECKING([whether parallel io is enabled in hdf5])
    AC_MSG_RESULT([$hdf5_parallel])
 
-   # Check to see if we need to search for and link against szlib.
+   # Check to see if HDF5 library is 1.10.2 or greater. If so, allows parallel_zip.
+   if test "x$ac_cv_func_H5DOread_chunk" = xyes; then
+      AC_DEFINE([HDF5_1_10_2], [1], [if true, HDF5 is at least version 1.10.2 and allows parallel I/O with zip])
+   fi
+   AC_MSG_CHECKING([whether HDF5 is version 1.10.2 or greater])
+   AC_MSG_RESULT([$ac_cv_func_H5DOread_chunk])
+
+# Check to see if we need to search for and link against szlib.
    if test "x$ac_cv_func_H5Z_SZIP" = xyes; then
       AC_SEARCH_LIBS([SZ_BufftoBuffCompress], [szip sz], [],
       [AC_MSG_ERROR([libhdf5 installed with szip support, but cannot find or link to the szip library.])])
diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 69735e14d..32ecd7813 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -658,13 +658,13 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
         return NC_ENOTVAR;
     assert(var && var->hdr.id == varid);
 
-    /* Can't turn on parallel and deflate/fletcher32/szip/shuffle (for now). */
-    /* if (h5->parallel == NC_TRUE) */
-    /*     if (deflate || fletcher32 || shuffle) */
-    /*         return NC_EINVAL; */
-    /* if (h5->parallel == NC_TRUE) */
-    /*     if (fletcher32) */
-    /*         return NC_EINVAL; */
+    /* Can't turn on parallel and deflate/fletcher32/szip/shuffle
+     * before HDF5 1.10.2. */
+#ifndef HDF5_1_10_2
+    if (h5->parallel == NC_TRUE)
+        if (deflate || fletcher32 || shuffle)
+            return NC_EINVAL;
+#endif
 
     /* If the HDF5 dataset has already been created, then it is too
      * late to set all the extra stuff. */

From 6d2d92ddec5aac301d70541aca5b6fc5b94f19e9 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 06:30:23 -0700
Subject: [PATCH 07/28] added new tests to cmake, also additional test
 development

---
 nc_test4/CMakeLists.txt       |  1 +
 nc_test4/run_par_test.sh.in   |  5 +++
 nc_test4/tst_parallel_zlib2.c | 77 +++++++++++++++++------------------
 3 files changed, 43 insertions(+), 40 deletions(-)

diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt
index 524312681..48a017eba 100644
--- a/nc_test4/CMakeLists.txt
+++ b/nc_test4/CMakeLists.txt
@@ -86,6 +86,7 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_parallel4)
   build_bin_test(tst_parallel5)
   build_bin_test(tst_parallel_zlib)
+  build_bin_test(tst_parallel_zlib2)
   build_bin_test(tst_nc4perf)
   build_bin_test(tst_mode)
   build_bin_test(tst_simplerw_coll_r)
diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in
index cd789f237..4fcb680df 100644
--- a/nc_test4/run_par_test.sh.in
+++ b/nc_test4/run_par_test.sh.in
@@ -45,3 +45,8 @@ echo
 echo "Parallel I/O test with zlib."
 @MPIEXEC@ -n 1 ./tst_parallel_zlib
 @MPIEXEC@ -n 4 ./tst_parallel_zlib
+
+echo
+echo "Parallel I/O more tests with zlib."
+@MPIEXEC@ -n 1 ./tst_parallel_zlib2
+@MPIEXEC@ -n 4 ./tst_parallel_zlib2
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index 999a20cf2..104511e3c 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -11,7 +11,7 @@ Ed Hartnett, 12/19/2019
 #include "err_macros.h"
 #include <mpi.h>
 
-#define FILE "tst_parallel_zlib2.nc"
+#define FILE_NAME "tst_parallel_zlib2.nc"
 #define NDIMS 3
 #define DIMSIZE 24
 #define QTR_DATA (DIMSIZE * DIMSIZE / 4)
@@ -32,68 +32,65 @@ main(int argc, char **argv)
 
     int i, res;
     int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
-    char file_name[NC_MAX_NAME + 1];
 
     /* Initialize MPI. */
     MPI_Init(&argc, &argv);
     MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
     MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
 
-    if (mpi_rank == 0)
-    {
-       printf("\n*** Testing parallel writes with zlib some more.\n");
-       printf("*** testing simple write with zlib...");
-    }
-
     /* Create phony data. We're going to write a 24x24 array of ints,
        in 4 sets of 144. */
     for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
        slab_data[i] = mpi_rank;
 
-    /* Create a parallel netcdf-4 file. */
-    /*nc_set_log_level(3);*/
-    /* sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); */
-    sprintf(file_name, "%s", FILE);
-    if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR;
+    if (!mpi_rank)
+    {
+       printf("\n*** Testing parallel writes with zlib some more.\n");
+       printf("*** testing simple write with zlib...");
+    }
+    {
+        /* Create a parallel netcdf-4 file. */
+        if ((res = nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid))) ERR;
 
-    /* Create three dimensions. */
-    if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
-    if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
-    if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
+        /* Create three dimensions. */
+        if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
+        if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
+        if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
 
-    /* Create one var. */
-    if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
-    if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
+        /* Create one var. Turn on deflation. */
+        if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
+        if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
 
-    /* Write metadata to file. */
-    if ((res = nc_enddef(ncid))) ERR;
+        /* Write metadata to file. */
+        if ((res = nc_enddef(ncid))) ERR;
 
-    /* Set up slab for this process. */
-    start[0] = mpi_rank * DIMSIZE/mpi_size;
-    start[1] = 0;
-    count[0] = DIMSIZE/mpi_size;
-    count[1] = DIMSIZE;
-    count[2] = 1;
-    /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
-      mpi_rank, start[0], start[1], count[0], count[1]);*/
+        /* Set up slab for this process. */
+        start[0] = mpi_rank * DIMSIZE/mpi_size;
+        start[1] = 0;
+        count[0] = DIMSIZE/mpi_size;
+        count[1] = DIMSIZE;
+        count[2] = 1;
+        /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
+          mpi_rank, start[0], start[1], count[0], count[1]);*/
 
-    if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
+        if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
 /*    if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
 
-    /* Write slabs of phoney data. */
-    for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
-       if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
+        /* Write slabs of data. */
+        for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
+            if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
 
-    /* Close the netcdf file. */
-    if ((res = nc_close(ncid)))	ERR;
+        /* Close the netcdf file. */
+        if ((res = nc_close(ncid))) ERR;
+    }
+    if (!mpi_rank)
+        SUMMARIZE_ERR;
 
     /* Shut down MPI. */
     MPI_Finalize();
 
-    if (mpi_rank == 0)
-    {
-       SUMMARIZE_ERR;
+    if (!mpi_rank)
        FINAL_RESULTS;
-    }
+
     return 0;
 }

From 663540b3a773d783bf4c513536da022aef10374c Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 06:44:14 -0700
Subject: [PATCH 08/28] adjusted tst_parallel_zlib to HDF5 version flag

---
 nc_test4/tst_parallel_zlib.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index 8d53f59d4..34c67276d 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -105,7 +105,14 @@ main(int argc, char **argv)
 
     /* Create one var. */
     if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
-    if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
+    res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
+    /* Setting deflate only will work for HDF5-1.10.2 and later
+     * versions. */
+#ifdef HDF5_1_10_2
+    if (res) ERR;
+#else
+    if (res != NC_EINVAL) ERR;
+#endif
 
     /* Write metadata to file. */
     if ((res = nc_enddef(ncid))) ERR;

From 0c6641eb9880e378224865fd044319858ffe1dfd Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 06:46:13 -0700
Subject: [PATCH 09/28] adjusted tst_parallel_zlib2 to HDF5 version flag

---
 nc_test4/tst_parallel_zlib.c  | 3 ++-
 nc_test4/tst_parallel_zlib2.c | 9 ++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index 34c67276d..9766061ff 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -105,9 +105,10 @@ main(int argc, char **argv)
 
     /* Create one var. */
     if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
-    res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
+
     /* Setting deflate only will work for HDF5-1.10.2 and later
      * versions. */
+    res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
 #ifdef HDF5_1_10_2
     if (res) ERR;
 #else
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index 104511e3c..338f1d05d 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -59,8 +59,15 @@ main(int argc, char **argv)
 
         /* Create one var. Turn on deflation. */
         if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
-        if ((res = nc_def_var_deflate(ncid, 0, 0, 1, 1))) ERR;
 
+        /* Setting deflate only will work for HDF5-1.10.2 and later
+         * versions. */
+        res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
+#ifdef HDF5_1_10_2
+        if (res) ERR;
+#else
+        if (res != NC_EINVAL) ERR;
+#endif
         /* Write metadata to file. */
         if ((res = nc_enddef(ncid))) ERR;
 

From f980743be1f7a4ba9cca9952f0b889f6abbb3820 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 06:48:31 -0700
Subject: [PATCH 10/28] now also can set fletcher32 in parallel I/O writes

---
 nc_test4/tst_parallel_zlib2.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index 338f1d05d..d9e9e4d44 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -68,6 +68,16 @@ main(int argc, char **argv)
 #else
         if (res != NC_EINVAL) ERR;
 #endif
+
+        /* Setting fletcher32 only will work for HDF5-1.10.2 and later
+         * versions. */
+        res = nc_def_var_fletcher32(ncid, 0, 1);
+#ifdef HDF5_1_10_2
+        if (res) ERR;
+#else
+        if (res != NC_EINVAL) ERR;
+#endif
+
         /* Write metadata to file. */
         if ((res = nc_enddef(ncid))) ERR;
 

From 4b7f8396660fd07b7c5bfae52b7fd37c2e103062 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:00:12 -0700
Subject: [PATCH 11/28] swtich to collective access when filters are applied

---
 libhdf5/hdf5var.c             | 15 +++++++++++++--
 nc_test4/tst_parallel_zlib2.c |  5 +++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 32ecd7813..6f7f11e3e 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -687,13 +687,18 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
         if (!var->ndims)
             return NC_NOERR;
 
-        /* Well, if we couldn't find any errors, I guess we have to take
-         * the users settings. Darn! */
+        /* Set the deflate settings. */
         var->contiguous = NC_FALSE;
         var->deflate = *deflate;
         if (*deflate)
             var->deflate_level = *deflate_level;
         LOG((3, "%s: *deflate_level %d", __func__, *deflate_level));
+
+        /* If deflate was turned on with parallel I/O writes, then
+         * switch to collective access. HDF5 requires collevtive
+         * access for filter use with parallel I/O. */
+        if (h5->parallel && var->deflate)
+            var->parallel_access = NC_COLLECTIVE;
     }
 
     /* Shuffle filter? */
@@ -708,6 +713,12 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     {
         var->fletcher32 = *fletcher32;
         var->contiguous = NC_FALSE;
+
+        /* If fletcher32 was turned on with parallel I/O writes, then
+         * switch to collective access. HDF5 requires collevtive
+         * access for filter use with parallel I/O. */
+        if (h5->parallel && var->deflate)
+            var->parallel_access = NC_COLLECTIVE;
     }
 
     /* Does the user want a contiguous dataset? Not so fast! Make sure
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index d9e9e4d44..ffc9a176f 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -49,6 +49,7 @@ main(int argc, char **argv)
        printf("*** testing simple write with zlib...");
     }
     {
+        nc_set_log_level(3);
         /* Create a parallel netcdf-4 file. */
         if ((res = nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid))) ERR;
 
@@ -90,8 +91,8 @@ main(int argc, char **argv)
         /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
           mpi_rank, start[0], start[1], count[0], count[1]);*/
 
-        if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
-/*    if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
+        /* if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR; */
+        /* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR; */
 
         /* Write slabs of data. */
         for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)

From 2136063d69f39b014de4a4569a2ae018061d1852 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:05:23 -0700
Subject: [PATCH 12/28] better documentation

---
 libdispatch/dvar.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c
index 1b9c94bb2..9e2de04d7 100644
--- a/libdispatch/dvar.c
+++ b/libdispatch/dvar.c
@@ -325,8 +325,12 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
    If this function is called on a scalar variable, it is ignored.
 
    @note Parallel I/O reads work with compressed data. Parallel I/O
-   writes work with compressed data starting in netcdf-c-4.7.4 and
-   later release, using hdf5-1.10.2 and later releases.
+   writes work with compressed data in netcdf-c-4.7.4 and later
+   releases, using hdf5-1.10.2 and later releases. Using the zlib (or
+   any) filter requires that collective access be used with the
+   variable. Turning on deflate for a variable in a file opened for
+   parallel I/O will automatically switch the access for that variable
+   to collective access.
 
    @param ncid NetCDF or group ID, from a previous call to nc_open(),
    nc_create(), nc_def_grp(), or associated inquiry functions such as
@@ -422,6 +426,14 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
    data, with default chunksizes. Use nc_def_var_chunking() to tune
    performance with user-defined chunksizes.
 
+   @note Parallel I/O reads work with fletcher32 encoded
+   data. Parallel I/O writes work with fletcher32 in netcdf-c-4.7.4
+   and later releases, using hdf5-1.10.2 and later releases. Using the
+   fletcher32 (or any) filter requires that collective access be used
+   with the variable. Turning on fletcher32 for a variable in a file
+   opened for parallel I/O will automatically switch the access for
+   that variable to collective access.
+
    @param ncid NetCDF or group ID, from a previous call to nc_open(),
    nc_create(), nc_def_grp(), or associated inquiry functions such as
    nc_inq_ncid().

From 8681b0d2414c39e9c82c9c5c14cdaf635df49b11 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:10:13 -0700
Subject: [PATCH 13/28] more documentaiton

---
 libdispatch/dparallel.c       | 10 +++++++++-
 nc_test4/tst_parallel_zlib2.c |  5 +++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/libdispatch/dparallel.c b/libdispatch/dparallel.c
index 21e5e5ee3..7862f0d49 100644
--- a/libdispatch/dparallel.c
+++ b/libdispatch/dparallel.c
@@ -305,6 +305,13 @@ To obtain a good I/O performance, users are recommended to use collective mode.
 In addition, switching between collective and independent I/O mode can be
 expensive.
 
+In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib and
+fletcher32 filters may be used when writing data with parallel
+I/O. The use of these filters require collective access. Turning on
+the zlib (deflate) or fletcher32 filter for a variable will
+automatically set its access to collective. Attempts to set access to
+independent will return ::NC_EINVAL.
+
 \param ncid NetCDF or group ID, from a previous call to nc_open_par(),
 nc_create_par(), nc_def_grp(), or associated inquiry functions such as
 nc_inq_ncid().
@@ -317,7 +324,8 @@ nc_inq_ncid().
 \returns ::NC_EBADID Invalid ncid passed.
 \returns ::NC_ENOTVAR Invalid varid passed.
 \returns ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
-\returns ::NC_EINVAL Invalid par_access specified.
+\returns ::NC_EINVAL Invalid par_access specified, or attempt to set
+filtered variable to independent access.
 
 <h1>Example</h1>
 
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index ffc9a176f..c405548a2 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -91,8 +91,9 @@ main(int argc, char **argv)
         /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
           mpi_rank, start[0], start[1], count[0], count[1]);*/
 
-        /* if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR; */
-        /* if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR; */
+        /* Should not be allowed to change access to independent,
+         * because filters are in use. */
+        if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR;
 
         /* Write slabs of data. */
         for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)

From 8f8dfac9f6eb87f3ad6f259ec26dc1e6c54f0295 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:16:26 -0700
Subject: [PATCH 14/28] now prevent attempt to change to independent access
 when deflate, shuffle, or fletcher32 filters are in use

---
 libsrc4/nc4var.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c
index b953bc821..f8cd4ee79 100644
--- a/libsrc4/nc4var.c
+++ b/libsrc4/nc4var.c
@@ -422,6 +422,13 @@ NC4_var_par_access(int ncid, int varid, int par_access)
     if (!var) return NC_ENOTVAR;
     assert(var->hdr.id == varid);
 
+    /* If zlib, shuffle, or fletcher32 filters are in use, then access
+     * must be collective. Fail an attempt to set such a variable to
+     * independent access. */
+    if ((var->deflate || var->shuffle || var->fletcher32) &&
+        par_access == NC_INDEPENDENT)
+        return NC_EINVAL;
+
     if (par_access)
         var->parallel_access = NC_COLLECTIVE;
     else

From accb83a8b5a548f33f25ae6c4c444a2e1d4a0d8a Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:20:02 -0700
Subject: [PATCH 15/28] even more documentation updates

---
 libdispatch/dvar.c | 10 +++++-----
 libhdf5/hdf5var.c  |  8 +++++++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c
index 9e2de04d7..76fb16fe1 100644
--- a/libdispatch/dvar.c
+++ b/libdispatch/dvar.c
@@ -326,11 +326,11 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
 
    @note Parallel I/O reads work with compressed data. Parallel I/O
    writes work with compressed data in netcdf-c-4.7.4 and later
-   releases, using hdf5-1.10.2 and later releases. Using the zlib (or
-   any) filter requires that collective access be used with the
-   variable. Turning on deflate for a variable in a file opened for
-   parallel I/O will automatically switch the access for that variable
-   to collective access.
+   releases, using hdf5-1.10.2 and later releases. Using the zlib,
+   shuffle (or any other) filter requires that collective access be
+   used with the variable. Turning on deflate and/or shuffle for a
+   variable in a file opened for parallel I/O will automatically
+   switch the access for that variable to collective access.
 
    @param ncid NetCDF or group ID, from a previous call to nc_open(),
    nc_create(), nc_def_grp(), or associated inquiry functions such as
diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index 6f7f11e3e..df40fa4aa 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -706,6 +706,12 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     {
         var->shuffle = *shuffle;
         var->contiguous = NC_FALSE;
+
+        /* If shuffle was turned on with parallel I/O writes, then
+         * switch to collective access. HDF5 requires collevtive
+         * access for filter use with parallel I/O. */
+        if (h5->parallel && var->shuffle)
+            var->parallel_access = NC_COLLECTIVE;
     }
 
     /* Fletcher32 checksum error protection? */
@@ -717,7 +723,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
         /* If fletcher32 was turned on with parallel I/O writes, then
          * switch to collective access. HDF5 requires collevtive
          * access for filter use with parallel I/O. */
-        if (h5->parallel && var->deflate)
+        if (h5->parallel && var->fletcher32)
             var->parallel_access = NC_COLLECTIVE;
     }
 

From b9b5a8ff432e6927f457b04e28dc009457c95c1d Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:43:39 -0700
Subject: [PATCH 16/28] fixed include issue

---
 include/netcdf_par.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/netcdf_par.h b/include/netcdf_par.h
index a733a56c7..17303c7b4 100644
--- a/include/netcdf_par.h
+++ b/include/netcdf_par.h
@@ -22,15 +22,15 @@
 
 #include <mpi.h>
 
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
 /** Use with nc_var_par_access() to set parallel access to independent. */
 #define NC_INDEPENDENT 0
 /** Use with nc_var_par_access() to set parallel access to collective. */
 #define NC_COLLECTIVE 1
 
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
 /* Create a file and enable parallel I/O. */
     EXTERNL int
     nc_create_par(const char *path, int cmode, MPI_Comm comm, MPI_Info info,

From a06df0e4ebb35a1658ae99aa96b478eb1b2a58cc Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 07:52:00 -0700
Subject: [PATCH 17/28] fixing for non-parallel builds

---
 libhdf5/hdf5var.c | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index df40fa4aa..df798d6a1 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -693,12 +693,6 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
         if (*deflate)
             var->deflate_level = *deflate_level;
         LOG((3, "%s: *deflate_level %d", __func__, *deflate_level));
-
-        /* If deflate was turned on with parallel I/O writes, then
-         * switch to collective access. HDF5 requires collevtive
-         * access for filter use with parallel I/O. */
-        if (h5->parallel && var->deflate)
-            var->parallel_access = NC_COLLECTIVE;
     }
 
     /* Shuffle filter? */
@@ -706,12 +700,6 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     {
         var->shuffle = *shuffle;
         var->contiguous = NC_FALSE;
-
-        /* If shuffle was turned on with parallel I/O writes, then
-         * switch to collective access. HDF5 requires collevtive
-         * access for filter use with parallel I/O. */
-        if (h5->parallel && var->shuffle)
-            var->parallel_access = NC_COLLECTIVE;
     }
 
     /* Fletcher32 checksum error protection? */
@@ -719,13 +707,19 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
     {
         var->fletcher32 = *fletcher32;
         var->contiguous = NC_FALSE;
+    }
 
-        /* If fletcher32 was turned on with parallel I/O writes, then
-         * switch to collective access. HDF5 requires collevtive
-         * access for filter use with parallel I/O. */
-        if (h5->parallel && var->fletcher32)
+#ifdef USE_PARALLEL
+        /* If deflate, shuffle, or fletcher32 was turned on with
+         * parallel I/O writes, then switch to collective access. HDF5
+         * requires collevtive access for filter use with parallel
+         * I/O. */
+    if (deflate || shuffle || fletcher32)
+    {
+        if (h5->parallel && (var->deflate || var->shuffle || var->fletcher32))
             var->parallel_access = NC_COLLECTIVE;
     }
+#endif /* USE_PARALLEL */
 
     /* Does the user want a contiguous dataset? Not so fast! Make sure
      * that there are no unlimited dimensions, and no filters in use

From 32a776badf32ca29c5fd83fdd469fcd79a626624 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 09:43:56 -0700
Subject: [PATCH 18/28] getting parallel I/O with zlib working in cmake builds

---
 CMakeLists.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0a2771baa..af193a8a7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -743,6 +743,9 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
 	SET(HDF5_CC h5cc)
   ENDIF()
 
+  # Check to see if this is hdf5-1.10.2 or later.
+  CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5DOread_chunk "" HDF5_1_10_2)
+
   SET(H5_USE_16_API 1)
   OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
   IF(NOT NC_ENABLE_HDF_16_API)

From e6ded1a8eccfee94bfd7377715f7ff3c1ae0e4a2 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 11:42:54 -0700
Subject: [PATCH 19/28] more testing

---
 nc_test4/tst_parallel_zlib2.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index c405548a2..27878b7f6 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -51,7 +51,7 @@ main(int argc, char **argv)
     {
         nc_set_log_level(3);
         /* Create a parallel netcdf-4 file. */
-        if ((res = nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid))) ERR;
+        if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
 
         /* Create three dimensions. */
         if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
@@ -80,7 +80,7 @@ main(int argc, char **argv)
 #endif
 
         /* Write metadata to file. */
-        if ((res = nc_enddef(ncid))) ERR;
+        if (nc_enddef(ncid)) ERR;
 
         /* Set up slab for this process. */
         start[0] = mpi_rank * DIMSIZE/mpi_size;
@@ -100,7 +100,13 @@ main(int argc, char **argv)
             if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
 
         /* Close the netcdf file. */
-        if ((res = nc_close(ncid))) ERR;
+        if (nc_close(ncid)) ERR;
+
+        /* Reopen the file for parallel access. */
+        if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+
+        /* Close the netcdf file. */
+        if (nc_close(ncid)) ERR;
     }
     if (!mpi_rank)
         SUMMARIZE_ERR;

From 8e660e14fad25a92ef4e382de57a504cf41b6bed Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 11:48:40 -0700
Subject: [PATCH 20/28] more tests

---
 nc_test4/tst_parallel_zlib2.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index 27878b7f6..b0afe36b2 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -102,11 +102,20 @@ main(int argc, char **argv)
         /* Close the netcdf file. */
         if (nc_close(ncid)) ERR;
 
-        /* Reopen the file for parallel access. */
-        if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+        /* Check file. */
+        {
+            int shuffle_in, deflate_in, deflate_level_in;
 
-        /* Close the netcdf file. */
-        if (nc_close(ncid)) ERR;
+            /* Reopen the file for parallel access. */
+            if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+
+            /* Check state of deflate. */
+            if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
+            if (shuffle_in || !deflate_in || deflate_level_in != 1) ERR;
+
+            /* Close the netcdf file. */
+            if (nc_close(ncid)) ERR;
+        }
     }
     if (!mpi_rank)
         SUMMARIZE_ERR;

From 909212454a432fafad242831923fc687c73d6176 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 12:43:52 -0700
Subject: [PATCH 21/28] more testing

---
 nc_test4/tst_parallel_zlib2.c | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index b0afe36b2..468549cb6 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -31,16 +31,19 @@ main(int argc, char **argv)
     size_t start[NDIMS], count[NDIMS];
 
     int i, res;
-    int slab_data[DIMSIZE * DIMSIZE / 4]; /* one slab */
+    int *slab_data; /* one slab */
 
     /* Initialize MPI. */
     MPI_Init(&argc, &argv);
     MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
     MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
 
+    /* Allocate data. */
+    if (!(slab_data = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
+
     /* Create phony data. We're going to write a 24x24 array of ints,
        in 4 sets of 144. */
-    for (i = 0; i < DIMSIZE * DIMSIZE / 4; i++)
+    for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
        slab_data[i] = mpi_rank;
 
     if (!mpi_rank)
@@ -105,6 +108,10 @@ main(int argc, char **argv)
         /* Check file. */
         {
             int shuffle_in, deflate_in, deflate_level_in;
+            int *slab_data_in;
+
+            /* Allocate data. */
+            if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
 
             /* Reopen the file for parallel access. */
             if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
@@ -113,9 +120,21 @@ main(int argc, char **argv)
             if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
             if (shuffle_in || !deflate_in || deflate_level_in != 1) ERR;
 
+            /* Use parallel I/O to read the data. */
+            for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
+            {
+                if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR;
+                for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
+                    if (slab_data_in[i] != mpi_rank) ERR;
+            }
+
             /* Close the netcdf file. */
             if (nc_close(ncid)) ERR;
+
+            free(slab_data_in);
         }
+
+        free(slab_data);
     }
     if (!mpi_rank)
         SUMMARIZE_ERR;

From 522705423d92f203a2e40b152d5b8ce08a7f0280 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 12:51:36 -0700
Subject: [PATCH 22/28] more test development

---
 nc_test4/tst_parallel_zlib2.c | 155 ++++++++++++++++++----------------
 1 file changed, 81 insertions(+), 74 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index 468549cb6..cab464eb4 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -17,6 +17,7 @@ Ed Hartnett, 12/19/2019
 #define QTR_DATA (DIMSIZE * DIMSIZE / 4)
 #define NUM_PROC 4
 #define NUM_SLABS 10
+#define NUM_SHUFFLE_SETTINGS 2
 
 int
 main(int argc, char **argv)
@@ -49,95 +50,101 @@ main(int argc, char **argv)
     if (!mpi_rank)
     {
        printf("\n*** Testing parallel writes with zlib some more.\n");
-       printf("*** testing simple write with zlib...");
     }
     {
-        nc_set_log_level(3);
-        /* Create a parallel netcdf-4 file. */
-        if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
-
-        /* Create three dimensions. */
-        if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
-        if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
-        if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
-
-        /* Create one var. Turn on deflation. */
-        if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
-
-        /* Setting deflate only will work for HDF5-1.10.2 and later
-         * versions. */
-        res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
-#ifdef HDF5_1_10_2
-        if (res) ERR;
-#else
-        if (res != NC_EINVAL) ERR;
-#endif
-
-        /* Setting fletcher32 only will work for HDF5-1.10.2 and later
-         * versions. */
-        res = nc_def_var_fletcher32(ncid, 0, 1);
-#ifdef HDF5_1_10_2
-        if (res) ERR;
-#else
-        if (res != NC_EINVAL) ERR;
-#endif
-
-        /* Write metadata to file. */
-        if (nc_enddef(ncid)) ERR;
-
-        /* Set up slab for this process. */
-        start[0] = mpi_rank * DIMSIZE/mpi_size;
-        start[1] = 0;
-        count[0] = DIMSIZE/mpi_size;
-        count[1] = DIMSIZE;
-        count[2] = 1;
-        /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
-          mpi_rank, start[0], start[1], count[0], count[1]);*/
-
-        /* Should not be allowed to change access to independent,
-         * because filters are in use. */
-        if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR;
-
-        /* Write slabs of data. */
-        for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
-            if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
-
-        /* Close the netcdf file. */
-        if (nc_close(ncid)) ERR;
-
-        /* Check file. */
+        int s;
+        for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
         {
-            int shuffle_in, deflate_in, deflate_level_in;
-            int *slab_data_in;
+            printf("*** testing simple write with zlib shuffle %d...", s);
 
-            /* Allocate data. */
-            if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
+            /* nc_set_log_level(3); */
+            /* Create a parallel netcdf-4 file. */
+            if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
 
-            /* Reopen the file for parallel access. */
-            if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+            /* Create three dimensions. */
+            if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR;
+            if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[1])) ERR;
+            if (nc_def_dim(ncid, "d3", NUM_SLABS, &dimids[2])) ERR;
 
-            /* Check state of deflate. */
-            if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
-            if (shuffle_in || !deflate_in || deflate_level_in != 1) ERR;
+            /* Create one var. Turn on deflation. */
+            if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
 
-            /* Use parallel I/O to read the data. */
+            /* Setting deflate only will work for HDF5-1.10.2 and later
+             * versions. */
+            res = nc_def_var_deflate(ncid, 0, s, 1, 1);
+#ifdef HDF5_1_10_2
+            if (res) ERR;
+#else
+            if (res != NC_EINVAL) ERR;
+#endif
+
+            /* Setting fletcher32 only will work for HDF5-1.10.2 and later
+             * versions. */
+            res = nc_def_var_fletcher32(ncid, 0, 1);
+#ifdef HDF5_1_10_2
+            if (res) ERR;
+#else
+            if (res != NC_EINVAL) ERR;
+#endif
+
+            /* Write metadata to file. */
+            if (nc_enddef(ncid)) ERR;
+
+            /* Set up slab for this process. */
+            start[0] = mpi_rank * DIMSIZE/mpi_size;
+            start[1] = 0;
+            count[0] = DIMSIZE/mpi_size;
+            count[1] = DIMSIZE;
+            count[2] = 1;
+            /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
+              mpi_rank, start[0], start[1], count[0], count[1]);*/
+
+            /* Should not be allowed to change access to independent,
+             * because filters are in use. */
+            if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR;
+
+            /* Write slabs of data. */
             for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
-            {
-                if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR;
-                for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
-                    if (slab_data_in[i] != mpi_rank) ERR;
-            }
+                if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
 
             /* Close the netcdf file. */
             if (nc_close(ncid)) ERR;
 
-            free(slab_data_in);
-        }
+            /* Check file. */
+            {
+                int shuffle_in, deflate_in, deflate_level_in;
+                int *slab_data_in;
 
+                /* Allocate data. */
+                if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
+
+                /* Reopen the file for parallel access. */
+                if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
+
+                /* Check state of deflate. */
+                if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
+                if ((s && !shuffle_in) || (!s && shuffle_in)) ERR;
+                if (!deflate_in || deflate_level_in != 1) ERR;
+
+                /* Use parallel I/O to read the data. */
+                for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
+                {
+                    if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR;
+                    for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
+                        if (slab_data_in[i] != mpi_rank) ERR;
+                }
+
+                /* Close the netcdf file. */
+                if (nc_close(ncid)) ERR;
+
+                free(slab_data_in);
+            }
+
+            if (!mpi_rank)
+                SUMMARIZE_ERR;
+        } /* next shuffle filter test */
         free(slab_data);
     }
-    if (!mpi_rank)
-        SUMMARIZE_ERR;
 
     /* Shut down MPI. */
     MPI_Finalize();

From c16e7ef3cecb0e0e2c1c78d46b5748c28b4ba9b0 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 12:52:38 -0700
Subject: [PATCH 23/28] more test development

---
 nc_test4/tst_parallel_zlib2.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index cab464eb4..fffadc58c 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -1,10 +1,10 @@
 /* Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and
-redistribution conditions.
+   redistribution conditions.
 
-This program tests netcdf-4 parallel I/O using zlib compression while
-writing.
+   This program tests netcdf-4 parallel I/O using zlib compression while
+   writing.
 
-Ed Hartnett, 12/19/2019
+   Ed Hartnett, 12/19/2019
 */
 
 #include <nc_tests.h>

From 680e44f628fdf325181fa060bcd2af74fcc3f129 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 13:58:01 -0700
Subject: [PATCH 24/28] changed name of macro

---
 CMakeLists.txt                | 2 +-
 configure.ac                  | 2 +-
 libhdf5/hdf5var.c             | 2 +-
 nc_test4/tst_parallel_zlib.c  | 4 ++--
 nc_test4/tst_parallel_zlib2.c | 4 ++--
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index af193a8a7..e31217fa4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -744,7 +744,7 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
   ENDIF()
 
   # Check to see if this is hdf5-1.10.2 or later.
-  CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5DOread_chunk "" HDF5_1_10_2)
+  CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5DOread_chunk "" HDF5_SUPPORTS_PAR_FILTERS)
 
   SET(H5_USE_16_API 1)
   OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
diff --git a/configure.ac b/configure.ac
index 72acfaf04..dcd216bf8 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1055,7 +1055,7 @@ if test "x$enable_hdf5" = xyes; then
 
    # Check to see if HDF5 library is 1.10.2 or greater. If so, allows parallel_zip.
    if test "x$ac_cv_func_H5DOread_chunk" = xyes; then
-      AC_DEFINE([HDF5_1_10_2], [1], [if true, HDF5 is at least version 1.10.2 and allows parallel I/O with zip])
+      AC_DEFINE([HDF5_SUPPORTS_PAR_FILTERS], [1], [if true, HDF5 is at least version 1.10.2 and allows parallel I/O with zip])
    fi
    AC_MSG_CHECKING([whether HDF5 is version 1.10.2 or greater])
    AC_MSG_RESULT([$ac_cv_func_H5DOread_chunk])
diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c
index a17a3f1c6..c13825d47 100644
--- a/libhdf5/hdf5var.c
+++ b/libhdf5/hdf5var.c
@@ -663,7 +663,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
 
     /* Can't turn on parallel and deflate/fletcher32/szip/shuffle
      * before HDF5 1.10.2. */
-#ifndef HDF5_1_10_2
+#ifndef HDF5_SUPPORTS_PAR_FILTERS
     if (h5->parallel == NC_TRUE)
         if (deflate || fletcher32 || shuffle)
             return NC_EINVAL;
diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index 9766061ff..69e44a682 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -109,7 +109,7 @@ main(int argc, char **argv)
     /* Setting deflate only will work for HDF5-1.10.2 and later
      * versions. */
     res = nc_def_var_deflate(ncid, 0, 0, 1, 1);
-#ifdef HDF5_1_10_2
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
     if (res) ERR;
 #else
     if (res != NC_EINVAL) ERR;
@@ -133,8 +133,8 @@ main(int argc, char **argv)
     /*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
       mpi_rank, start[0], start[1], count[0], count[1]);*/
 
+    /* Not necessary, but harmless. */
     if (nc_var_par_access(ncid, v1id, NC_COLLECTIVE)) ERR;
-/*    if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT)) ERR;*/
 
     for (start[2] = 0; start[2] < NUM_SLABS; start[2]++)
     {
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index fffadc58c..a4e5441af 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -72,7 +72,7 @@ main(int argc, char **argv)
             /* Setting deflate only will work for HDF5-1.10.2 and later
              * versions. */
             res = nc_def_var_deflate(ncid, 0, s, 1, 1);
-#ifdef HDF5_1_10_2
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
             if (res) ERR;
 #else
             if (res != NC_EINVAL) ERR;
@@ -81,7 +81,7 @@ main(int argc, char **argv)
             /* Setting fletcher32 only will work for HDF5-1.10.2 and later
              * versions. */
             res = nc_def_var_fletcher32(ncid, 0, 1);
-#ifdef HDF5_1_10_2
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
             if (res) ERR;
 #else
             if (res != NC_EINVAL) ERR;

From 9ded9b1dfdfdcf01dfb49e3ce89d287ff07ad780 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 14:00:59 -0700
Subject: [PATCH 25/28] restored tst_mode tests

---
 nc_test4/run_par_test.sh.in | 2 ++
 nc_test4/tst_mode.c         | 8 ++++++++
 2 files changed, 10 insertions(+)

diff --git a/nc_test4/run_par_test.sh.in b/nc_test4/run_par_test.sh.in
index 4fcb680df..ec931c4e2 100644
--- a/nc_test4/run_par_test.sh.in
+++ b/nc_test4/run_par_test.sh.in
@@ -8,6 +8,8 @@
 set -x
 set -e
 echo
+echo "Testing MPI parallel I/O with various other mode flags..."
+@MPIEXEC@ -n 1 ./tst_mode
 echo
 echo "Testing MPI parallel I/O without netCDF..."
 @MPIEXEC@ -n 4 ./tst_mpi_parallel
diff --git a/nc_test4/tst_mode.c b/nc_test4/tst_mode.c
index 03d0f6c0d..a0a69cf88 100644
--- a/nc_test4/tst_mode.c
+++ b/nc_test4/tst_mode.c
@@ -32,7 +32,11 @@ main(int argc, char** argv)
     if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
     if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
     retval = nc_def_var_fletcher32(ncid,varid,NC_FLETCHER32);
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
+    if(retval != NC_NOERR) ERR;
+#else
     if(retval != NC_EINVAL) ERR;
+#endif
     if ((retval = nc_abort(ncid)))
     {
         fprintf(stderr,"XXX: err=%d\n",retval);
@@ -44,7 +48,11 @@ main(int argc, char** argv)
     if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
     if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
     retval = nc_def_var_deflate(ncid,varid, NC_NOSHUFFLE, 1, 1);
+#ifdef HDF5_SUPPORTS_PAR_FILTERS
+    if(retval != NC_NOERR) ERR;
+#else
     if(retval != NC_EINVAL) ERR;
+#endif
     if ((retval = nc_abort(ncid))) {
         fprintf(stderr,"XXX: nc_abort: %d\n",retval); fflush(stderr);
         ERR;

From 61477db38787e6c66af4bbf9029b6c5f98b07254 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 14:03:48 -0700
Subject: [PATCH 26/28] fixed comments

---
 nc_test4/tst_parallel_zlib.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index 69e44a682..14c0353d4 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -1,11 +1,13 @@
 /*
-Copyright 2018, UCAR/Unidata
-See COPYRIGHT file for copying and redistribution conditions.
+  Copyright 2019, UCAR/Unidata
+  See COPYRIGHT file for copying and redistribution conditions.
 
-This program tests netcdf-4 parallel I/O using zlib compression while
-writing.
+  This program tests netcdf-4 parallel I/O using zlib, shuffle, and
+  fletcher32 fliters while writing with parallel I/O. This works for
+  HDF5-1.10.2 and later. In this case HDF5_SUPPORTS_PAR_FILTERS will
+  be defined during configure.
 
-Ed Hartnett, 2019/12/19
+  Ed Hartnett, 12/19/2019
 */
 
 /* Defining USE_MPE causes the MPE trace library to be used (and you

From c4a8671a0d23f62719974428c677eb7c5bb1c59f Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 14:04:57 -0700
Subject: [PATCH 27/28] fixed comments

---
 nc_test4/tst_parallel_zlib.c  | 8 ++++----
 nc_test4/tst_parallel_zlib2.c | 9 ++++++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index 14c0353d4..bef05b0a9 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -2,10 +2,10 @@
   Copyright 2019, UCAR/Unidata
   See COPYRIGHT file for copying and redistribution conditions.
 
-  This program tests netcdf-4 parallel I/O using zlib, shuffle, and
-  fletcher32 fliters while writing with parallel I/O. This works for
-  HDF5-1.10.2 and later. In this case HDF5_SUPPORTS_PAR_FILTERS will
-  be defined during configure.
+  This program tests netcdf-4 parallel I/O using the zlib fliter while
+  writing with parallel I/O. This works for HDF5-1.10.2 and later. In
+  this case HDF5_SUPPORTS_PAR_FILTERS will be defined during
+  configure.
 
   Ed Hartnett, 12/19/2019
 */
diff --git a/nc_test4/tst_parallel_zlib2.c b/nc_test4/tst_parallel_zlib2.c
index a4e5441af..5979b5a2f 100644
--- a/nc_test4/tst_parallel_zlib2.c
+++ b/nc_test4/tst_parallel_zlib2.c
@@ -1,8 +1,11 @@
-/* Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and
+/*
+   Copyright 2019, UCAR/Unidata See COPYRIGHT file for copying and
    redistribution conditions.
 
-   This program tests netcdf-4 parallel I/O using zlib compression while
-   writing.
+   This program tests netcdf-4 parallel I/O using zlib, shuffle, and
+   fletcher32 fliters while writing with parallel I/O. This works for
+   HDF5-1.10.2 and later. In this case HDF5_SUPPORTS_PAR_FILTERS will
+   be defined during configure.
 
    Ed Hartnett, 12/19/2019
 */

From af0b28ffd5e0d0aea4d7845fc7cfc462f7ce48bb Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Fri, 20 Dec 2019 14:05:57 -0700
Subject: [PATCH 28/28] removed unneeded delete

---
 nc_test4/tst_parallel_zlib.c | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c
index bef05b0a9..74e7f423f 100644
--- a/nc_test4/tst_parallel_zlib.c
+++ b/nc_test4/tst_parallel_zlib.c
@@ -162,9 +162,6 @@ main(int argc, char **argv)
     MPE_Log_event(e_close, 0, "end close file");
 #endif /* USE_MPE */
 
-    /* Delete this large file. */
-    /* remove(file_name); */
-
     /* Shut down MPI. */
     MPI_Finalize();