From 9e0ee2790c243c089654b57ff564c3d3eaea3844 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Mon, 2 Sep 2024 05:47:45 -0600
Subject: [PATCH 1/9] adding test for HDF5 error message

---
 CMakeLists.txt          | 5 +++++
 configure.ac            | 1 +
 nc_test4/CMakeLists.txt | 2 ++
 nc_test4/Makefile.am    | 8 ++++----
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 27edd4d6a..fb8b12ea6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -953,6 +953,11 @@ if(NETCDF_ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
     file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_test.sh"
       DESTINATION ${netCDF_BINARY_DIR}/nc_test4
       FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+    configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_warn_test.sh.in"
+      "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
+    file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_warn_test.sh"
+      DESTINATION ${netCDF_BINARY_DIR}/nc_test4
+      FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
     configure_file("${netCDF_SOURCE_DIR}/h5_test/run_par_tests.sh.in"
       "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" @ONLY NEWLINE_STYLE LF)
     file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
diff --git a/configure.ac b/configure.ac
index 6ed7fdd05..8b7af0cf9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2247,6 +2247,7 @@ AC_CONFIG_FILES(dap4_test/findtestserver4.c:ncdap_test/findtestserver.c.in)
 AC_CONFIG_FILES(dap4_test/pingurl4.c:ncdap_test/pingurl.c)
 AC_CONFIG_FILES([h5_test/run_par_tests.sh], [chmod ugo+x h5_test/run_par_tests.sh])
 AC_CONFIG_FILES([nc_test4/run_par_test.sh], [chmod ugo+x nc_test4/run_par_test.sh])
+AC_CONFIG_FILES([nc_test4/run_par_warn_test.sh], [chmod ugo+x nc_test4/run_par_warn_test.sh])
 AC_CONFIG_FILES([nc_perf/run_par_bm_test.sh], [chmod ugo+x nc_perf/run_par_bm_test.sh])
 AC_CONFIG_FILES([nc_perf/run_gfs_test.sh], [chmod ugo+x nc_perf/run_gfs_test.sh])
 AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt
index 8463992aa..2b4927e48 100644
--- a/nc_test4/CMakeLists.txt
+++ b/nc_test4/CMakeLists.txt
@@ -132,7 +132,9 @@ IF(TEST_PARALLEL4)
   build_bin_test(tst_parallel_compress)
   build_bin_test(tst_nc4perf)
   build_bin_test(tst_mode)
+  build_bin_test(tst_warn)
   build_bin_test(tst_simplerw_coll_r)
   build_bin_test(tst_quantize_par)
   add_sh_test(nc_test4 run_par_test)
+  add_sh_test(nc_test4 run_par_warn_test)
 ENDIF()
diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am
index be76c6d88..5f09b7267 100644
--- a/nc_test4/Makefile.am
+++ b/nc_test4/Makefile.am
@@ -127,8 +127,8 @@ if TEST_PARALLEL4
 check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3		\
 tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r	\
 tst_mode tst_parallel_zlib tst_parallel_compress tst_quantize_par	\
-tst_parallel6
-TESTS += run_par_test.sh
+tst_parallel6 tst_warn
+TESTS += run_par_test.sh run_par_warn_test.sh
 endif # TEST_PARALLEL4
 
 EXTRA_DIST = run_par_test.sh.in ref_tst_compounds.nc			\
@@ -145,7 +145,7 @@ tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c		\
 noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl			\
 tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5			\
 ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh		\
-tst_filter_misc.sh run_zstd_test.sh
+tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in
 
 CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc	\
 tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt       \
@@ -160,7 +160,7 @@ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1               \
 tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp                 \
 tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl
 
-DISTCLEANFILES = findplugin.sh run_par_test.sh
+DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh
 
 clean-local:
 	rm -fr testdir_* testset_*

From f0272a2264c18b860ea5d396322c540574f4e250 Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Mon, 2 Sep 2024 06:09:07 -0600
Subject: [PATCH 2/9] adding test for HDF5 error message

---
 libhdf5/hdf5internal.c           |  8 +++++++-
 nc_test4/run_par_warn_test.sh.in | 19 +++++++++++++++++
 nc_test4/tst_warn.c              | 35 ++++++++++++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)
 create mode 100644 nc_test4/run_par_warn_test.sh.in
 create mode 100644 nc_test4/tst_warn.c

diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c
index 49f823d5b..ed1a3d91b 100644
--- a/libhdf5/hdf5internal.c
+++ b/libhdf5/hdf5internal.c
@@ -51,7 +51,13 @@ extern int nc_log_level;
 int nc4_hdf5_initialized = 0; /**< True if initialization has happened. */
 
 /**
- * @internal Provide a wrapper for H5Eset_auto
+ * @internal Provide a wrapper for H5Eset_auto.
+ *
+ * If preprocessor symbol DEBUGH5 is set (at the top of this file),
+ * then error messages will be pronted by the h5catch() function. If
+ * not, a NULL will be passed as the second argument to H5eset_auto2()
+ * and error messages will not be printed by HDF5.
+ *
  * @param func Pointer to func.
  * @param client_data Client data.
  *
diff --git a/nc_test4/run_par_warn_test.sh.in b/nc_test4/run_par_warn_test.sh.in
new file mode 100644
index 000000000..89a1a35c3
--- /dev/null
+++ b/nc_test4/run_par_warn_test.sh.in
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# This .in file is processed at build time into a shell that runs some
+# parallel I/O tests for netCDF/HDF5 parallel I/O. This tests for
+# unexpected output from the HDF5 library. See
+# https://github.com/Unidata/netcdf-c/issues/3003.
+
+# Ed Hartnett
+
+set -e
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+echo
+echo "Testing MPI parallel I/O with various other mode flags..."
+@MPIEXEC@ -n 1 ./tst_mode &> tst_mode_out.txt
+cat tst_mode_out.txt
+
diff --git a/nc_test4/tst_warn.c b/nc_test4/tst_warn.c
new file mode 100644
index 000000000..fee879016
--- /dev/null
+++ b/nc_test4/tst_warn.c
@@ -0,0 +1,35 @@
+/* Copyright 2024
+   University Corporation for Atmospheric Research/Unidata. */
+
+/* This program tests for undesired HDF5 output in parallel
+   programs. See https://github.com/Unidata/netcdf-c/issues/3003.
+
+   This is intended to be run on one processor only, but with MPI.
+   
+   Edward Hartnett, 9/2/24
+*/
+#include "nc_tests.h"
+#include "err_macros.h"
+#include "netcdf_par.h"
+
+#define FILE_NAME "tst_warn.nc"
+
+int
+main(int argc, char** argv)
+{
+    int ncid,varid,dimid;
+    int retval;
+
+    printf("\n*** Testing for HDF5 extra output\n");
+
+    MPI_Init(&argc,&argv);
+
+    printf("*** Testing create of parallel I/O file...");
+    if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
+    if ((retval = nc_close(ncid))) ERR;
+
+    MPI_Finalize();
+
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
+}

From b7a54758027ca7d46c40d8d5e0e77ed9092ffdaa Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Mon, 2 Sep 2024 06:22:23 -0600
Subject: [PATCH 3/9] adding test for HDF5 error message

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index fb8b12ea6..42a05f17e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -954,7 +954,7 @@ if(NETCDF_ENABLE_PARALLEL4 AND NETCDF_ENABLE_HDF5)
       DESTINATION ${netCDF_BINARY_DIR}/nc_test4
       FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
     configure_file("${netCDF_SOURCE_DIR}/nc_test4/run_par_warn_test.sh.in"
-      "${netCDF_BINARY_DIR}/tmp/run_par_test.sh" @ONLY NEWLINE_STYLE LF)
+      "${netCDF_BINARY_DIR}/tmp/run_par_warn_test.sh" @ONLY NEWLINE_STYLE LF)
     file(COPY "${netCDF_BINARY_DIR}/tmp/run_par_warn_test.sh"
       DESTINATION ${netCDF_BINARY_DIR}/nc_test4
       FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)

From 771b3a9081c10d62974f546cfe3bba3c3cf3fa7a Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Mon, 2 Sep 2024 06:59:37 -0600
Subject: [PATCH 4/9] adding test for HDF5 error message

---
 nc_test4/run_par_warn_test.sh.in | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nc_test4/run_par_warn_test.sh.in b/nc_test4/run_par_warn_test.sh.in
index 89a1a35c3..e71d305d0 100644
--- a/nc_test4/run_par_warn_test.sh.in
+++ b/nc_test4/run_par_warn_test.sh.in
@@ -14,6 +14,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
 
 echo
 echo "Testing MPI parallel I/O with various other mode flags..."
-@MPIEXEC@ -n 1 ./tst_mode &> tst_mode_out.txt
-cat tst_mode_out.txt
+@MPIEXEC@ -n 1 ./tst_warn &> tst_warn_out.txt
+cat tst_warn_out.txt
 

From 8b7a83b44429d4d635345824678daeb8aad9155d Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Wed, 4 Sep 2024 07:24:43 -0600
Subject: [PATCH 5/9] more testing of HDF5 error

---
 nc_test4/Makefile.am              | 5 +++--
 nc_test4/ref_tmp_tst_warn_out.txt | 8 ++++++++
 nc_test4/run_par_warn_test.sh.in  | 4 ++--
 3 files changed, 13 insertions(+), 4 deletions(-)
 create mode 100644 nc_test4/ref_tmp_tst_warn_out.txt

diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am
index 5f09b7267..683b9a114 100644
--- a/nc_test4/Makefile.am
+++ b/nc_test4/Makefile.am
@@ -145,7 +145,8 @@ tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c		\
 noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl			\
 tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5			\
 ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh		\
-tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in
+tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in		\
+ref_tmp_tst_warn_out.txt
 
 CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc	\
 tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt       \
@@ -160,7 +161,7 @@ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1               \
 tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp                 \
 tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl
 
-DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh
+DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh	
 
 clean-local:
 	rm -fr testdir_* testset_*
diff --git a/nc_test4/ref_tmp_tst_warn_out.txt b/nc_test4/ref_tmp_tst_warn_out.txt
new file mode 100644
index 000000000..ea5021c2e
--- /dev/null
+++ b/nc_test4/ref_tmp_tst_warn_out.txt
@@ -0,0 +1,8 @@
+
+*** Testing for HDF5 extra output
+*** Testing create of parallel I/O file...ok.
+*** Tests successful!
+HDF5-DIAG: Error detected in HDF5 (1.14.3) thread 0:
+  #000: H5FD.c line 373 in H5FDunregister(): not a file driver
+    major: Invalid arguments to routine
+    minor: Inappropriate type
diff --git a/nc_test4/run_par_warn_test.sh.in b/nc_test4/run_par_warn_test.sh.in
index e71d305d0..670f8db5c 100644
--- a/nc_test4/run_par_warn_test.sh.in
+++ b/nc_test4/run_par_warn_test.sh.in
@@ -14,6 +14,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
 
 echo
 echo "Testing MPI parallel I/O with various other mode flags..."
-@MPIEXEC@ -n 1 ./tst_warn &> tst_warn_out.txt
-cat tst_warn_out.txt
+@MPIEXEC@ -n 1 ./tst_warn > tmp_tst_warn_out.txt 2>&1
+cat tmp_tst_warn_out.txt
 

From 023f8b98068bc9a198dbb0e2790334cd187ff20d Mon Sep 17 00:00:00 2001
From: Edward Hartnett <edward.hartnett@noaa.gov>
Date: Wed, 4 Sep 2024 07:35:34 -0600
Subject: [PATCH 6/9] removed error message from ref file

---
 nc_test4/ref_tmp_tst_warn_out.txt | 5 +----
 nc_test4/run_par_warn_test.sh.in  | 1 +
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/nc_test4/ref_tmp_tst_warn_out.txt b/nc_test4/ref_tmp_tst_warn_out.txt
index ea5021c2e..00fdeb77e 100644
--- a/nc_test4/ref_tmp_tst_warn_out.txt
+++ b/nc_test4/ref_tmp_tst_warn_out.txt
@@ -2,7 +2,4 @@
 *** Testing for HDF5 extra output
 *** Testing create of parallel I/O file...ok.
 *** Tests successful!
-HDF5-DIAG: Error detected in HDF5 (1.14.3) thread 0:
-  #000: H5FD.c line 373 in H5FDunregister(): not a file driver
-    major: Invalid arguments to routine
-    minor: Inappropriate type
+
diff --git a/nc_test4/run_par_warn_test.sh.in b/nc_test4/run_par_warn_test.sh.in
index 670f8db5c..35abb7a8a 100644
--- a/nc_test4/run_par_warn_test.sh.in
+++ b/nc_test4/run_par_warn_test.sh.in
@@ -16,4 +16,5 @@ echo
 echo "Testing MPI parallel I/O with various other mode flags..."
 @MPIEXEC@ -n 1 ./tst_warn > tmp_tst_warn_out.txt 2>&1
 cat tmp_tst_warn_out.txt
+diff -w tmp_tst_warn_out.txt ref_tmp_tst_warn_out.txt
 

From 6b1a755771b363307cfcfd62e7d2992a430b6e77 Mon Sep 17 00:00:00 2001
From: Ward Fisher <wfisher@ucar.edu>
Date: Wed, 4 Sep 2024 11:54:12 -0600
Subject: [PATCH 7/9] Integrating fix suggested by jhendersonHDF in support of
 https://github.com/Unidata/netcdf-c/issues/2990

---
 libhdf5/H5FDhttp.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libhdf5/H5FDhttp.c b/libhdf5/H5FDhttp.c
index d48aeb8c3..917c73a11 100644
--- a/libhdf5/H5FDhttp.c
+++ b/libhdf5/H5FDhttp.c
@@ -257,7 +257,7 @@ EXTERNL hid_t
 H5FD_http_finalize(void)
 {
     /* Reset VFL ID */
-    if (H5FD_HTTP_g)
+    if (H5FD_HTTP_g && (H5Iis_valid(H5FD_HTTP_g) > 0))
          H5FDunregister(H5FD_HTTP_g);
     H5FD_HTTP_g = 0;
 

From 59e2b3a5f06238240332c2b8f4b378f9f26ca1eb Mon Sep 17 00:00:00 2001
From: Ward Fisher <wfisher@ucar.edu>
Date: Wed, 4 Sep 2024 17:58:09 -0500
Subject: [PATCH 8/9] Ensure that the reference file for run_par_warn_test is
 copied to the build directory for cmake builds.

---
 nc_test4/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt
index 2b4927e48..caa75aff1 100644
--- a/nc_test4/CMakeLists.txt
+++ b/nc_test4/CMakeLists.txt
@@ -110,7 +110,7 @@ IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING)
 ENDIF()
 
 # Copy some test files from current source dir to out-of-tree build dir.
-FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
+FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref_tmp_tst_warn_out.txt)
 FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
 IF(WIN32)
   FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)

From e2e45f8ccb9dee6fb89a684306354038f6c2d1bc Mon Sep 17 00:00:00 2001
From: Ward Fisher <wfisher@ucar.edu>
Date: Wed, 4 Sep 2024 18:17:46 -0500
Subject: [PATCH 9/9] Remove stray whitespace that was not being ignored by
 diff.

---
 nc_test4/ref_tmp_tst_warn_out.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nc_test4/ref_tmp_tst_warn_out.txt b/nc_test4/ref_tmp_tst_warn_out.txt
index 00fdeb77e..087ca56d2 100644
--- a/nc_test4/ref_tmp_tst_warn_out.txt
+++ b/nc_test4/ref_tmp_tst_warn_out.txt
@@ -2,4 +2,3 @@
 *** Testing for HDF5 extra output
 *** Testing create of parallel I/O file...ok.
 *** Tests successful!
-