From 4f75d0873bd4790a768d03acc832b43fa612392a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 17 Oct 2018 07:16:00 -0600 Subject: [PATCH 1/4] cleanup of Makefile.am --- nc_test/Makefile.am | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index 9e4a8c637..0d3374a59 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -79,11 +79,8 @@ if USE_PNETCDF TESTS += run_pnetcdf_test.sh endif -# Distribute the .c files so that m4 isn't required on the users -# machine. - -# Distribute the .c files so that m4 isn't required on the users -# machine. +# The .c files that are generated with m4 are already distributed, but +# we also include the original m4 files, plus test scripts data. EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \ run_diskless5.sh run_mmap.sh run_pnetcdf_test.sh test_read.m4 \ test_write.m4 ref_tst_diskless2.cdl tst_diskless5.cdl run_inmemory.sh \ @@ -91,13 +88,9 @@ f03tst_open_mem.nc \ CMakeLists.txt # These files are created by the tests. -CLEANFILES = nc_test_classic.nc nc_test_64bit.nc nc_test_netcdf4.nc \ -tst_*.nc t_nc.nc large_files.nc quick_large_files.nc tst_diskless.nc \ -tst_diskless2.nc tst_diskless3.nc tst_diskless3_file.cdl \ -tst_diskless3_memory.cdl tst_diskless4.cdl tst_diskless4.nc \ -tst_formatx.nc nc_test_cdf5.nc tst_inq_type.nc tst_elatefill.nc \ -tst_global_fillval.nc tst_large_cdf5.nc tst_max_var_dims.nc \ -benchmark.nc tst_def_var_fill.nc +CLEANFILES = nc_test_*.nc tst_*.nc t_nc.nc large_files.nc \ +quick_large_files.nc tst_diskless3_file.cdl tst_diskless3_memory.cdl \ +tst_diskless4.cdl benchmark.nc EXTRA_DIST += bad_cdf5_begin.nc run_cdf5.sh if ENABLE_CDF5 @@ -108,7 +101,6 @@ if ENABLE_CDF5 check_PROGRAMS += tst_open_cdf5 if LARGE_FILE_TESTS TESTPROGRAMS += tst_large_cdf5 tst_cdf5_begin - CLEANFILES += tst_large_cdf5.nc tst_cdf5_begin.nc endif endif From d959512a2e8f93b61b2c9519fa34de801d8d96a2 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 17 Oct 2018 07:29:30 -0600 Subject: [PATCH 2/4] adding tests for ELATEFILL in all available formats --- nc_test/tst_formats.c | 101 +++++++++++++++++++++++++----------------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/nc_test/tst_formats.c b/nc_test/tst_formats.c index c77cccef1..62ee6bc86 100644 --- a/nc_test/tst_formats.c +++ b/nc_test/tst_formats.c @@ -13,6 +13,10 @@ #define FILE_NAME_BASE "tst_formats" #define HDF4_FILE "ref_contiguous.hdf4" +#define NDIM1 1 +#define DIM_LEN 10 +#define VAR_NAME "Copernicus_var" +#define DIM_NAME "Copernicus_dim" /* Determine how many formats are available, and what they are. */ void @@ -99,9 +103,6 @@ main(int argc, char **argv) printf("\n*** Testing netcdf format functions.\n"); { int ncid; - int expected_mode; - int expected_extended_format; - char file_name[NC_MAX_NAME + 1]; int f; int format[MAX_NUM_FORMATS]; int num_formats; @@ -112,46 +113,66 @@ main(int argc, char **argv) for (f = 0; f < num_formats; f++) { printf("*** testing nc_inq_format() and nc_inq_format_extended() with format %d...", format[f]); - sprintf(file_name, "%s_%d.nc", FILE_NAME_BASE, format[f]); + { + char file_name[NC_MAX_NAME + 1]; + int expected_mode; + int expected_extended_format; - /* Set up test. */ - switch (format[f]) { - case NC_FORMAT_CLASSIC: - expected_extended_format = NC_FORMATX_NC3; - expected_mode = 0; - break; - case NC_FORMAT_64BIT_OFFSET: - expected_extended_format = NC_FORMATX_NC3; - expected_mode = NC_64BIT_OFFSET; - break; - case NC_FORMAT_CDF5: - expected_extended_format = NC_FORMATX_NC3; - expected_mode = NC_CDF5; - break; - case NC_FORMAT_NETCDF4: - expected_extended_format = NC_FORMATX_NC4; - expected_mode = NC_NETCDF4; - break; - case NC_FORMAT_NETCDF4_CLASSIC: - expected_extended_format = NC_FORMATX_NC4; - expected_mode = NC_NETCDF4|NC_CLASSIC_MODEL; - break; + sprintf(file_name, "%s_%d.nc", FILE_NAME_BASE, format[f]); + + /* Set up test. */ + switch (format[f]) { + case NC_FORMAT_CLASSIC: + expected_extended_format = NC_FORMATX_NC3; + expected_mode = 0; + break; + case NC_FORMAT_64BIT_OFFSET: + expected_extended_format = NC_FORMATX_NC3; + expected_mode = NC_64BIT_OFFSET; + break; + case NC_FORMAT_CDF5: + expected_extended_format = NC_FORMATX_NC3; + expected_mode = NC_CDF5; + break; + case NC_FORMAT_NETCDF4: + expected_extended_format = NC_FORMATX_NC4; + expected_mode = NC_NETCDF4; + break; + case NC_FORMAT_NETCDF4_CLASSIC: + expected_extended_format = NC_FORMATX_NC4; + expected_mode = NC_NETCDF4|NC_CLASSIC_MODEL; + break; + } + if (nc_set_default_format(format[f], NULL)) ERR; + + /* Create a file. */ + if (nc_create(file_name, 0, &ncid)) ERR; + if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR; + if (nc_close(ncid)) ERR; + + /* Re-open the file and check it again. */ + if (nc_open(file_name, 0, &ncid)) ERR; + /* Classic flag is not set on mode in nc_open(). Not sure if + * this is a bug or not. */ + if (format[f] == NC_FORMAT_NETCDF4_CLASSIC) + expected_mode = NC_NETCDF4; + if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR; + if (nc_close(ncid)) ERR; } - if (nc_set_default_format(format[f], NULL)) ERR; + SUMMARIZE_ERR; + printf("*** testing NC_ELATEFILL errors with format %d...", format[f]); + { + char file_name[NC_MAX_NAME + 1]; + int dimid, varid; - /* Create a file. */ - if (nc_create(file_name, 0, &ncid)) ERR; - if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR; - if (nc_close(ncid)) ERR; - - /* Re-open the file and check it again. */ - if (nc_open(file_name, 0, &ncid)) ERR; - /* Classic flag is not set on mode in nc_open(). Not sure if - * this is a bug or not. */ - if (format[f] == NC_FORMAT_NETCDF4_CLASSIC) - expected_mode = NC_NETCDF4; - if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR; - if (nc_close(ncid)) ERR; + sprintf(file_name, "%s_%d_elatefill.nc", FILE_NAME_BASE, format[f]); + if (nc_set_default_format(format[f], NULL)) ERR; + if (nc_create(file_name, 0, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM1, &dimid, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + if (nc_close(ncid)) ERR; + } SUMMARIZE_ERR; } /* next format */ } From 91ec0109f3c173e237566faa5f707fa375d0234c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 17 Oct 2018 09:44:18 -0600 Subject: [PATCH 3/4] more testing for late fill setting --- libdispatch/dvar.c | 84 +++++++++++++++++++++++++------------------ nc_test/tst_formats.c | 42 +++++++++++++++++++--- 2 files changed, 87 insertions(+), 39 deletions(-) diff --git a/libdispatch/dvar.c b/libdispatch/dvar.c index 2dcc7289e..77343eae4 100644 --- a/libdispatch/dvar.c +++ b/libdispatch/dvar.c @@ -546,42 +546,57 @@ NC_getshape(int ncid, int varid, int ndims, size_t* shape) return status; } -/*! Set the fill value for a variable. - -\ingroup variables - -\param ncid NetCDF ID, from a previous call to nc_open or -nc_create. - -\param varid Variable ID. - -\param no_fill Set to NC_NOFILL to turn off fill mode for this -variable. Set to NC_FILL (the default) to turn on fill mode for the -variable. - -\param fill_value the fill value to be used for this variable. Must be -the same type as the variable. This must point to enough free memory -to hold one element of the data type of the variable. (For example, an -NC_INT will require 4 bytes for it's fill value, which is also an -NC_INT.) +/** + * @ingroup variables + * + * Set the fill value for a variable. + * + * @note For netCDF classic, 64-bit offset, and CDF5 formats, it is + * allowed (but not good practice) to set the fill value after data + * have been written to the variable. In this case, unless the + * variable has been completely specified (without gaps in the data), + * any existing filled values will not be recognized as fill values by + * applications reading the data. Best practice is to set the fill + * value after the variable has been defined, but before any data have + * been written to that varibale. In NetCDF-4 files, this is enforced + * by the HDF5 library. For netCDF-4 files, an error is returned if + * the user attempts to set the fill value after writing data to the + * variable. + * @param ncid NetCDF ID, from a previous call to nc_open or + * nc_create. + * @param varid Variable ID. + * @param no_fill Set to NC_NOFILL to turn off fill mode for this + * variable. Set to NC_FILL (the default) to turn on fill mode for the + * variable. + * @param fill_value the fill value to be used for this variable. Must + * be the same type as the variable. This must point to enough free + * memory to hold one element of the data type of the variable. (For + * example, an NC_INT will require 4 bytes for it's fill value, which + * is also an NC_INT.) + * * @returns ::NC_NOERR No error. * @returns ::NC_EBADID Bad ID. - * @returns ::NC_ENOTINDEFINE Not in define mode. This is returned for -netCDF classic, 64-bit offset, or 64-bit data files, or for netCDF-4 files, -when they were created with NC_STRICT_NC3 flag. See \ref nc_create. + * @returns ::NC_ENOTINDEFINE Not in define mode. This is returned + * for netCDF classic, 64-bit offset, or 64-bit data files, or for + * netCDF-4 files, when they were created with NC_STRICT_NC3 flag. See + * @ref nc_create. * @returns ::NC_EPERM Attempt to create object in read-only file. - -\section nc_def_var_fill_example Example - -In this example from libsrc4/tst_vars.c, a variable is defined, and -the fill mode turned off. Then nc_inq_fill() is used to check that the -setting is correct. Then some data are written to the variable. Since -the data that are written do not cover the full extent of the -variable, the missing values will just be random. If fill value mode -was turned on, the missing values would get the fill value. - -\code + * @returns ::NC_ELATEDEF (NetCDF-4 only). Returned when user attempts + * to set fill value after data are written. + * @returns ::NC_EGLOBAL Attempt to set fill value on NC_GLOBAL. + * + * @section nc_def_var_fill_example Example + * + * In this example from libsrc4/tst_vars.c, a variable is defined, and + * the fill mode turned off. Then nc_inq_fill() is used to check that + * the setting is correct. Then some data are written to the + * variable. Since the data that are written do not cover the full + * extent of the variable, the missing values will just be random. If + * fill value mode was turned on, the missing values would get the + * fill value. + * + @code #define DIM7_LEN 2 #define DIM7_NAME "dim_7_from_Indiana" #define VAR7_NAME "var_7_from_Idaho" @@ -608,7 +623,8 @@ was turned on, the missing values would get the fill value. if (nc_get_var1_ushort(ncid, varid, index, &ushort_data_in)) ERR; if (nc_close(ncid)) ERR; -\endcode + @endcode + * @author Glenn Davis, Ed Hartnett, Dennis Heimbigner */ int nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) @@ -617,7 +633,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) int stat = NC_check_id(ncid,&ncp); if(stat != NC_NOERR) return stat; - /* Dennis Heimbigner: (Using NC_GLOBAL is ilegal, as this API) has no + /* Dennis Heimbigner: Using NC_GLOBAL is illegal, as this API has no * provision for specifying the type of the fillvalue, it must of necessity * be using the type of the variable to interpret the bytes of the * fill_value argument. diff --git a/nc_test/tst_formats.c b/nc_test/tst_formats.c index 62ee6bc86..d1895cfc9 100644 --- a/nc_test/tst_formats.c +++ b/nc_test/tst_formats.c @@ -17,6 +17,7 @@ #define DIM_LEN 10 #define VAR_NAME "Copernicus_var" #define DIM_NAME "Copernicus_dim" +#define NUM_FILL_WRITE_TESTS 2 /* Determine how many formats are available, and what they are. */ void @@ -103,7 +104,7 @@ main(int argc, char **argv) printf("\n*** Testing netcdf format functions.\n"); { int ncid; - int f; + int f, d; int format[MAX_NUM_FORMATS]; int num_formats; @@ -160,20 +161,51 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; - printf("*** testing NC_ELATEFILL errors with format %d...", format[f]); + for (d = 0; d < NUM_FILL_WRITE_TESTS; d++) { + printf("*** testing late fill handling with format %d writing %d...", format[f], d); char file_name[NC_MAX_NAME + 1]; int dimid, varid; + size_t index = {DIM_LEN - 1}; + int data = TEST_VAL_42; + int data_in; + int fill_value = TEST_VAL_42 * 2; - sprintf(file_name, "%s_%d_elatefill.nc", FILE_NAME_BASE, format[f]); + /* Try to set fill mode after data have been written. */ + sprintf(file_name, "%s_%d_%d_elatefill.nc", FILE_NAME_BASE, format[f], d); if (nc_set_default_format(format[f], NULL)) ERR; if (nc_create(file_name, 0, &ncid)) ERR; if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)) ERR; if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM1, &dimid, &varid)) ERR; if (nc_enddef(ncid)) ERR; + /* For netCDF-4, we don't actually have to write data to + * prevent future setting of the fill value. Once the user + * leaves calls enddef after defining a var, fill values + * can no longer be set. */ + if (d) + if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; + if (nc_redef(ncid)) ERR; + /* Setting the fill value after data are written is + * allowed in classic formats, but not netcdf-4. */ + if (format[f] == NC_FORMAT_CLASSIC || format[f] == NC_FORMAT_64BIT_OFFSET || + format[f] == NC_FORMAT_CDF5) + { + if (nc_def_var_fill(ncid, varid, NC_FILL, &fill_value)) ERR; + } + else + { + if (nc_def_var_fill(ncid, varid, NC_FILL, &fill_value) != NC_ELATEDEF) ERR; + } + if (nc_enddef(ncid)) ERR; if (nc_close(ncid)) ERR; - } - SUMMARIZE_ERR; + + /* Open the file and check data. */ + if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR; + if (nc_get_var1_int(ncid, varid, &index, &data_in)) ERR; + if (data_in != (d ? data : NC_FILL_INT)) ERR; + if (nc_close(ncid)) ERR; + SUMMARIZE_ERR; + } /* next fill val write test */ } /* next format */ } FINAL_RESULTS; From 37a36395d73b7793dd60e5bab9c01eec50eea554 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Wed, 17 Oct 2018 11:14:53 -0600 Subject: [PATCH 4/4] more tests for late fill --- nc_test/tst_formats.c | 100 +++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/nc_test/tst_formats.c b/nc_test/tst_formats.c index d1895cfc9..814904a5a 100644 --- a/nc_test/tst_formats.c +++ b/nc_test/tst_formats.c @@ -18,6 +18,7 @@ #define VAR_NAME "Copernicus_var" #define DIM_NAME "Copernicus_dim" #define NUM_FILL_WRITE_TESTS 2 +#define NUM_FILL_WRITE_METHOD_TESTS 2 /* Determine how many formats are available, and what they are. */ void @@ -104,9 +105,10 @@ main(int argc, char **argv) printf("\n*** Testing netcdf format functions.\n"); { int ncid; - int f, d; + int f, d, a; int format[MAX_NUM_FORMATS]; int num_formats; + int ret; /* How many formats to be tested? */ determine_test_formats(&num_formats, format); @@ -161,50 +163,66 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; + /* Test without and with actual data write. */ for (d = 0; d < NUM_FILL_WRITE_TESTS; d++) { - printf("*** testing late fill handling with format %d writing %d...", format[f], d); - char file_name[NC_MAX_NAME + 1]; - int dimid, varid; - size_t index = {DIM_LEN - 1}; - int data = TEST_VAL_42; - int data_in; - int fill_value = TEST_VAL_42 * 2; - - /* Try to set fill mode after data have been written. */ - sprintf(file_name, "%s_%d_%d_elatefill.nc", FILE_NAME_BASE, format[f], d); - if (nc_set_default_format(format[f], NULL)) ERR; - if (nc_create(file_name, 0, &ncid)) ERR; - if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)) ERR; - if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM1, &dimid, &varid)) ERR; - if (nc_enddef(ncid)) ERR; - /* For netCDF-4, we don't actually have to write data to - * prevent future setting of the fill value. Once the user - * leaves calls enddef after defining a var, fill values - * can no longer be set. */ - if (d) - if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; - if (nc_redef(ncid)) ERR; - /* Setting the fill value after data are written is - * allowed in classic formats, but not netcdf-4. */ - if (format[f] == NC_FORMAT_CLASSIC || format[f] == NC_FORMAT_64BIT_OFFSET || - format[f] == NC_FORMAT_CDF5) + /* Test setting _FillValue directly or calling nc_def_var_fill(). */ + for (a = 0; a < NUM_FILL_WRITE_METHOD_TESTS; a++) { - if (nc_def_var_fill(ncid, varid, NC_FILL, &fill_value)) ERR; - } - else - { - if (nc_def_var_fill(ncid, varid, NC_FILL, &fill_value) != NC_ELATEDEF) ERR; - } - if (nc_enddef(ncid)) ERR; - if (nc_close(ncid)) ERR; + printf("*** testing late fill handling with format %d writing %d " + "using def_var_fill %d...", format[f], d, a); + char file_name[NC_MAX_NAME + 1]; + int dimid, varid; + size_t index = {DIM_LEN - 1}; + int data = TEST_VAL_42; + int data_in; + int fill_value = TEST_VAL_42 * 2; - /* Open the file and check data. */ - if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR; - if (nc_get_var1_int(ncid, varid, &index, &data_in)) ERR; - if (data_in != (d ? data : NC_FILL_INT)) ERR; - if (nc_close(ncid)) ERR; - SUMMARIZE_ERR; + /* Try to set fill mode after data have been written. */ + sprintf(file_name, "%s_%d_%d_%d_elatefill.nc", FILE_NAME_BASE, format[f], d, a); + if (nc_set_default_format(format[f], NULL)) ERR; + if (nc_create(file_name, 0, &ncid)) ERR; + if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)) ERR; + if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM1, &dimid, &varid)) ERR; + if (nc_enddef(ncid)) ERR; + /* For netCDF-4, we don't actually have to write data to + * prevent future setting of the fill value. Once the user + * leaves calls enddef after defining a var, fill values + * can no longer be set. */ + if (d) + if (nc_put_var1_int(ncid, varid, &index, &data)) ERR; + if (nc_redef(ncid)) ERR; + if (a) + { + ret = nc_def_var_fill(ncid, varid, NC_FILL, &fill_value); + } + else + { + ret = nc_put_att_int(ncid, varid, "_FillValue", NC_INT, 1, + &fill_value); + } + + /* Setting the fill value after data are written is + * allowed in classic formats, but not netcdf-4. */ + if (format[f] == NC_FORMAT_CLASSIC || format[f] == NC_FORMAT_64BIT_OFFSET || + format[f] == NC_FORMAT_CDF5) + { + if (ret) ERR; + } + else + { + if (ret != (a ? NC_ELATEDEF: NC_ELATEFILL)) ERR; + } + if (nc_enddef(ncid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file and check data. */ + if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR; + if (nc_get_var1_int(ncid, varid, &index, &data_in)) ERR; + if (data_in != (d ? data : NC_FILL_INT)) ERR; + if (nc_close(ncid)) ERR; + SUMMARIZE_ERR; + } /* next fill value method test */ } /* next fill val write test */ } /* next format */ }