From 4103faf0d572fa0aefc152742410db183a2607ad Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Thu, 2 Jul 2020 13:59:37 -0600 Subject: [PATCH] added some ncdump tests for szip when it is present --- libhdf5/hdf5filter.c | 3 +- ncdump/Makefile.am | 36 ++++++++++---------- ncdump/ref_roman_szip_simple.cdl | 14 ++++++++ ncdump/ref_roman_szip_unlim.cdl | 21 ++++++++++++ ncdump/tst_create_files.c | 57 +++++++++++++++++++++++++++++++- ncdump/tst_netcdf4.sh | 8 +++++ 6 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 ncdump/ref_roman_szip_simple.cdl create mode 100644 ncdump/ref_roman_szip_unlim.cdl diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c index 2c5c7fda1..c118bd23f 100644 --- a/libhdf5/hdf5filter.c +++ b/libhdf5/hdf5filter.c @@ -317,7 +317,8 @@ NC4_filter_actions(int ncid, int varid, int op, void* args) size_t num_elem = 1; int d; for (d = 0; d < var->ndims; d++) - num_elem *= var->dim[d]->len; + if (var->dim[d]->len) + num_elem *= var->dim[d]->len; /* Pixels per block must be <= number of elements. */ if (params[1] > num_elem) {stat = THROW(NC_EINVAL); goto done;} diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 9da3bf043..3a21192d1 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -132,30 +132,30 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \ ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \ ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \ tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \ -tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \ -run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \ -ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \ -ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \ -tst_charfill.cdl tst_charfill.sh tst_iter.sh tst_mud.sh \ -ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl \ -inttags.cdl inttags4.cdl ref_inttags.cdl ref_inttags4.cdl \ -ref_tst_ncf213.cdl tst_h_scalar.sh run_utf8_nc4_tests.sh \ -tst_formatx3.sh tst_formatx4.sh ref_tst_utf8_4.cdl \ -ref_tst_nc4_utf8_4.cdl tst_inttags.sh tst_inttags4.sh CMakeLists.txt \ -tst_bom.sh tst_inmemory_nc3.sh tst_dimsizes.sh \ -tst_inmemory_nc4.sh tst_fileinfo.sh run_ncgen_tests.sh \ -ref_test_360_day_1900.nc ref_test_365_day_1900.nc \ +tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh \ +ref_nc_test_netcdf4_4_0.nc run_back_comp_tests.sh \ +ref_nc_test_netcdf4.cdl ref_tst_special_atts3.cdl tst_brecs.cdl \ +ref_tst_grp_spec0.cdl ref_tst_grp_spec.cdl tst_grp_spec.sh \ +ref_tst_charfill.cdl tst_charfill.cdl tst_charfill.sh tst_iter.sh \ +tst_mud.sh ref_tst_mud4.cdl ref_tst_mud4-bc.cdl \ +ref_tst_mud4_chars.cdl inttags.cdl inttags4.cdl ref_inttags.cdl \ +ref_inttags4.cdl ref_tst_ncf213.cdl tst_h_scalar.sh \ +run_utf8_nc4_tests.sh tst_formatx3.sh tst_formatx4.sh \ +ref_tst_utf8_4.cdl ref_tst_nc4_utf8_4.cdl tst_inttags.sh \ +tst_inttags4.sh CMakeLists.txt tst_bom.sh tst_inmemory_nc3.sh \ +tst_dimsizes.sh tst_inmemory_nc4.sh tst_fileinfo.sh \ +run_ncgen_tests.sh ref_test_360_day_1900.nc ref_test_365_day_1900.nc \ ref_test_366_day_1900.nc ref_test_360_day_1900.cdl \ ref_test_365_day_1900.cdl ref_test_366_day_1900.cdl \ tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \ ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \ tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \ tst_ncgen4_cycle.sh tst_null_byte_padding.sh \ -ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc \ -ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ -ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh ref_no_ncproperty.nc \ -test_unicode_directory.sh - +ref_null_byte_padding_test.nc ref_tst_irish_rover.nc \ +ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \ +ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \ +ref_no_ncproperty.nc test_unicode_directory.sh \ +ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. diff --git a/ncdump/ref_roman_szip_simple.cdl b/ncdump/ref_roman_szip_simple.cdl new file mode 100644 index 000000000..07bf9416c --- /dev/null +++ b/ncdump/ref_roman_szip_simple.cdl @@ -0,0 +1,14 @@ +netcdf tst_roman_szip_simple { +dimensions: + Centuria = 100 ; +variables: + int Legio_tertia_Gallica(Centuria) ; +data: + + Legio_tertia_Gallica = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, + 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ; +} diff --git a/ncdump/ref_roman_szip_unlim.cdl b/ncdump/ref_roman_szip_unlim.cdl new file mode 100644 index 000000000..9c019d49c --- /dev/null +++ b/ncdump/ref_roman_szip_unlim.cdl @@ -0,0 +1,21 @@ +netcdf tst_roman_szip_unlim { +dimensions: + Primi_ordinis = UNLIMITED ; // (1 currently) + Centuria = 10 ; + heredia = 10 ; +variables: + int Legio_tertia_Gallica(Primi_ordinis, Centuria, heredia) ; +data: + + Legio_tertia_Gallica = + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ; +} diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c index be9d55662..c8122042a 100644 --- a/ncdump/tst_create_files.c +++ b/ncdump/tst_create_files.c @@ -5,7 +5,7 @@ This program creates some test files which ncdump will read. This is only done if netCDF-4 is enabled. - $Id: tst_create_files.c,v 1.16 2008/10/20 01:48:08 ed Exp $ + Ed Hartnett 2008/10/20 */ #include @@ -263,5 +263,60 @@ main(int argc, char **argv) SUMMARIZE_ERR; + /* These files only get created for builds in which szip support is + * present in HDF5. */ +#ifdef HAVE_H5Z_SZIP +#define SZIP_DIM_NAME "Centuria" +#define SZIP_DIM_LEN 100 +#define SZIP_VAR_NAME "Legio_tertia_Gallica" +#define FILE_NAME_SZIP_SIMPLE "tst_roman_szip_simple.nc" + printf("*** creating simple file with szip compression %s...", FILE_NAME_SZIP_SIMPLE); + { + int ncid, dimid, varid; + int data[SZIP_DIM_LEN]; + int i; + + for (i = 0; i < SZIP_DIM_LEN; i++) + data[i] = i; + + /* Create a file with szip compression. */ + if (nc_create(FILE_NAME_SZIP_SIMPLE, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR; + if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 1, &dimid, &varid)) ERR; + if (nc_def_var_szip(ncid, varid, 32, 32)) ERR; + if (nc_put_var(ncid, varid, data)) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; +#define NDIM3 3 +#define SZIP_OTHER_DIM_NAME "heredia" +#define SZIP_UNLIM_DIM_NAME "Primi_ordinis" +#define FILE_NAME_SZIP_UNLIM "tst_roman_szip_unlim.nc" +#define SZIP_DIM_LEN_10 10 + printf("*** creating file with szip compression and unlim dim %s...", FILE_NAME_SZIP_UNLIM); + { + int ncid, dimid[NDIM3], varid; + int data[SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10]; + size_t start[NDIM3] = {0, 0, 0}; + size_t count[NDIM3] = {1, SZIP_DIM_LEN_10, SZIP_DIM_LEN_10}; + int i; + + for (i = 0; i < SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10; i++) + data[i] = i; + + /* Create a file with szip compression. */ + if (nc_create(FILE_NAME_SZIP_UNLIM, NC_NETCDF4, &ncid)) ERR; + if (nc_def_dim(ncid, SZIP_UNLIM_DIM_NAME, NC_UNLIMITED, &dimid[0])) ERR; + if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN_10, &dimid[1])) ERR; + if (nc_def_dim(ncid, SZIP_OTHER_DIM_NAME, SZIP_DIM_LEN_10, &dimid[2])) ERR; + if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 3, dimid, &varid)) ERR; + if (nc_def_var_szip(ncid, varid, 32, 32)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, data)) ERR; + + if (nc_close(ncid)) ERR; + } + SUMMARIZE_ERR; +#endif /* HAVE_H5Z_SZIP */ FINAL_RESULTS; } diff --git a/ncdump/tst_netcdf4.sh b/ncdump/tst_netcdf4.sh index 5f7b98c4d..d3737e359 100755 --- a/ncdump/tst_netcdf4.sh +++ b/ncdump/tst_netcdf4.sh @@ -49,6 +49,14 @@ diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR ${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR +if test -f tst_roman_szip_simple.nc; then + echo "*** Testing szip compression." + ${NCDUMP} tst_roman_szip_simple.nc | sed 's/e+0/e+/g' > tst_roman_szip_simple.cdl ; ERR + diff -b tst_roman_szip_simple.cdl $srcdir/ref_roman_szip_simple.cdl ; ERR + ${NCDUMP} tst_roman_szip_unlim.nc | sed 's/e+0/e+/g' > tst_roman_szip_unlim.cdl ; ERR + diff -b tst_roman_szip_unlim.cdl $srcdir/ref_roman_szip_unlim.cdl ; ERR +fi + echo "*** Running tst_group_data.c to create test files." ${execdir}/tst_group_data ; ERR ${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR