mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1775 from NOAA-GSD/ejh_test_szip_unlim

Browse Source 
Allow szip to be used with unlimited dim
This commit is contained in:
Ward Fisher 2020-07-06 16:40:53 -06:00 committed by GitHub
commit 933a8979c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 243 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Bug Fix] Now allow szip to be used on variables with unlimited dimension [https://github.com/Unidata/netcdf-c/issues/1774].
* [Enhancement] Add support for cloud storage using a variant of the Zarr storage format. Warning: this feature is highly experimental and is subject to rapid evolution [https://www.unidata.ucar.edu/blogs/developer/en/entry/overview-of-zarr-support-in].
* [Bug Fix] Fix nccopy to properly set default chunking parameters when not otherwise specified. This can significantly improve performance in selected cases. Note that if seeing slow performance with nccopy, then, as a work-around, specifically set the chunking parameters. [https://github.com/Unidata/netcdf-c/issues/1763].
* [Bug Fix] Fix some protocol bugs/differences between the netcdf-c library and the OPeNDAP Hyrax server. Also cleanup checksum handling [https://github.com/Unidata/netcdf-c/issues/1712].

3
libhdf5/hdf5filter.c
View File

@ -317,7 +317,8 @@ NC4_filter_actions(int ncid, int varid, int op, void* args)
size_t num_elem = 1;
int d;
for (d = 0; d < var->ndims; d++)
num_elem *= var->dim[d]->len;
if (var->dim[d]->len)
num_elem *= var->dim[d]->len;
/* Pixels per block must be <= number of elements. */
if (params[1] > num_elem)
{stat = THROW(NC_EINVAL); goto done;}

122
nc_test4/tst_parallel_compress.c
View File

@ -18,7 +18,7 @@
#include "err_macros.h"
#include <mpi.h>
#define FILE_NAME "tst_parallel_zlib2.nc"
#define FILE_NAME "tst_parallel_compress.nc"
#define NDIMS 3
#define DIMSIZE 24
#define QTR_DATA (DIMSIZE * DIMSIZE / 4)
@ -43,7 +43,7 @@ main(int argc, char **argv)
int ncid, v1id, dimids[NDIMS];
size_t start[NDIMS], count[NDIMS];
int f, i, res;
int f, i, s, res;
int *slab_data; /* one slab */
/* Initialize MPI. */
@ -62,7 +62,6 @@ main(int argc, char **argv)
if (!mpi_rank)
printf("\n*** Testing parallel writes with compression filters.\n");
{
int s;
for (f = 0; f < NUM_COMPRESSION_FILTERS; f++)
{
for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
@ -177,6 +176,123 @@ main(int argc, char **argv)
SUMMARIZE_ERR;
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
/* Now run tests with unlimited dim. */
/* for (f = 0; f < NUM_COMPRESSION_FILTERS; f++) */
for (f = 1; f < NUM_COMPRESSION_FILTERS; f++)
{
for (s = 0; s < NUM_SHUFFLE_SETTINGS; s++)
{
if (!mpi_rank)
{
printf("*** testing write along unlim dim with %s shuffle %d...",
(f ? "szip" : "zlib"), s);
}
/* nc_set_log_level(3); */
/* Create a parallel netcdf-4 file. */
if (nc_create_par(FILE_NAME, NC_NETCDF4, comm, info, &ncid)) ERR;
/* Create three dimensions. */
if (nc_def_dim(ncid, "d1", DIMSIZE, &dimids[1])) ERR;
if (nc_def_dim(ncid, "d2", DIMSIZE, &dimids[2])) ERR;
if (nc_def_dim(ncid, "d3", NC_UNLIMITED, &dimids[0])) ERR;
/* Create one var. Turn on deflation. */
if ((res = nc_def_var(ncid, "v1", NC_INT, NDIMS, dimids, &v1id))) ERR;
/* Setting any filter only will work for HDF5-1.10.3 and later
* versions. */
if (!f)
res = nc_def_var_deflate(ncid, 0, s, 1, 1);
else
{
res = nc_def_var_deflate(ncid, 0, s, 0, 0);
if (!res)
res = nc_def_var_szip(ncid, 0, 32, 32);
}
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if (res) ERR;
#else
if (res != NC_EINVAL) ERR;
#endif
/* Setting fletcher32 only will work for HDF5-1.10.3 and later
* versions. */
res = nc_def_var_fletcher32(ncid, 0, 1);
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if (res) ERR;
#else
if (res != NC_EINVAL) ERR;
#endif
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
start[1] = mpi_rank * DIMSIZE/mpi_size;
start[2] = 0;
count[1] = DIMSIZE/mpi_size;
count[2] = DIMSIZE;
count[0] = 1;
/*printf("mpi_rank=%d start[0]=%d start[1]=%d count[0]=%d count[1]=%d\n",
mpi_rank, start[0], start[1], count[0], count[1]);*/
/* Should not be allowed to change access to independent,
* because filters are in use. */
if (nc_var_par_access(ncid, v1id, NC_INDEPENDENT) != NC_EINVAL) ERR;
/* Write slabs of data. */
for (start[0] = 0; start[0] < NUM_SLABS; start[0]++)
if (nc_put_vara_int(ncid, v1id, start, count, slab_data)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
/* Check file. */
{
int shuffle_in, deflate_in, deflate_level_in;
int options_mask_in, pixels_per_block_in;
int *slab_data_in;
/* Allocate data. */
if (!(slab_data_in = malloc(sizeof(int) * DIMSIZE * DIMSIZE / mpi_size))) ERR;
/* Reopen the file for parallel access. */
if (nc_open_par(FILE_NAME, NC_NOWRITE, comm, info, &ncid)) ERR;
/* Check state of compression. */
if (!f)
{
if (nc_inq_var_deflate(ncid, 0, &shuffle_in, &deflate_in, &deflate_level_in)) ERR;
if ((s && !shuffle_in) || (!s && shuffle_in)) ERR;
if (!deflate_in || deflate_level_in != 1) ERR;
}
else
{
if (nc_inq_var_deflate(ncid, 0, &shuffle_in, NULL, NULL)) ERR;
if ((s && !shuffle_in) || (!s && shuffle_in)) ERR;
if (nc_inq_var_szip(ncid, 0, &options_mask_in, &pixels_per_block_in)) ERR;
}
/* Use parallel I/O to read the data. */
for (start[0] = 0; start[0] < NUM_SLABS; start[0]++)
{
if (nc_get_vara_int(ncid, 0, start, count, slab_data_in)) ERR;
for (i = 0; i < DIMSIZE * DIMSIZE / mpi_size; i++)
if (slab_data_in[i] != mpi_rank) ERR;
}
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
free(slab_data_in);
}
if (!mpi_rank)
SUMMARIZE_ERR;
} /* next shuffle filter test */
} /* next compression filter (zlib and szip) */
free(slab_data);
}

43
ncdump/Makefile.am
View File

@ -132,30 +132,30 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \
ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \
ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \
tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \
tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \
run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \
ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \
ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \
tst_charfill.cdl tst_charfill.sh tst_iter.sh tst_mud.sh \
ref_tst_mud4.cdl ref_tst_mud4-bc.cdl ref_tst_mud4_chars.cdl \
inttags.cdl inttags4.cdl ref_inttags.cdl ref_inttags4.cdl \
ref_tst_ncf213.cdl tst_h_scalar.sh run_utf8_nc4_tests.sh \
tst_formatx3.sh tst_formatx4.sh ref_tst_utf8_4.cdl \
ref_tst_nc4_utf8_4.cdl tst_inttags.sh tst_inttags4.sh CMakeLists.txt \
tst_bom.sh tst_inmemory_nc3.sh tst_dimsizes.sh \
tst_inmemory_nc4.sh tst_fileinfo.sh run_ncgen_tests.sh \
ref_test_360_day_1900.nc ref_test_365_day_1900.nc \
tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh \
ref_nc_test_netcdf4_4_0.nc run_back_comp_tests.sh \
ref_nc_test_netcdf4.cdl ref_tst_special_atts3.cdl tst_brecs.cdl \
ref_tst_grp_spec0.cdl ref_tst_grp_spec.cdl tst_grp_spec.sh \
ref_tst_charfill.cdl tst_charfill.cdl tst_charfill.sh tst_iter.sh \
tst_mud.sh ref_tst_mud4.cdl ref_tst_mud4-bc.cdl \
ref_tst_mud4_chars.cdl inttags.cdl inttags4.cdl ref_inttags.cdl \
ref_inttags4.cdl ref_tst_ncf213.cdl tst_h_scalar.sh \
run_utf8_nc4_tests.sh tst_formatx3.sh tst_formatx4.sh \
ref_tst_utf8_4.cdl ref_tst_nc4_utf8_4.cdl tst_inttags.sh \
tst_inttags4.sh CMakeLists.txt tst_bom.sh tst_inmemory_nc3.sh \
tst_dimsizes.sh tst_inmemory_nc4.sh tst_fileinfo.sh \
run_ncgen_tests.sh ref_test_360_day_1900.nc ref_test_365_day_1900.nc \
ref_test_366_day_1900.nc ref_test_360_day_1900.cdl \
ref_test_365_day_1900.cdl ref_test_366_day_1900.cdl \
tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \
ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \
tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \
tst_ncgen4_cycle.sh tst_null_byte_padding.sh \
ref_null_byte_padding_test.nc ref_tst_irish_rover.nc ref_provenance_v1.nc \
ref_tst_radix.cdl tst_radix.cdl test_radix.sh \
ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh ref_no_ncproperty.nc \
test_unicode_directory.sh
ref_null_byte_padding_test.nc ref_tst_irish_rover.nc \
ref_provenance_v1.nc ref_tst_radix.cdl tst_radix.cdl test_radix.sh \
ref_nccopy_w.cdl tst_nccopy_w3.sh tst_nccopy_w4.sh \
ref_no_ncproperty.nc test_unicode_directory.sh \
ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl
# The L512.bin file is file containing exactly 512 bytes each of value 0.
# It is used for creating hdf5 files with varying offsets for testing.
@ -188,6 +188,7 @@ compound_datasize_test.nc compound_datasize_test2.nc ncf199.nc \
tst_c0.cdl tst_c0_4.cdl tst_c0_4c.cdl tst_c0_64.cdl \
tst_compound_datasize_test.cdl tst_compound_datasize_test2.cdl \
tst_ncf199.cdl tst_tst_gattenum.cdl tst_tst_usuffix.cdl ctest.c \
ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc \
tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c \
ctest_special_atts_4.c
ctest64.c nccopy3_subset_out.nc camrun.c tst_ncf213.cdl tst_ncf213.nc \
tst_radix.nc tmp_radix.cdl ctest_small_3.c ctest_small_4.c \
ctest_special_atts_4.c tst_roman_szip_simple.cdl \
tst_roman_szip_unlim.cdl

14
ncdump/ref_roman_szip_simple.cdl Normal file
View File

@ -0,0 +1,14 @@
netcdf tst_roman_szip_simple {
dimensions:
Centuria = 100 ;
variables:
int Legio_tertia_Gallica(Centuria) ;
data:
Legio_tertia_Gallica = 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ;
}

21
ncdump/ref_roman_szip_unlim.cdl Normal file
View File

@ -0,0 +1,21 @@
netcdf tst_roman_szip_unlim {
dimensions:
Primi_ordinis = UNLIMITED ; // (1 currently)
Centuria = 10 ;
heredia = 10 ;
variables:
int Legio_tertia_Gallica(Primi_ordinis, Centuria, heredia) ;
data:
Legio_tertia_Gallica =
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
40, 41, 42, 43, 44, 45, 46, 47, 48, 49,
50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
60, 61, 62, 63, 64, 65, 66, 67, 68, 69,
70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ;
}

57
ncdump/tst_create_files.c
View File

@ -5,7 +5,7 @@
This program creates some test files which ncdump will read. This
is only done if netCDF-4 is enabled.
$Id: tst_create_files.c,v 1.16 2008/10/20 01:48:08 ed Exp $
Ed Hartnett 2008/10/20
*/
#include <nc_tests.h>
@ -263,5 +263,60 @@ main(int argc, char **argv)
SUMMARIZE_ERR;
/* These files only get created for builds in which szip support is
* present in HDF5. */
#ifdef HAVE_H5Z_SZIP
#define SZIP_DIM_NAME "Centuria"
#define SZIP_DIM_LEN 100
#define SZIP_VAR_NAME "Legio_tertia_Gallica"
#define FILE_NAME_SZIP_SIMPLE "tst_roman_szip_simple.nc"
printf("*** creating simple file with szip compression %s...", FILE_NAME_SZIP_SIMPLE);
{
int ncid, dimid, varid;
int data[SZIP_DIM_LEN];
int i;
for (i = 0; i < SZIP_DIM_LEN; i++)
data[i] = i;
/* Create a file with szip compression. */
if (nc_create(FILE_NAME_SZIP_SIMPLE, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR;
if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 1, &dimid, &varid)) ERR;
if (nc_def_var_szip(ncid, varid, 32, 32)) ERR;
if (nc_put_var(ncid, varid, data)) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#define NDIM3 3
#define SZIP_OTHER_DIM_NAME "heredia"
#define SZIP_UNLIM_DIM_NAME "Primi_ordinis"
#define FILE_NAME_SZIP_UNLIM "tst_roman_szip_unlim.nc"
#define SZIP_DIM_LEN_10 10
printf("*** creating file with szip compression and unlim dim %s...", FILE_NAME_SZIP_UNLIM);
{
int ncid, dimid[NDIM3], varid;
int data[SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10];
size_t start[NDIM3] = {0, 0, 0};
size_t count[NDIM3] = {1, SZIP_DIM_LEN_10, SZIP_DIM_LEN_10};
int i;
for (i = 0; i < SZIP_DIM_LEN_10 * SZIP_DIM_LEN_10; i++)
data[i] = i;
/* Create a file with szip compression. */
if (nc_create(FILE_NAME_SZIP_UNLIM, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, SZIP_UNLIM_DIM_NAME, NC_UNLIMITED, &dimid[0])) ERR;
if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN_10, &dimid[1])) ERR;
if (nc_def_dim(ncid, SZIP_OTHER_DIM_NAME, SZIP_DIM_LEN_10, &dimid[2])) ERR;
if (nc_def_var(ncid, SZIP_VAR_NAME, NC_INT, 3, dimid, &varid)) ERR;
if (nc_def_var_szip(ncid, varid, 32, 32)) ERR;
if (nc_put_vara_int(ncid, varid, start, count, data)) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#endif /* HAVE_H5Z_SZIP */
FINAL_RESULTS;
}

8
ncdump/tst_netcdf4.sh
View File

@ -49,6 +49,14 @@ diff -b tst_solar_1.cdl $srcdir/ref_tst_solar_1.cdl ; ERR
${NCDUMP} tst_solar_2.nc | sed 's/e+0/e+/g' > tst_solar_2.cdl ; ERR
diff -b tst_solar_2.cdl $srcdir/ref_tst_solar_2.cdl ; ERR
if test -f tst_roman_szip_simple.nc; then
echo "*** Testing szip compression."
${NCDUMP} tst_roman_szip_simple.nc | sed 's/e+0/e+/g' > tst_roman_szip_simple.cdl ; ERR
diff -b tst_roman_szip_simple.cdl $srcdir/ref_roman_szip_simple.cdl ; ERR
${NCDUMP} tst_roman_szip_unlim.nc | sed 's/e+0/e+/g' > tst_roman_szip_unlim.cdl ; ERR
diff -b tst_roman_szip_unlim.cdl $srcdir/ref_roman_szip_unlim.cdl ; ERR
fi
echo "*** Running tst_group_data.c to create test files."
${execdir}/tst_group_data ; ERR
${NCDUMP} tst_group_data.nc | sed 's/e+0/e+/g' > tst_group_data.cdl ; ERR