mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-27 07:30:33 +08:00
aeb3ac2809
re: https://github.com/Unidata/netcdf-c/issues/1836 Revert the internal filter code to simplify it. From the user's point of view, the only visible changes should be: 1. The functions that convert text to filter specs have had their signature reverted and have been moved to netcdf_aux.h 2. Some filter API functions now return NC_ENOFILTER when inquiry is made about some filter. Internally,the dispatch table has been modified to get rid of the filter_actions entry and associated complex structures. It has been replaced with inq_var_filter_ids and inq_var_filter_info entries and the dispatch table version has been bumped to 3. Corresponding NOOP and NOTNC4 functions were added to libdispatch/dnotnc4.c. Also, the filter_action entries in dispatch tables were replaced for all dispatch code bases (HDF5, DAP2, etc). This should only impact UDF users. In the process, it became clear that the form of the filters field in NC_VAR_INFO_T was format dependent, so I converted it to be of type void* and pushed its management into the various dispatch code bases. Specifically libhdf5 and libnczarr now manage the filters field in their own way. The auxilliary functions for parsing textual filter specifications were moved to netcdf_aux.h and were renamed to the following: * ncaux_h5filterspec_parse * ncaux_h5filterspec_parselist * ncaux_h5filterspec_free * ncaux_h5filter_fix8 Misc. Other Changes: 1. Document NUG/filters.md updated to reflect the changes above. 2. All the old data types (structs and enums) used by filter_actions actions were deleted. The exception is the NC_H5_Filterspec because it is needed by ncaux_h5filterspec_parselist. 3. Clientside filters were removed -- another enhancement for which no-one ever asked. 4. The ability to remove filters was itself removed. 5. Some functionality needed by nczarr was moved from libhdf5 to libsrc4 e.g. nc4_find_default_chunksizes 6. All the filterx code was removed 7. ncfilter.h and nc4filter.c no longer used Misc. Unrelated Changes: 1. The nczarr_test makefile clean was leaving some directories; so add clean-local to take care of them.
358 lines
14 KiB
C
358 lines
14 KiB
C
/* This is part of the netCDF package. Copyright 2005-2018 University
|
|
Corporation for Atmospheric Research/Unidata. See COPYRIGHT file
|
|
for conditions of use.
|
|
|
|
Test handling of formats.
|
|
|
|
Ed Hartnett, 11/22/17
|
|
*/
|
|
|
|
#include "config.h"
|
|
#include <nc_tests.h>
|
|
#include "err_macros.h"
|
|
|
|
#define FILE_NAME_BASE "tst_formats"
|
|
#define HDF4_FILE "ref_contiguous.hdf4"
|
|
#define NDIM1 1
|
|
#define DIM_LEN 10
|
|
#define VAR_NAME "Copernicus_var"
|
|
#define DIM_NAME "Copernicus_dim"
|
|
#define NUM_FILL_WRITE_TESTS 2
|
|
#define NUM_FILL_WRITE_METHOD_TESTS 2
|
|
|
|
/* Determine how many formats are available, and what they are. */
|
|
void
|
|
determine_test_formats(int *num_formats, int *format)
|
|
{
|
|
int ind = 0;
|
|
int num;
|
|
|
|
/* Check inputs. */
|
|
assert(num_formats && format);
|
|
|
|
/* We always have classic and 64-bit offset */
|
|
num = 2;
|
|
format[ind++] = NC_FORMAT_CLASSIC;
|
|
format[ind++] = NC_FORMAT_64BIT_OFFSET;
|
|
|
|
/* Do we have netCDF-4 and netCDF-4 classic? */
|
|
#ifdef USE_HDF5
|
|
num += 2;
|
|
format[ind++] = NC_FORMAT_NETCDF4;
|
|
format[ind++] = NC_FORMAT_NETCDF4_CLASSIC;
|
|
#endif /* USE_HDF5 */
|
|
|
|
/* Do we have CDF5? */
|
|
#ifdef ENABLE_CDF5
|
|
num++;
|
|
format[ind++] = NC_FORMAT_CDF5;
|
|
#endif /* ENABLE_CDF5 */
|
|
|
|
*num_formats = num;
|
|
}
|
|
|
|
/* Function to test nc_inq_format(). */
|
|
int
|
|
check_inq_format(int ncid, int expected_format, int expected_extended_format, int expected_mode)
|
|
{
|
|
int format;
|
|
int extended_format;
|
|
int mode;
|
|
|
|
if (nc_inq_format(ncid + 66000, NULL) != NC_EBADID) ERR;
|
|
if (nc_inq_format(ncid, NULL)) ERR;
|
|
if (nc_inq_format(ncid, &format)) ERR;
|
|
if (format != expected_format) {
|
|
printf("format %d expected_format %d\n", format, expected_format);
|
|
ERR;
|
|
}
|
|
if (nc_inq_format_extended(ncid + 66000, &extended_format, &mode) != NC_EBADID) ERR;
|
|
{
|
|
int mode;
|
|
if (nc_inq_format_extended(ncid, NULL, &mode)) ERR;
|
|
if (mode != expected_mode) {
|
|
printf("expected_mode %x mode %x\n", expected_mode, mode);
|
|
/*ERR;*/
|
|
}
|
|
}
|
|
{
|
|
int extended_format;
|
|
if (nc_inq_format_extended(ncid, &extended_format, NULL)) ERR;
|
|
if (extended_format != expected_extended_format) ERR;
|
|
}
|
|
|
|
if (nc_inq_format_extended(ncid, &extended_format, &mode)) ERR;
|
|
if (mode != expected_mode) ERR;
|
|
if (extended_format != expected_extended_format) ERR;
|
|
|
|
/* Nothing to do with inq_format, but let's check the base_pe
|
|
* functions. */
|
|
if (nc_set_base_pe(ncid, 0)) ERR;
|
|
if (nc_inq_base_pe(ncid, NULL)) ERR;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
main(int argc, char **argv)
|
|
{
|
|
printf("\n*** Testing netcdf format functions.\n");
|
|
{
|
|
int ncid;
|
|
int f, d, a;
|
|
int format[MAX_NUM_FORMATS];
|
|
int num_formats;
|
|
int ret;
|
|
|
|
/* How many formats to be tested? */
|
|
determine_test_formats(&num_formats, format);
|
|
|
|
for (f = 0; f < num_formats; f++)
|
|
{
|
|
printf("*** testing nc_inq_format() and nc_inq_format_extended() with format %d...", format[f]);
|
|
{
|
|
char file_name[NC_MAX_NAME + 1];
|
|
int expected_mode;
|
|
int expected_extended_format;
|
|
|
|
sprintf(file_name, "%s_%d.nc", FILE_NAME_BASE, format[f]);
|
|
|
|
/* Set up test. */
|
|
switch (format[f]) {
|
|
case NC_FORMAT_CLASSIC:
|
|
expected_extended_format = NC_FORMATX_NC3;
|
|
expected_mode = 0;
|
|
break;
|
|
case NC_FORMAT_64BIT_OFFSET:
|
|
expected_extended_format = NC_FORMATX_NC3;
|
|
expected_mode = NC_64BIT_OFFSET;
|
|
break;
|
|
case NC_FORMAT_CDF5:
|
|
expected_extended_format = NC_FORMATX_NC3;
|
|
expected_mode = NC_CDF5;
|
|
break;
|
|
case NC_FORMAT_NETCDF4:
|
|
expected_extended_format = NC_FORMATX_NC4;
|
|
expected_mode = NC_NETCDF4;
|
|
break;
|
|
case NC_FORMAT_NETCDF4_CLASSIC:
|
|
expected_extended_format = NC_FORMATX_NC4;
|
|
expected_mode = NC_NETCDF4|NC_CLASSIC_MODEL;
|
|
break;
|
|
}
|
|
if (nc_set_default_format(format[f], NULL)) ERR;
|
|
|
|
/* Create a file. */
|
|
if (nc_create(file_name, 0, &ncid)) ERR;
|
|
if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
|
|
/* Re-open the file and check it again. */
|
|
if (nc_open(file_name, 0, &ncid)) ERR;
|
|
/* Classic flag is not set on mode in nc_open(). Not sure if
|
|
* this is a bug or not. */
|
|
if (format[f] == NC_FORMAT_NETCDF4_CLASSIC)
|
|
expected_mode = NC_NETCDF4;
|
|
if (check_inq_format(ncid, format[f], expected_extended_format, expected_mode)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
}
|
|
SUMMARIZE_ERR;
|
|
/* Test without and with actual data write. */
|
|
for (d = 0; d < NUM_FILL_WRITE_TESTS; d++)
|
|
{
|
|
/* Test setting _FillValue directly or calling nc_def_var_fill(). */
|
|
for (a = 0; a < NUM_FILL_WRITE_METHOD_TESTS; a++)
|
|
{
|
|
printf("*** testing late fill handling with format %d writing %d "
|
|
"using def_var_fill %d...", format[f], d, a);
|
|
char file_name[NC_MAX_NAME + 1];
|
|
int dimid, varid;
|
|
size_t index = {DIM_LEN - 1};
|
|
int data = TEST_VAL_42;
|
|
int data_in;
|
|
int fill_value = TEST_VAL_42 * 2;
|
|
int shuffle_in, deflate_in, deflate_level_in;
|
|
int options_mask_in, pixels_per_block_in;
|
|
int storage_in;
|
|
|
|
unsigned int filterid;
|
|
size_t nfilters;
|
|
|
|
/* Try to set fill mode after data have been written. */
|
|
sprintf(file_name, "%s_%d_%d_%d_elatefill.nc", FILE_NAME_BASE, format[f], d, a);
|
|
if (nc_set_default_format(format[f], NULL)) ERR;
|
|
if (nc_create(file_name, 0, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)) ERR;
|
|
if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIM1, &dimid, &varid)) ERR;
|
|
|
|
/* There is no deflate on this var, and that is true in
|
|
* all formats. */
|
|
if (nc_inq_var_deflate(ncid, varid, &shuffle_in, &deflate_in,
|
|
&deflate_level_in)) ERR;
|
|
if (shuffle_in || deflate_in || deflate_level_in) ERR;
|
|
|
|
/* There is no szip on this var, and that is true in
|
|
* all formats. */
|
|
if (nc_inq_var_szip(ncid, varid, &options_mask_in, &pixels_per_block_in)) ERR;
|
|
if (options_mask_in || pixels_per_block_in) ERR;
|
|
|
|
/* Since chunking is unset for netCDF-4 files, and
|
|
* unavailable for classic formats, this will tell us
|
|
* the var is contiguous. */
|
|
if (nc_inq_var_chunking(ncid, varid, &storage_in, NULL)) ERR;
|
|
if (storage_in != NC_CONTIGUOUS) ERR;
|
|
|
|
/* Since there are no filters defined, all of these
|
|
should succeed */
|
|
if (nc_inq_var_filter(ncid, varid, &filterid, NULL, NULL)) ERR;
|
|
filterid = H5Z_FILTER_DEFLATE;
|
|
if (nc_inq_var_filter_info(ncid, varid, filterid, NULL, NULL) != NC_ENOFILTER) ERR;
|
|
nfilters = 0;
|
|
if (nc_inq_var_filter_ids(ncid, varid, &nfilters, NULL)) ERR;
|
|
if(nfilters != 0) ERR;
|
|
|
|
if (nc_enddef(ncid)) ERR;
|
|
/* For netCDF-4, we don't actually have to write data to
|
|
* prevent future setting of the fill value. Once the user
|
|
* leaves calls enddef after defining a var, fill values
|
|
* can no longer be set. */
|
|
if (d)
|
|
if (nc_put_var1_int(ncid, varid, &index, &data)) ERR;
|
|
if (nc_redef(ncid)) ERR;
|
|
if (a)
|
|
{
|
|
ret = nc_def_var_fill(ncid, varid, NC_FILL, &fill_value);
|
|
}
|
|
else
|
|
{
|
|
ret = nc_put_att_int(ncid, varid, "_FillValue", NC_INT, 1,
|
|
&fill_value);
|
|
}
|
|
|
|
/* Setting the fill value after data are written is
|
|
* allowed in classic formats, but not netcdf-4. */
|
|
if (format[f] == NC_FORMAT_CLASSIC || format[f] == NC_FORMAT_64BIT_OFFSET ||
|
|
format[f] == NC_FORMAT_CDF5)
|
|
{
|
|
if (ret) ERR;
|
|
}
|
|
else
|
|
{
|
|
if (ret != (a ? NC_ELATEDEF: NC_ELATEFILL)) ERR;
|
|
}
|
|
if (nc_enddef(ncid)) ERR;
|
|
|
|
/* There is (still!) no deflate on this var, and that
|
|
* is true in all formats. */
|
|
if (nc_inq_var_deflate(ncid, varid, &shuffle_in, &deflate_in,
|
|
&deflate_level_in)) ERR;
|
|
if (shuffle_in || deflate_in || deflate_level_in) ERR;
|
|
|
|
/* There is (still!) no szip on this var, and that is
|
|
* true in all formats. */
|
|
if (nc_inq_var_szip(ncid, varid, &options_mask_in, &pixels_per_block_in)) ERR;
|
|
if (options_mask_in || pixels_per_block_in) ERR;
|
|
|
|
/* Storage is (still) contiguous. */
|
|
if (nc_inq_var_chunking(ncid, varid, &storage_in, NULL)) ERR;
|
|
if (storage_in != NC_CONTIGUOUS) ERR;
|
|
|
|
if (nc_close(ncid)) ERR;
|
|
|
|
/* Open the file and check data. */
|
|
if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR;
|
|
if (nc_get_var1_int(ncid, varid, &index, &data_in)) ERR;
|
|
if (data_in != (d ? data : NC_FILL_INT)) ERR;
|
|
if (nc_close(ncid)) ERR;
|
|
SUMMARIZE_ERR;
|
|
} /* next fill value method test */
|
|
} /* next fill val write test */
|
|
|
|
#define NDIM2 2
|
|
#define DIM1_NAME "dim1"
|
|
#define DIM2_NAME "dim2"
|
|
#define NTYPE 6
|
|
#define DATA_LEN 4
|
|
|
|
printf("*** testing handling of null strides with format %d... ",
|
|
format[f]);
|
|
{
|
|
char file_name[NC_MAX_NAME + 1];
|
|
char var_name[NC_MAX_NAME + 1];
|
|
int dimid[NDIM2];
|
|
int xtype[NTYPE] = {NC_BYTE, NC_CHAR, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE};
|
|
int type_size[NTYPE] = {1, 1, 2, 4, 4, 8};
|
|
int varid[NTYPE];
|
|
size_t start[NDIM2] = {0, 0};
|
|
size_t count[NDIM2] = {2, 2};
|
|
signed char data_byte[DATA_LEN] = {1, 2, 3, 4};
|
|
unsigned char data_char[DATA_LEN] = {1, 2, 3, 4};
|
|
short data_short[DATA_LEN] = {1, 2, 3, 4};
|
|
int data_int[DATA_LEN] = {1, 2, 3, 4};
|
|
float data_float[DATA_LEN] = {1, 2, 3, 4};
|
|
double data_double[DATA_LEN] = {1, 2, 3, 4};
|
|
void *data_ptr[NTYPE] = {data_byte, data_char, data_short, data_int, data_float, data_double};
|
|
int t;
|
|
|
|
/* Create the test file. */
|
|
sprintf(file_name, "%s_%d_null_strides.nc", FILE_NAME_BASE, format[f]);
|
|
if (nc_set_default_format(format[f], NULL)) ERR;
|
|
if (nc_create(file_name, 0, &ncid)) ERR;
|
|
if (nc_def_dim(ncid, DIM1_NAME, DIM_LEN, &dimid[0])) ERR;
|
|
if (nc_def_dim(ncid, DIM2_NAME, DIM_LEN, &dimid[1])) ERR;
|
|
for (t = 0; t < NTYPE; t++)
|
|
{
|
|
sprintf(var_name, "var_%d", xtype[t]);
|
|
if (nc_def_var(ncid, var_name, xtype[t], NDIM2, dimid, &varid[t])) ERR;
|
|
}
|
|
if (nc_enddef(ncid)) ERR;
|
|
|
|
/* Write some data. */
|
|
for (t = 0; t < NTYPE; t++)
|
|
{
|
|
if (nc_put_vars(ncid, varid[t], start, count, NULL, data_ptr[t])) ERR;
|
|
}
|
|
if (nc_close(ncid)) ERR;
|
|
|
|
/* Open the file and check data. */
|
|
{
|
|
int ndims, nvars, ngatts, unlimdimid;
|
|
|
|
if (nc_open(file_name, NC_NOWRITE, &ncid)) ERR;
|
|
if (nc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)) ERR;
|
|
if (ndims != 2 || nvars != NTYPE || ngatts != 0 || unlimdimid != -1) ERR;
|
|
for (t = 0; t < NTYPE; t++)
|
|
{
|
|
nc_type my_type;
|
|
int var_ndims, natts;
|
|
int var_dimid[NDIM2];
|
|
void *data_in;
|
|
|
|
if (nc_inq_var(ncid, varid[t], NULL, &my_type, &var_ndims, var_dimid, &natts)) ERR;
|
|
if (my_type != xtype[t] || var_ndims != 2 || var_dimid[0] != dimid[0] ||
|
|
var_dimid[1] != dimid[1] || natts != 0) ERR;
|
|
if (!(data_in = malloc(DATA_LEN * type_size[t]))) ERR;
|
|
if (nc_get_vars(ncid, varid[t], start, count, NULL, data_in)) ERR;
|
|
if (memcmp(data_in, data_ptr[t], DATA_LEN * type_size[t])) ERR;
|
|
free(data_in);
|
|
|
|
}
|
|
if (nc_close(ncid)) ERR;
|
|
}
|
|
|
|
}
|
|
SUMMARIZE_ERR;
|
|
printf("*** testing bad name for nc_open/nc_create with format %d... ", format[f]);
|
|
{
|
|
int ncid;
|
|
if (nc_set_default_format(format[f], NULL)) ERR;
|
|
if (nc_create(NULL, 0, &ncid) != NC_EINVAL) ERR;
|
|
if (nc_open(NULL, NC_NOWRITE, &ncid) != NC_EINVAL) ERR;
|
|
if (nc_delete(NULL) != NC_EINVAL) ERR;
|
|
}
|
|
SUMMARIZE_ERR;
|
|
} /* next format */
|
|
}
|
|
FINAL_RESULTS;
|
|
}
|