Merge pull request #1925 from DennisHeimbigner/filterorder.dmh

Add tests for filter order on read and write cases
This commit is contained in:
Ward Fisher 2021-01-04 16:11:53 -07:00 committed by GitHub
commit f474e6d692
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 160 additions and 113 deletions

View File

@ -852,6 +852,13 @@ FIND_PACKAGE(CURL)
ADD_DEFINITIONS(-DCURL_STATICLIB=1)
INCLUDE_DIRECTORIES(${CURL_INCLUDE_DIRS})
# Define a test flag for have curl library
IF(CURL_LIBRARIES OR CURL_LIBRARY)
SET(FOUND_CURL TRUE)
ELSE()
SET(FOUND_CURL FALSE)
ENDIF()
# Check to see if CURLOPT_USERNAME is defined.
# It is present starting version 7.19.1.
CHECK_C_SOURCE_COMPILES("
@ -919,16 +926,6 @@ ENDIF()
# Option to support byte-range reading of remote datasets
OPTION(ENABLE_BYTERANGE "Enable byte-range access to remote datasets.." OFF)
IF(NOT CURL_LIBRARIES AND NOT CURL_LIBRARY)
IF(ENABLE_BYTERANGE)
MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.")
ENDIF()
IF(ENABLE_DAP2 OR ENABLE_DAP4)
MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.")
ENDIF()
ENDIF()
# Check for the math library so it can be explicitly linked.
IF(NOT WIN32)
FIND_LIBRARY(HAVE_LIBM NAMES math m libm)
@ -1018,6 +1015,25 @@ IF(NOT ENABLE_S3_SDK)
ENDIF()
ENDIF()
# Start disabling if curl not found
IF(NOT FOUND_CURL)
IF(ENABLE_BYTERANGE)
MESSAGE(FATAL_ERROR "Byte-range support specified, CURL libraries are not found.")
ENDIF()
IF(ENABLE_DAP2 OR ENABLE_DAP4)
MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.")
ENDIF()
IF(ENABLE_HDF5_ROS3)
MESSAGE(FATAL_ERROR "ROS3 support specified, CURL libraries are not found.")
ENDIF()
IF(ENABLE_NCZARR_S3)
MESSAGE(FATAL_ERROR "NCZarr S3 support specified, CURL libraries are not found.")
ENDIF()
ENDIF(NOT FOUND_CURL)
##
# Enable Tests
##

View File

@ -16,8 +16,9 @@
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
extern FILE* fdopen(int fd, const char *mode);
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "nclog.h"

View File

@ -92,7 +92,7 @@ IF(USE_HDF5)
ENDIF()
ENDIF()
IF(USE_DAP)
IF(FOUND_CURL)
SET(TLL_LIBS ${TLL_LIBS} ${CURL_LIBRARY})
ENDIF()

View File

@ -99,8 +99,8 @@ ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \
ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_order.txt ref_filter_repeat.txt \
ref_fillonly.cdl test_fillonly.sh
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc \
@ -110,7 +110,8 @@ tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp ref_grp_rename.cdl \
foo1.nc tst_*.h4 test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc bigvars.nc *.gz MSGCPP_*.nc \
floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \
testfilter_reg.nc filterorder.txt filterrepeat.txt tmp_fillonly.nc
testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \
testfilter_order.nc crfilterorder.txt rdfilterorder.txt
DISTCLEANFILES = findplugin.sh run_par_test.sh

View File

@ -1,9 +0,0 @@
test1: filter order.
test1: compression.
cd_nelmts=1 cd_values= 0
cd_nelmts=1 cd_values= 1
test1: decompression.
cd_nelmts=1 cd_values= 1
cd_nelmts=1 cd_values= 0
data comparison: |array|=256
no data errors

View File

@ -0,0 +1,12 @@
test1: filter order: create
def filter id[0]=40000 nparams=1 params[0]=0
def filter id[1]=40001 nparams=1 params[0]=1
inq filter ids(2)=[40000,40001]
inq filter id[0]=40000 nparams=1 params[0]=0
inq filter id[1]=40001 nparams=1 params[0]=1
test1: compression.
test1: decompression.
data comparison: |array|=256
no data errors
direction=compress id=40000 cd_nelmts=1 cd_values= 0
direction=compress id=40001 cd_nelmts=1 cd_values= 1

View File

@ -0,0 +1,10 @@
test2: filter order: read
test2: decompression.
inq filter ids(2)=[40000,40001]
inq filter id[0]=40000 nparams=1 params[0]=0
inq filter id[1]=40001 nparams=1 params[0]=1
test2: decompression.
direction=decompress id=40001 cd_nelmts=1 cd_values= 1
direction=decompress id=40000 cd_nelmts=1 cd_values= 0
data comparison: |array|=256
no data errors

View File

@ -1,8 +1,8 @@
test1: def filter repeat .
test1: compression.
cd_nelmts=2 cd_values= 0 18
direction=compress id=40000 cd_nelmts=2 cd_values= 0 18
test1: decompression.
filter(40000): |params|=2 params= 0 18
cd_nelmts=2 cd_values= 0 18
direction=decompress id=40000 cd_nelmts=2 cd_values= 0 18
data comparison: |array|=256
no data errors

View File

@ -25,7 +25,7 @@
#define MAXDIMS 8
#define TESTFILE "testfilter_reg.nc"
#define TESTFILE "testfilter_order.nc"
#define NPARAMS 1
#define PARAMVAL 17
@ -36,6 +36,7 @@ static size_t chunksize[NDIMS] = {4,4,4,4};
static size_t ndims = NDIMS;
static int creating = 1; /* Default is to do filter test 1 */
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
@ -124,25 +125,37 @@ create(void)
static void
deffilters(void)
{
unsigned int params[1];
/* Register filter 0 */
params[0] = 0;
printf("def filter id[0]=%d nparams=1 params[0]=%d\n",(int)FILTER_ID,(int)params[0]); fflush(stdout);
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID,1,params));
params[0] = 1;
printf("def filter id[1]=%d nparams=1 params[0]=%d\n",(int)FILTER_ID+1,(int)params[0]); fflush(stdout);
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+1,1,params));
}
static void
inqfilters(void)
{
unsigned int params[1];
unsigned int filterids[2];
size_t nfilters = 0;
size_t nparams = 0;
/* Register filter 0 */
params[0] = 0;
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID,1,params));
params[0] = 1;
CHECK(nc_def_var_filter(ncid,varid,FILTER_ID+1,1,params));
CHECK(nc_inq_var_filter_ids(ncid,varid,&nfilters,filterids));
printf("inq filter ids(%d)=[%d,%d]\n",(int)nfilters,(int)filterids[0],(int)filterids[1]); fflush(stdout);
if(nfilters != 2) REPORT("nfilters mismatch");
if(filterids[0] != FILTER_ID+0) REPORT("0: filterids mismatch");
if(filterids[1] != FILTER_ID+1) REPORT("1: filterids mismatch");
CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+0,&nparams,params));;
CHECK(nc_inq_var_filter_info(ncid,varid,filterids[0],&nparams,params));;
printf("inq filter id[0]=%d nparams=%d params[0]=%d\n",(int)filterids[0],(int)nparams,(int)params[0]); fflush(stdout);
if(nparams != 1) REPORT("0: nparams mismatch");
if(params[0] != 0) REPORT("0: param mismatch");
CHECK(nc_inq_var_filter_info(ncid,varid,FILTER_ID+1,&nparams,params));
CHECK(nc_inq_var_filter_info(ncid,varid,filterids[1],&nparams,params));
printf("inq filter id[1]=%d nparams=%d params[0]=%d\n",(int)filterids[1],(int)nparams,(int)params[0]); fflush(stdout);
if(nparams != 1) REPORT("1: nparams mismatch");
if(params[0] != 1) REPORT("1: param mismatch");
}
@ -214,7 +227,7 @@ static int
compare(void)
{
int errs = 0;
printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct); fflush(stdout);
if(1)
{
int i;
@ -245,10 +258,11 @@ compare(void)
}
if(errs == 0)
printf("no data errors\n");
{printf("no data errors\n"); fflush(stdout);}
return (errs == 0);
}
/* Test filter order on creation */
static int
filter_test1(void)
{
@ -256,23 +270,50 @@ filter_test1(void)
reset();
printf("test1: filter order.\n");
printf("test1: filter order: create\n"); fflush(stdout);
create();
setchunking();
deffilters();
inqfilters();
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
printf("test1: compression.\n");
nc_sync(ncid);
printf("test1: compression.\n"); fflush(stdout);
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
printf("test1: decompression.\n"); fflush(stdout);
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
printf("test1: decompression.\n");
return ok;
}
/* Test filter order on read */
static int
filter_test2(void)
{
int ok = 1;
reset();
printf("test2: filter order: read\n"); fflush(stdout);
/* Fill in the array */
fill();
printf("test2: decompression.\n"); fflush(stdout);
reset();
openfile();
inqfilters();
printf("test2: decompression.\n"); fflush(stdout);
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
@ -343,6 +384,11 @@ static void
init(int argc, char** argv)
{
int i;
creating = 1; /* default is test1 */
if(argc > 1 && strcmp(argv[1],"read")==0)
creating = 0;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
@ -368,6 +414,10 @@ main(int argc, char **argv)
nc_set_log_level(1);
#endif
init(argc,argv);
if(!filter_test1()) ERRR;
if(creating) {
if(!filter_test1()) ERRR;
} else {
if(!filter_test2()) ERRR;
}
exit(nerrs > 0?1:0);
}

View File

@ -11,8 +11,8 @@ UNK=1
NGC=1
MISC=1
MULTI=1
ORDER=1
REP=1
ORDER=1
# Load the findplugins function
. ${builddir}/findplugin.sh
@ -220,13 +220,6 @@ diff -b -w ${srcdir}/ref_nccopyF.cdl ./ncgenFs.cdl
echo "*** Pass: multiple filters"
fi
if test "x$ORDER" = x1 ; then
echo "*** Testing multiple filter order of invocation"
rm -f filterorder.txt
${execdir}/test_filter_order >filterorder.txt
diff -b -w ${srcdir}/ref_filter_order.txt filterorder.txt
fi
if test "x$REP" = x1 ; then
echo "*** Testing filter re-definition invocation"
rm -f filterrepeat.txt
@ -234,6 +227,20 @@ ${execdir}/test_filter_repeat >filterrepeat.txt
diff -b -w ${srcdir}/ref_filter_repeat.txt filterrepeat.txt
fi
if test "x$ORDER" = x1 ; then
echo "*** Testing multiple filter order of invocation on create"
rm -f crfilterorder.txt
${execdir}/test_filter_order create >crfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_create.txt crfilterorder.txt
echo "*** Testing multiple filter order of invocation on read"
rm -f rdfilterorder.txt
${execdir}/test_filter_order read >rdfilterorder.txt
diff -b -w ${srcdir}/ref_filter_order_read.txt rdfilterorder.txt
fi
echo "*** Pass: all selected tests passed"
#cleanup
@ -249,6 +256,7 @@ rm -f test_bzip2.c
rm -f multifilter.nc multi.cdl smulti.cdl
rm -f nccopyF.nc nccopyF.cdl ncgenF.nc ncgenF.cdl
rm -f ncgenFs.cdl nccopyFs.cdl
#rm -f crfilterorder.txt rdfilterorder.txt
exit 0

View File

@ -1,48 +0,0 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# Which test cases to exercise
ORDER=1
# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| cat > $2
}
# Function to extract _Filter attribute from a file
# These attributes might be platform dependent
getfilterattr() {
sed -e '/var.*:_Filter/p' -ed <$1 >$2
}
trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
# Execute the specified tests
if test "x$ORDER" = x1 ; then
echo "*** Testing multiple filter order of invocation"
rm -f filterorder.txt
${execdir}/test_filter_order >filterorder.txt
diff -b -w ${srcdir}/ref_filter_order.txt filterorder.txt
fi
echo "*** Pass: all selected tests passed"
#cleanup
rm -f filterorder.txt
exit 0

View File

@ -159,7 +159,6 @@ name_path(const char *path)
#define FILE_DELIMITER '/'
#endif
#ifdef USE_DAP
/* See if this is a url */
{
char* base;
@ -168,7 +167,6 @@ name_path(const char *path)
}
/* else fall thru and treat like a file path */
}
#endif /*USE_DAP*/
cp = strrchr(path, FILE_DELIMITER);
if (cp == 0) /* no delimiter */

View File

@ -79,7 +79,9 @@ IF(ENABLE_TESTS)
add_sh_test(nczarr_test run_fillonlyz)
ENDIF()
add_sh_test(nczarr_test run_ncgen4)
IF(FALSE) # Suppress for now
add_sh_test(nczarr_test run_chunkcases)
ENDIF()
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_TESTS)

View File

@ -50,11 +50,15 @@ endif
TESTS += run_ncgen4.sh
if AX_IGNORE
# suppress for now
check_PROGRAMS += tst_chunkcases
tst_chunkcases_SOURCES = tst_chunkcases.c ${tstcommonsrc}
TESTS += run_chunkcases.sh
endif
endif
if BUILD_BENCHMARKS
if BUILD_UTILITIES

View File

@ -14,10 +14,10 @@
#endif
#if NOOP_INSTANCE == 1
const static int instance1 = 1;
const static int instance[1] = {1};
#endif
#if NOOP_INSTANCE == 0
const static int instance0 = 1;
const static int instance[1] = {0};
#endif
/* use a temporary */
@ -96,18 +96,15 @@ H5Z_filter_noop(unsigned int flags, size_t cd_nelmts,
{
void* newbuf;
size_t i;
const char* direction = (flags & H5Z_FLAG_REVERSE) ? "decompress" : "compress";
#if NOOP_INSTANCE == 0
NC_UNUSED(instance0);
#endif
#if NOOP_INSTANCE == 1
NC_UNUSED(instance1);
#endif
NC_UNUSED(instance);
printf("cd_nelmts=%lu cd_values=",(unsigned long)cd_nelmts);
printf("direction=%s id=%lu cd_nelmts=%lu cd_values=",direction,(unsigned long)H5Z_NOOP[0].id,(unsigned long)cd_nelmts);
for(i=0;i<cd_nelmts;i++)
printf(" %u",cd_values[i]);
printf("\n");
fflush(stdout);
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
@ -127,7 +124,7 @@ H5Z_filter_noop(unsigned int flags, size_t cd_nelmts,
*buf = newbuf;
} else {
/* Replace buffer */
/* Replace buffer */
#ifdef HAVE_H5ALLOCATE_MEMORY
newbuf = H5allocate_memory(*buf_size,0);
#else

View File

@ -30,7 +30,7 @@
#include "bzlib_private.h"
extern FILE* fdopen(int,const char*);
//extern FILE* fdopen(int,const char*);
/*---------------------------------------------------*/
/*--- Compression stuff ---*/
@ -1382,8 +1382,8 @@ const char * BZ_API(BZ2_bzlibVersion)(void)
#endif
static
BZFILE * bzopen_or_bzdopen
( const char *path, /* no use when bzdopen */
int fd, /* no use when bzdopen */
( const char *path, /* not used when bzdopen */
int fd, /* not used when bzdopen */
const char *mode,
int open_mode) /* bzopen: 0, bzdopen:1 */
{
@ -1426,10 +1426,14 @@ BZFILE * bzopen_or_bzdopen
fp = fopen(path,mode2);
}
} else {
#if 0
#ifdef BZ_STRICT_ANSI
fp = NULL;
#else
fp = fdopen(fd,mode2);
#endif
#else
fp = NULL;
#endif
}
if (fp == NULL) return NULL;
@ -1467,13 +1471,14 @@ BZFILE * BZ_API(BZ2_bzopen)
/*---------------------------------------------------*/
#if 0
BZFILE * BZ_API(BZ2_bzdopen)
( int fd,
const char *mode )
{
return bzopen_or_bzdopen(NULL,fd,mode,/*bzdopen*/1);
}
#endif
/*---------------------------------------------------*/
int BZ_API(BZ2_bzread) (BZFILE* b, void* buf, int len )