mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-05 16:20:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ejh_notbuilt_errors

Browse Source
This commit is contained in:
Ed Hartnett 2017-11-15 03:38:49 -07:00 committed by GitHub
commit e928964e9e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
90 changed files with 11924 additions and 2315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -689,10 +689,11 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_deflate "" HAVE_H5PSET_DEFLATE)
#Check to see if H5Z_SZIP exists in HDF5_Libraries. If so, we must use szip.
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5P_SZIP "" USE_SZIP)
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Z_SZIP "" USE_SZIP)
IF(USE_SZIP)
FIND_LIBRARY(SZIP NAMES szip sz)
IF(SZIP)
SET(HAVE_H5Z_SZIP 1)
SET(SZIP_LIBRARY ${SZIP})
SET(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${SZIP})
ELSE()

3
RELEASE_NOTES.md
View File

@ -5,6 +5,9 @@ Release Notes {#RELEASE_NOTES}
This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
## 4.6.0 - TBD
* [Enhancement] Full support for using HDF5 dynamic filters, both for
reading and writing. See the file docs/filters.md.
## 4.5.1 - TBD

18
cf
View File

@ -6,6 +6,7 @@ FAST=1
HDF5=1
DAP=1
SZIP=1
#HDF4=1
PNETCDF=1
#PAR4=1
@ -18,11 +19,6 @@ if test "x$PNETCDF" = x1 -o "x$PAR4" = x1 ; then
MPIO=1
fi
#RPC=1
#M32=1
#M64=1
CFLAGS=""
#CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wconversion ${CFLAGS}"
CFLAGS="-Wall -Wno-unused-but-set-variable -Wno-unused-variable -Wno-unused-parameter -Wno-char-subscripts -Wno-pointer-sign -Wno-format ${CFLAGS}"
@ -47,6 +43,11 @@ if test "x$HDF4" = x1 ; then
HDF5=1
fi
# !HDF5=>!SZIP
if test "x$HDF5" != x1 ; then
SZIP=0
fi
CC=gcc
MALLOC_CHECK=""
@ -69,6 +70,10 @@ if test "x$HDF4" = "x1" ; then
LDFLAGS="$LDFLAGS -ljpeg"
fi
if test "x$SZIP" = "x1" ; then
LDFLAGS="$LDFLAGS -lsz -laec"
fi
export PKG_CONFIG_PATH=/usr/lib/pkgconfig
if test "x$DAP" = "x1" ; then
if curl-config --version >/dev/null ; then
@ -100,7 +105,7 @@ FLAGS="$FLAGS --enable-extra-tests"
#FLAGS="$FLAGS --disable-testsets"
#FLAGS="$FLAGS --disable-dap-remote-tests"
#FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
#FLAGS="$FLAGS --enable-doxygen"
#FLAGS="$FLAGS --enable-doxygen --enable-internal-docs"
FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --enable-mmap"
@ -112,6 +117,7 @@ FLAGS="$FLAGS --enable-logging"
#FLAGS="$FLAGS --disable-properties-attribute"
#FLAGS="$FLAGS --disable-silent-rules"
#FLAGS="$FLAGS --with-testservers=remotestserver.localhost:8083"
FLAGS="$FLAGS --enable-filter-test"
if test "x$PAR4" != x1 ; then
FLAGS="$FLAGS --disable-parallel4"

8
cf.cmake
View File

@ -48,19 +48,19 @@ NCLIB=`pwd`
if test "x$VS" != x ; then
# Visual Studio
#CFG="RelWithDebInfo"
CFG="Release"
NCLIB="${NCLIB}/build/liblib/$CFG"
export PATH="${NCLIB}:${PATH}"
cmake $FLAGS ..
if test "x$VSSETUP" = x ; then
cmake --build . --config ${CFG}
cmake --build . --config ${CFG} --target RUN_TESTS
fi
else
# GCC
NCLIB="${NCLIB}/build/liblib"
G="-GUnix Makefiles"
cmake "${G}" $FLAGS ..
make all
make test
#make all
#make test
fi
exit

17
configure.ac
View File

@ -1386,6 +1386,21 @@ if test "x$enable_jna" = xyes ; then
AC_DEFINE([JNA], [1], [if true, include JNA bug fix])
fi
# Control filter test(s)
AC_MSG_CHECKING([If filter_test should be enabled (default is no)])
AC_ARG_ENABLE([filter-test],
[AS_HELP_STRING([--enable-filter-test],
[enable filter test])])
test "x$enable_filter_test" = xyes || enable_filter_test=no
AC_MSG_RESULT($enable_filter_test)
if test "x$enable_netcdf_4" = xno ; then
AC_MSG_WARN([disabling --enable-filter-test; it requires netcdf-4])
enable_filter_test=no
fi
# Define a Generic AM_CONDITIONAL to allow disabling
# of arbitrary parts of Makefile.am files
AM_CONDITIONAL(ENABLE_FILTER_TEST, [test x$enable_filter_test = xyes])
AC_SUBST(NC_LIBS,[$NC_LIBS])
AC_SUBST(HAS_DAP,[$enable_dap])
AC_SUBST(HAS_DAP2,[$enable_dap])
@ -1515,6 +1530,7 @@ AC_CONFIG_FILES([Makefile
docs/images/Makefile
nctest/Makefile
nc_test4/Makefile
nc_test4/filter_test/Makefile
nc_test/Makefile
ncdap_test/Makefile
ncdap_test/testdata3/Makefile
@ -1528,3 +1544,4 @@ AC_OUTPUT()
#mv -f ${abs_top_srcdir}/test_common.sh ${abs_top_builddir}/test_common.sh
cat libnetcdf.settings

2
dap4_test/tst_raw.sh
View File

@ -5,8 +5,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ${srcdir}/d4test_common.sh
set -x
# Compute the set of testfiles
PUSHD ${srcdir}/daptestfiles
F=`ls -1d *.dap`

1
docs/Doxyfile.in
View File

@ -759,6 +759,7 @@ INPUT = \
@abs_top_srcdir@/docs/tutorial.dox \
@abs_top_srcdir@/docs/notes.md \
@abs_top_srcdir@/docs/auth.md \
@abs_top_srcdir@/docs/filters.md \
@abs_top_srcdir@/docs/all-error-codes.md \
@abs_top_srcdir@/docs/FAQ.md \
@abs_top_srcdir@/docs/known_problems.md \

2
docs/Makefile.am
View File

@ -11,7 +11,7 @@ EXTRA_DIST = netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html \
building-with-cmake.md CMakeLists.txt \
groups.dox install.md notes.md install-fortran.md \
all-error-codes.md cmake_faq.md credits.md \
auth.md obsolete/fan_utils.html bestpractices.md
auth.md obsolete/fan_utils.html bestpractices.md filters.md
# Turn off parallel builds in this directory.
.NOTPARALLEL:

396
docs/filters.md Normal file
View File

@ -0,0 +1,396 @@
Filter Support in netCDF-4 (Enhanced)
============================
<!-- double header is needed to workaround doxygen bug -->
Filter Support in netCDF-4 (Enhanced) {#compress}
=================================
[TOC]
Introduction {#compress_intro}
==================
The HDF5 library (1.8.11 and later)
supports a general filter mechanism to apply various
kinds of filters to datasets before reading or writing.
The netCDF enhanced (aka netCDF-4) library inherits this
capability since it depends on the HDF5 library.
Filters assume that a variable has chunking
defined and each chunk is filtered before
writing and "unfiltered" after reading and
before passing the data to the user.
The most common kind of filter is a compression-decompression
filter, and that is the focus of this document.
HDF5 supports dynamic loading of compression filters using the following
process for reading of compressed data.
1. Assume that we have a dataset with one or more variables that
were compressed using some algorithm. How the dataset was compressed
will be discussed subsequently.
2. Shared libraries or DLLs exist that implement the compress/decompress
algorithm. These libraries have a specific API so that the HDF5 library
can locate, load, and utilize the compressor.
These libraries are expected to installed in a specific
directory.
Enabling A Compression Filter {#Enable}
=============================
In order to compress a variable, the netcdf-c library
must be given three pieces of information:
(1) some unique identifier for the filter to be used,
(2) a vector of parameters for
controlling the action of the compression filter, and
(3) a shared library implementation of the filter.
The meaning of the parameters is, of course,
completely filter dependent and the filter
description [3] needs to be consulted. For
bzip2, for example, a single parameter is provided
representing the compression level.
It is legal to provide a zero-length set of parameters.
Defaults are not provided, so this assumes that
the filter can operate with zero parameters.
Filter ids are assigned by the HDF group. See [4]
for a current list of assigned filter ids.
Note that ids above 32767 can be used for testing without
registration.
The first two pieces of information can be provided in one of three ways:
using __ncgen__, via an API call, or via command line parameters to __nccopy__.
In any case, remember that filtering also requires setting chunking, so the
variable must also be marked with chunking information.
Using The API {#API}
-------------
The necessary API methods are included in __netcdf.h__ by default.
One API method is for setting the filter to be used
when writing a variable. The relevant signature is
as follows.
````
int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms);
````
This must be invoked after the variable has been created and before
__nc_enddef__ is invoked.
A second API methods makes it possible to query a variable to
obtain information about any associated filter using this signature.
````
int nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparams, unsigned int* params);
````
The filter id wil be returned in the __idp__ argument (if non-NULL),
the number of parameters in __nparamsp__ and the actual parameters in
__params__. As is usual with the netcdf API, one is expected to call
this function twice. The first time to get __nparams__ and the
second to get the parameters in client-allocated memory.
Using ncgen {#NCGEN}
-------------
In a CDL file, compression of a variable can be specified
by annotating it with the following attribute:
* ''_Filter'' -- a string containing a comma separated list of
constants specifying (1) the filter id to apply, and (2)
a vector of constants representing the
parameters for controlling the operation of the specified filter.
See the section on the <a href="#Syntax">parameter encoding syntax</a>
for the details on the allowable kinds of constants.
This is a "special" attribute, which means that
it will normally be invisible when using
__ncdump__ unless the -s flag is specified.
Example CDL File (Data elided)
------------------------------
````
netcdf bzip2 {
dimensions:
dim0 = 4 ; dim1 = 4 ; dim2 = 4 ; dim3 = 4 ;
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Filter = "307,9" ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
data:
...
}
````
Using nccopy {#NCCOPY}
-------------
When copying a netcdf file using __nccopy__ it is possible
to specify filter information for any output variable by
using the "-F" option on the command line; for example:
````
nccopy -F "var,307,9" unfiltered.nc filtered.nc
````
Assume that __unfiltered.nc__ has a chunked but not bzip2 compressed
variable named "var". This command will create that variable in
the __filtered.nc__ output file but using filter with id 307
(i.e. bzip2) and with parameter(s) 9 indicating the compression level.
See the section on the <a href="#Syntax">parameter encoding syntax</a>
for the details on the allowable kinds of constants.
The "-F" option can be used repeatedly as long as the variable name
part is different. A different filter id and parameters can be
specified for each occurrence.
Note that if the input file has compressed variables, that fact
will be invisble to nccopy because it is handled within the
netcdf-c/hdf5 library code. This is true for any program that calls
the netcdf-c library.
Parameter Encoding {#ParamEncode}
==========
The parameters passed to a filter are encoded internally as a vector
of 32-bit unsigned integers. It may be that the parameters
required by a filter can naturally be encoded as unsigned integers.
The bzip2 compression filter, for example, expects a single
integer value from zero thru nine. This encodes naturally as a
single unsigned integer.
Note that signed integers and single-precision (32-bit) float values
also can easily be represented as 32 bit unsigned integers by
proper casting to an unsigned integer so that the bit pattern
is preserved. Simple integer values of type short or char
(or the unsigned versions) can also be mapped to an unsigned
integer by either sign extension or just padding with zeros in some
consistent way.
Machine byte order (aka endian-ness) is an issue for passing
some kinds of parameters. You might define the parameters when
compressing on a little endian machine, but later do the
decompression on a big endian machine. Byte order is not an
issue for 32-bit values because HDF5 takes care of converting
them between the local machine byte order and network byte
order.
Parameters whose size is larger than 32-bits present a byte order problem.
This typically includes double precision floats and (signed or unsigned)
64-bit integers. For these cases, the machine byte order must be
handled by the compression code. This is because HDF5 will treat,
for example, an unsigned long long as two 32-bit unsigned integers
and will convert each to network order separately. This means that
on a machine whose byte order is different than the machine in which
the parameters were initially created, the two integers are out of order
and must be swapped to get the correct unsigned long long value.
Consider this example. Suppose we have this little endian unsigned long long.
1000000230000004
In network byte order, it will be stored as two 32-bit integers.
20000001 40000003
On a big endian machine, this will be given to the filter in that form.
2000000140000003
But note that the proper big endian unsigned long long form is this.
4000000320000001
So, the two words need to be swapped.
But consider the case when both original and final machines are big endian.
1. 4000000320000001
2. 40000003 20000001
3. 40000003 20000001
where #1 is the original number, #2 is the network order and
#3 is the what is given to the filter. In this case we do not
want to swap words.
The solution is to forcibly encode the original number using some
specified endianness so that the filter always assumes it is getting
its parameters in that order and will always do swapping as needed.
This is irritating, but one needs to be aware of it. Since most
machines are little-endian. We choose to use that as the endianness
for handling 64 bit entities.
Filter Specification Syntax {#Syntax}
==========
Both of the utilities
<a href="#NCGEN">__ncgen__</a>
and
<a href="#NCCOPY">__nccopy__</a>
allow the specification of filter parameters.
These specifications consist of a sequence of comma
separated constants. The constants are converted
within the utility to a proper set of unsigned int
constants (see the <a href="#ParamEncode">parameter encoding section</a>).
To simplify things, various kinds of constants can be specified
rather than just simple unsigned integers. The utilities will encode
them properly using the rules specified in
the <a href="#ParamEncode">parameter encoding section</a>.
The currently supported constants are as follows.
<table>
<tr halign="center"><th>Example<th>Type<th>Format Tag<th>Notes
<tr><td>-17b<td>signed 8-bit byte<td>b|B<td>
<tr><td>23ub<td>unsigned 8-bit byte<td>u|U b|B<td>
<tr><td>-25S<td>signed 16-bit short<td>s|S<td>
<tr><td>27US<td>unsigned 16-bit short<td>u|U s|S<td>
<tr><td>-77<td>implicit signed 32-bit integer<td>Leading minus sign and no tag<td>
<tr><td>77<td>implicit unsigned 32-bit integer<td>No tag<td>
<tr><td>93U<td>explicit unsigned 32-bit integer<td>u|U<td>
<tr><td>789f<td>32-bit float<td>f|F<td>
<tr><td>12345678.12345678d<td>64-bit double<td>d|D<td>Network byte order
<tr><td>-9223372036854775807L<td>64-bit signed long long<td>l|L<td>Network byte order
<tr><td>18446744073709551615UL<td>64-bit unsigned long long<td>u|U l|L<td>Network byte order
</table>
Some things to note.
1. In all cases, except for an untagged positive integer,
the format tag is required and determines how the constant
is converted to one or two unsigned int values.
The positive integer case is for backward compatibility.
2. For signed byte and short, the value is sign extended to 32 bits
and then treated as an unsigned int value.
3. For double, and signed|unsigned long long, they are converted
to network byte order and then treated as two unsigned int values.
This is consistent with the <a href="#ParamEncode">parameter encoding</a>.
Dynamic Loading Process {#Process}
==========
The documentation[1,2] for the HDF5 dynamic loading was (at the time
this was written) out-of-date with respect to the actual HDF5 code
(see HDF5PL.c). So, the following discussion is largely derived
from looking at the actual code. This means that it is subject to change.
Plugin directory {#Plugindir}
----------------
The HDF5 loader expects plugins to be in a specified plugin directory.
The default directory is:
* "/usr/local/hdf5/lib/plugin” for linux/unix operating systems (including Cygwin)
* “%ALLUSERSPROFILE%\\hdf5\\lib\\plugin” for Windows systems, although the code
does not appear to explicitly use this path.
The default may be overridden using the environment variable
__HDF5_PLUGIN_PATH__.
Plugin Library Naming {#Pluginlib}
---------------------
Given a plugin directory, HDF5 examines every file in that
directory that conforms to a specified name pattern
as determined by the platform on which the library is being executed.
<table>
<tr halign="center"><th>Platform<th>Basename<th>Extension
<tr halign="left"><td>Linux<td>lib*<td>.so*
<tr halign="left"><td>OSX<td>lib*<td>.dylib*
<tr halign="left"><td>Cygwin<td>cyg*<td>.dll*
<tr halign="left"><td>Windows<td>*<td>.dll
</table>
Plugin Verification {#Pluginverify}
-------------------
For each dynamic library located using the previous patterns,
HDF5 attempts to load the library and attempts to obtain information
from it. Specifically, It looks for two functions with the following
signatures.
1. __H5PL_type_t H5PLget_plugin_type(void)__ --
This function is expected to return the constant value
__H5PL_TYPE_FILTER__ to indicate that this is a filter library.
2. __const void* H5PLget_plugin_info(void)__ --
This function returns a pointer to a table of type __H5Z_class2_t__.
This table contains the necessary information needed to utilize the
filter both for reading and for writing. In particular, it specifies
the filter id implemented by the library and if must match that id
specified for the variable in __nc_def_var_filter__ in order to be used.
If plugin verification fails, then that plugin is ignored and
the search continues for another, matching plugin.
Debugging {#Debug}
-------
Debugging plugins can be very difficult. You will probably
need to use the old printf approach for debugging the filter itself.
One case worth mentioning is when you have a dataset that is
using an unknown filter. For this situation, you need to
identify what filter(s) are used in the dataset. This can
be accomplished using this command.
````
ncdump -s -h <dataset filename>
````
Since ncdump is not being asked to access the data (the -h flag), it
can obtain the filter information without failures. Then it can print
out the filter id and the parameters (the -s flag).
Test Case {#TestCase}
-------
Within the netcdf-c source tree, the directory __netcdf-c/nc_test4/filter_test__ contains
a test case for testing dynamic filter writing and reading
using bzip2. The test case is fragile and is only known to work for Linux
and Cygwin. To avoid spurious failures, it must be explicitly
invoked by entering the directory and issuing the command
````
make clean all makebzip2 check
````
Note that the Make target 'makebzip2' creates the bzip2 shared object.
It is the target most likely to fail when executed on an untested or
non-standard platform.
Although it is fragile, this test can serve as a complete example for building
plugins for other filters.
Appendix A. Byte Swap Code {#AppendixA}
==========
Since in some cases, it is necessary for a filter to
byte swap from little-endian to big-endian, This appendix
provides sample code for doing this. It also provides
a code snippet for testing if the machine the
endianness of a machine.
Byte swap an 8-byte chunk of memory
-------
````
static void
byteswap8(unsigned char* mem)
{
register unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
````
Test for Machine Endianness
-------
````
static const unsigned char b[4] = {0x0,0x0,0x0,0x1}; /* value 1 in big-endian*/
int endianness = (1 == *(unsigned int*)b); /* 1=>big 0=>little endian
````
References {#References}
==========
1. https://support.hdfgroup.org/HDF5/doc/Advanced/DynamicallyLoadedFilters/HDF5DynamicallyLoadedFilters.pdf
2. https://support.hdfgroup.org/HDF5/doc/TechNotes/TechNote-HDF5-CompressionTroubleshooting.pdf
3. https://support.hdfgroup.org/services/filters.html
4. https://support.hdfgroup.org/services/contributions.html#filters

1
docs/guide.dox
View File

@ -10,6 +10,7 @@
- \subpage netcdf_utilities_guide
- \subpage dap2
- \subpage dap4
- \subpage compress
- \subpage BestPractices
- \subpage users_guide_appendices

10
docs/install.md
View File

@ -97,7 +97,15 @@ HDF5 and zlib packages are available from the <a href="http://www.hdfgroup.org/d
### Optional: szip support {#op_szip_support}
*Optionally*, you can also build netCDF-4 with the szip library (a.k.a. szlib). If building with szlib, get szip 2.0 or later. NetCDF cannot create szipped data files, but can read HDF5 data files that have used szip. To determine whether license restrictions on the use of szip apply to your situation, see the <a href="http://www.hdfgroup.org/doc_resource/SZIP/">web page on szip compression in HDF products</a>.
*Optionally*, you can also build netCDF-4 with the szip library (a.k.a. szlib). If building with szlib, get szip 2.0 or later. Technically, we mean that
the HDF5 library is built with szip support. The netcdf build will then
inherit szip support from the HDF5 library.
If you intend to write files with szip compression, then we suggest that you
use [libaec](https://gitlab.dkrz.de/k202009/libaec.git)
to avoid patent problems. That library can be used as a
drop-in replacement for the standard szip library.
If you plan to use the standard szip library,
then determine whether license restrictions on the use of szip apply to your situation. See the <a href="http://www.hdfgroup.org/doc_resource/SZIP/">web page on szip compression in HDF products</a>.
If `make check` fails for either `zlib` or `HDF5`, the problem must be resolved before the netCDF-4 installation can continue. For HDF5 problems, see the <a href="http://www.hdfgroup.org/services/support.html">HDF5 help services</a>.

2
include/Makefile.am
View File

@ -19,7 +19,7 @@ nclist.h ncuri.h ncutf8.h ncdispatch.h ncdimscale.h \
netcdf_f.h err_macros.h ncbytes.h nchashmap.h ceconstraints.h rnd.h \
nclog.h ncconfigure.h nc4internal.h nctime.h nc3internal.h \
onstack.h nc_hashmap.h ncrc.h ncauth.h ncoffsets.h nctestserver.h \
nc4dispatch.h nc3dispatch.h ncexternl.h ncwinpath.h
nc4dispatch.h nc3dispatch.h ncexternl.h ncwinpath.h ncfilter.h
if USE_DAP
noinst_HEADERS += ncdap.h

6
include/nc4dispatch.h
View File

@ -126,7 +126,8 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
extern int
NC4_inq_varid(int ncid, const char *name, int *varidp);
@ -249,6 +250,9 @@ NC4_def_var_fill(int, int, int, const void *);
extern int
NC4_def_var_endian(int, int, int);
extern int
NC4_def_var_filter(int, int, unsigned int, size_t, const unsigned int*);
extern int
NC4_set_var_chunk_cache(int, int, size_t, size_t, float);

9
include/nc4internal.h
View File

@ -179,9 +179,6 @@ typedef struct NC_VAR_INFO
int deflate_level;
nc_bool_t shuffle; /* True if var has shuffle filter applied */
nc_bool_t fletcher32; /* True if var has fletcher32 filter applied */
nc_bool_t szip; /* True if var has szip filter applied */
int options_mask;
int pixels_per_block;
size_t chunk_cache_size, chunk_cache_nelems;
float chunk_cache_preemption;
#ifdef USE_HDF4
@ -189,7 +186,11 @@ typedef struct NC_VAR_INFO
int sdsid;
int hdf4_data_type;
#endif /* USE_HDF4 */
/* Stuff below for diskless data files. */
/* Stuff for arbitrary filters */
unsigned int filterid;
size_t nparams;
unsigned int* params;
/* Stuff for diskless data files. */
void *diskless_data;
} NC_VAR_INFO_T;

8
include/ncdispatch.h
View File

@ -246,7 +246,8 @@ int (*inq_var_all)(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
int (*var_par_access)(int, int, int);
@ -290,6 +291,7 @@ int (*def_var_fletcher32)(int, int, int);
int (*def_var_chunking)(int, int, int, const size_t*);
int (*def_var_fill)(int, int, int, const void*);
int (*def_var_endian)(int, int, int);
int (*def_var_filter)(int, int, unsigned int, size_t, const unsigned int*);
int (*set_var_chunk_cache)(int, int, size_t, size_t, float);
int (*get_var_chunk_cache)(int ncid, int varid, size_t *sizep, size_t *nelemsp, float *preemptionp);
#endif /*USE_NETCDF4*/
@ -398,8 +400,8 @@ NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* paramsp
);
extern int
NCDISPATCH_get_att(int ncid, int varid, const char* name, void* value, nc_type t);

25
include/ncfilter.h Normal file
View File

@ -0,0 +1,25 @@
/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
See the COPYRIGHT file for more information. */
#ifndef NCFILTER_H
#define NCFILTER_H 1
/* API for libdispatch/dfilter.c */
/* Must match values in <H5Zpublic.h> */
#ifndef H5Z_FILTER_SZIP
#define H5Z_FILTER_SZIP 4
#endif
#if defined(__cplusplus)
extern "C" {
#endif
/* Provide consistent filter spec parser */
EXTERNL int NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp);
#if defined(__cplusplus)
}
#endif
#endif /* NCFILTER_H */

15
include/netcdf.h
View File

@ -448,9 +448,9 @@ by the desired type. */
#define NC_ECANTEXTEND (-130) /**< Attempt to extend dataset during ind. I/O operation. */
#define NC_EMPI (-131) /**< MPI operation failed. */
#define NC_ERCFILE (-132) /**< RC file failure */
#define NC4_LAST_ERROR (-132)
#define NC_EFILTER (-132) /**< Filter operation failed. */
#define NC_ERCFILE (-133) /**< RC file failure */
#define NC4_LAST_ERROR (-134)
/* This is used in netCDF-4 files for dimensions without coordinate
* vars. */
@ -460,7 +460,6 @@ by the desired type. */
* mistake of having chunksizes be first ints, then size_t. Doh! */
#define NC_HAVE_NEW_CHUNKING_API 1
/*Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
#define NC_EURL (NC_EDAPURL) /* Malformed URL */
#define NC_ECONSTRAINT (NC_EDAPCONSTRAINT) /* Malformed Constraint*/
@ -860,6 +859,14 @@ nc_def_var_endian(int ncid, int varid, int endian);
EXTERNL int
nc_inq_var_endian(int ncid, int varid, int *endianp);
/* Define a filter for a variable */
EXTERNL int
nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms);
/* Learn about the filter on a variable */
EXTERNL int
nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparams, unsigned int* params);
/* Set the fill mode (classic or 64-bit offset files only). */
EXTERNL int
nc_set_fill(int ncid, int fillmode, int *old_modep);

17
libdap2/ncd2dispatch.c
View File

@ -174,6 +174,7 @@ NCD2_def_var_fletcher32,
NCD2_def_var_chunking,
NCD2_def_var_fill,
NCD2_def_var_endian,
NCD2_def_var_filter,
NCD2_set_var_chunk_cache,
NCD2_get_var_chunk_cache,
@ -2391,7 +2392,8 @@ NCD2_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
int* shufflep, int* deflatep, int* deflate_levelp,
int* fletcher32p, int* contiguousp, size_t* chunksizesp,
int* no_fill, void* fill_valuep, int* endiannessp,
int* options_maskp, int* pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
NC* drno;
int ret;
@ -2401,7 +2403,8 @@ NCD2_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
shufflep, deflatep, deflate_levelp,
fletcher32p, contiguousp, chunksizesp,
no_fill, fill_valuep, endiannessp,
options_maskp, pixels_per_blockp);
idp,nparamsp,params
);
return THROW(ret);
}
@ -2780,6 +2783,16 @@ NCD2_def_var_endian(int ncid, int p2, int p3)
return THROW(ret);
}
int
NCD2_def_var_filter(int ncid, int varid, unsigned int id, size_t n, const unsigned int* params)
{
NC* drno;
int ret;
if((ret = NC_check_id(ncid, (NC**)&drno)) != NC_NOERR) return THROW(ret);
ret = nc_def_var_filter(getnc3id(drno), varid, id, n, params);
return THROW(ret);
}
int
NCD2_set_var_chunk_cache(int ncid, int p2, size_t p3, size_t p4, float p5)
{

9
libdap2/ncd2dispatch.h
View File

@ -135,7 +135,8 @@ NCD2_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
extern int
NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
@ -143,7 +144,8 @@ NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
extern int
NCD2_inq_varid(int ncid, const char *name, int *varidp);
@ -245,6 +247,9 @@ NCD2_var_par_access(int, int, int);
extern int
NCD2_def_var_endian(int, int, int);
extern int
NCD2_def_var_filter(int, int, unsigned int, size_t, const unsigned int*);
extern int
NCD2_inq_unlimdims(int, int *, int *);

6
libdap4/d4meta.c
View File

@ -531,11 +531,15 @@ buildCompound(NCD4meta* builder, NCD4node* cmpdtype, NCD4node* group, char* name
field->name, field->meta.offset,
field->basetype->meta.id)));
} else if(rank > 0) { /* array */
int idimsizes[NC_MAX_VAR_DIMS];
int j;
getDimsizes(field,dimsizes);
/* nc_insert_array_compound: dimsizes arg is not size_t */
for(j=0;j<rank;j++) idimsizes[j] = (int)dimsizes[j];
NCCHECK((nc_insert_array_compound(group->meta.id, cmpdtype->meta.id,
field->name, field->meta.offset,
field->basetype->meta.id,
rank, dimsizes)));
rank, idimsizes)));
}
}

12
libdap4/ncd4dispatch.c
View File

@ -238,6 +238,12 @@ NCD4_def_var_endian(int ncid, int p2, int p3)
return (NC_EPERM);
}
static int
NCD4_def_var_filter(int ncid, int varid, unsigned int id, size_t n, const unsigned int* parms)
{
return (NC_EPERM);
}
static int
NCD4_set_var_chunk_cache(int ncid, int p2, size_t p3, size_t p4, float p5)
{
@ -413,7 +419,8 @@ NCD4_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
int* shufflep, int* deflatep, int* deflate_levelp,
int* fletcher32p, int* contiguousp, size_t* chunksizesp,
int* no_fill, void* fill_valuep, int* endiannessp,
int* options_maskp, int* pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
NC* ncp;
int ret;
@ -425,7 +432,7 @@ NCD4_inq_var_all(int ncid, int varid, char *name, nc_type* xtypep,
shufflep, deflatep, deflate_levelp,
fletcher32p, contiguousp, chunksizesp,
no_fill, fill_valuep, endiannessp,
options_maskp, pixels_per_blockp);
idp, nparamsp, params);
return (ret);
}
@ -883,6 +890,7 @@ NCD4_def_var_fletcher32,
NCD4_def_var_chunking,
NCD4_def_var_fill,
NCD4_def_var_endian,
NCD4_def_var_filter,
NCD4_set_var_chunk_cache,
NCD4_get_var_chunk_cache,

2
libdispatch/CMakeLists.txt
View File

@ -1,7 +1,7 @@
SET(libdispatch_SOURCES dparallel.c dcopy.c dfile.c ddim.c datt.c dattinq.c dattput.c dattget.c derror.c dvar.c dvarget.c dvarput.c dvarinq.c ddispatch.c nclog.c dstring.c dutf8.c dinternal.c doffsets.c ncuri.c nclist.c ncbytes.c nchashmap.c nctime.c nc.c nclistmgr.c utf8proc.h utf8proc.c dwinpath.c dutil.c drc.c dauth.c)
IF(USE_NETCDF4)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c ncaux.c)
SET(libdispatch_SOURCES ${libdispatch_SOURCES} dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c ncaux.c dfilter.c)
ENDIF(USE_NETCDF4)
IF(BUILD_V2)

5
libdispatch/Makefile.am
View File

@ -28,7 +28,7 @@ libdispatch_la_SOURCES += utf8proc.c utf8proc.h
# Add functions only found in netCDF-4.
if USE_NETCDF4
libdispatch_la_SOURCES += dgroup.c dvlen.c dcompound.c dtype.c denum.c \
dopaque.c ncaux.c
dopaque.c dfilter.c ncaux.c
endif # USE_NETCDF4
# Turn on pre-processor flag when building a DLL for windows.
@ -38,7 +38,7 @@ endif # BUILD_DLL
# Add V2 API convenience library if needed.
if BUILD_V2
noinst_LTLIBRARIES += libnetcdf2.la
noinst_LTLIBRARIES += libnetcdf2.la
libnetcdf2_la_SOURCES = dv2i.c
libnetcdf2_la_CPPFLAGS = ${AM_CPPFLAGS} -DDLL_EXPORT
endif # BUILD_V2
@ -70,4 +70,3 @@ test_ncuri_SOURCES = test_ncuri.c ncuri.c ncbytes.c nclist.c
test_pathcvt_SOURCES = test_pathcvt.c dwinpath.c
check_PROGRAMS = test_ncuri test_pathcvt
TESTS = test_ncuri test_pathcvt

2
libdispatch/derror.c
View File

@ -252,6 +252,8 @@ const char *nc_strerror(int ncerr1)
"when netCDF was built.";
case NC_EDISKLESS:
return "NetCDF: Error in using diskless access";
case NC_EFILTER:
return "NetCDF: Filter error: bad id or parameters or filter library non-existent";
case NC_ECANTEXTEND:
return "NetCDF: Attempt to extend dataset during NC_INDEPENDENT I/O operation. Use nc_var_par_access to set mode NC_COLLECTIVE before extending variable.";
case NC_EMPI: return "NetCDF: MPI operation failed.";

201
libdispatch/dfilter.c Normal file
View File

@ -0,0 +1,201 @@
/*
* Copyright 1996, University Corporation for Atmospheric Research
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#ifdef _MSC_VER
#include <io.h>
#endif
#include "netcdf.h"
/*
Common utilities related to filters.
*/
/* Forward */
#ifdef WORDS_BIGENDIAN
static void byteswap8(unsigned char* mem);
#endif
/**************************************************/
/*
Parse a filter spec string into a filter id + a vector
of unsigned ints.
@param spec0 - a string containing the spec as a sequence of
constants separated by commas.
@param idp - store the parsed filter id here
@param nparamsp - store number of parsed filter params here
@param paramsp - store the vector of parsed filter params here
@return 1 if parse succeeded, 0 otherwise.
*/
EXTERNL int
NC_parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp)
{
char* p;
char* sdata = NULL;
int stat;
unsigned int id;
size_t count; /* no. of comma delimited params */
size_t nparams; /* final no. of unsigned ints */
size_t i;
unsigned int* ulist = NULL;
unsigned char mem[8]; /* to convert to network byte order */
if(spec == NULL || strlen(spec) == 0) goto fail;
sdata = strdup(spec);
/* Count number of parameters + id and delimit */
p=sdata;
for(count=0;;count++) {
char* q = strchr(p,',');
if(q == NULL) break;
*q++ = '\0';
p = q;
}
count++; /* for final piece */
if(count == 0)
goto fail; /* no id and no parameters */
/* Extract the filter id */
p = sdata;
stat = sscanf(p,"%u",&id);
if(stat != 1) goto fail;
p = p + strlen(p) + 1; /* skip the filter id */
count--;
/* Allocate the max needed space; *2 in case the params are all doubles */
ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count)*2);
if(ulist == NULL) goto fail;
/* walk and convert */
nparams = 0;
for(i=0;i<count;i++) {
char* q;
unsigned long long val64u;
unsigned int val32u;
double vald;
float valf;
unsigned int *vector;
int isunsigned;
int isnegative;
int type;
/* Get trailing discrimination characters */
isunsigned = 0;
isnegative = 0;
type = 0;
if(strchr(p,'-') != NULL) isnegative = 1;
q = p+strlen(p)-2;
if(*q == 'U' || *q == 'u') isunsigned = 1;
q++;
switch (*q) {
case 'f': case 'F': type = 'f'; break; /* short */
case 'd': case 'D': type = 'd'; break; /* double */
case 'b': case 'B': type = 'b'; break; /* byte */
case 's': case 'S': type = 's'; break; /* short */
case 'l': case 'L': type = 'l'; break; /* long long */
case 'u': case 'U': type = 'i'; isunsigned = 1; break; /* integer */
case '.':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': type = 'i'; break;
default:
if(*q == '\0')
type = 'i';
else goto fail;
}
/* Now parse */
switch (type) {
case 'b':
case 's':
case 'i':
/* special case for a positive integer;for back compatibility.*/
if(!isnegative)
stat = sscanf(p,"%u",&val32u);
else
stat = sscanf(p,"%d",(int*)&val32u);
if(stat != 1) goto fail;
ulist[nparams++] = val32u;
break;
case 'f':
stat = sscanf(p,"%lf",&vald);
if(stat != 1) goto fail;
valf = (float)vald;
ulist[nparams++] = *(unsigned int*)&valf;
break;
case 'd':
stat = sscanf(p,"%lf",&vald);
if(stat != 1) goto fail;
/* convert to network byte order */
memcpy(mem,&vald,sizeof(mem));
#ifdef WORDS_BIGENDIAN
byteswap8(mem); /* convert big endian to little endian */
#endif
vector = (unsigned int*)mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
case 'l': /* long long */
if(isunsigned)
stat = sscanf(p,"%llu",&val64u);
else
stat = sscanf(p,"%lld",(long long*)&val64u);
if(stat != 1) goto fail;
/* convert to network byte order */
memcpy(mem,&val64u,sizeof(mem));
#ifdef WORDS_BIGENDIAN
byteswap8(mem); /* convert big endian to little endian */
#endif
vector = (unsigned int*)mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
default:
goto fail;
}
p = p + strlen(p) + 1; /* move to next param */
}
/* Now return results */
if(idp) *idp = id;
if(nparamsp) *nparamsp = nparams;
if(paramsp) {
*paramsp = ulist;
ulist = NULL; /* avoid duplicate free */
}
if(sdata) free(sdata);
if(ulist) free(ulist);
return 1;
fail:
if(sdata) free(sdata);
if(ulist) free(ulist);
return 0;
}
#ifdef WORDS_BIGENDIAN
/* Byte swap an 8-byte integer in place */
static void
byteswap8(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
#endif

6
libdispatch/dinternal.c
View File

@ -20,7 +20,8 @@ NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
@ -32,8 +33,7 @@ NCDISPATCH_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
contiguousp, chunksizesp,
no_fill, fill_valuep,
endiannessp,
options_maskp,
pixels_per_blockp);
idp, nparamsp, params);
}
int

9
libdispatch/dvar.c
View File

@ -1018,4 +1018,13 @@ nc_def_var_endian(int ncid, int varid, int endian)
return ncp->dispatch->def_var_endian(ncid,varid,endian);
}
int
nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
return ncp->dispatch->def_var_filter(ncid,varid,id,nparams,parms);
}
#endif /* USE_NETCDF4 */

243
libdispatch/dvarinq.c
View File

@ -6,6 +6,13 @@ Research/Unidata. See COPYRIGHT file for more info.
*/
#include "ncdispatch.h"
#ifdef USE_NETCDF4
#include <hdf5.h>
#endif
#ifndef H5Z_FILTER_SZIP
#define H5Z_FILTER_SZIP 4
#endif
/** \name Learning about Variables
@ -122,7 +129,8 @@ nc_inq_var(int ncid, int varid, char *name, nc_type *xtypep,
TRACE(nc_inq_var);
return ncp->dispatch->inq_var_all(ncid, varid, name, xtypep, ndimsp,
dimidsp, nattsp, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
NULL, NULL, NULL, NULL, NULL, NULL,
NULL,NULL,NULL);
}
/**
@ -243,7 +251,6 @@ nc_inq_varnatts(int ncid, int varid, int *nattsp)
nattsp);
}
#ifdef USE_NETCDF4
/** \ingroup variables
Learn the storage and deflate settings for a variable.
@ -294,63 +301,7 @@ nc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
NULL, /*optionsmaskp*/
NULL /*pixelsp*/
);
}
/** \ingroup variables
Learn the szip settings of a variable.
This function returns the szip settings for a variable. NetCDF does
not allow variables to be created with szip (due to license problems
with the szip library), but we do enable read-only access of HDF5
files with szip compression.
This is a wrapper for nc_inq_var_all().
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param options_maskp The szip options mask will be copied to this
pointer. \ref ignored_if_null.
\param pixels_per_blockp The szip pixels per block will be copied
here. \ref ignored_if_null.
\returns ::NC_NOERR No error.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_ENOTVAR Invalid variable ID.
*/
int
nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_szip);
return ncp->dispatch->inq_var_all(
ncid, varid,
NULL, /*name*/
NULL, /*xtypep*/
NULL, /*ndimsp*/
NULL, /*dimidsp*/
NULL, /*nattsp*/
NULL, /*shufflep*/
NULL, /*deflatep*/
NULL, /*deflatelevelp*/
NULL, /*fletcher32p*/
NULL, /*contiguousp*/
NULL, /*chunksizep*/
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
options_maskp, /*optionsmaskp*/
pixels_per_blockp /*pixelsp*/
NULL,NULL,NULL
);
}
@ -397,8 +348,7 @@ nc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
NULL, /*optionsmaskp*/
NULL /*pixelsp*/
NULL, NULL, NULL
);
}
@ -471,7 +421,8 @@ nc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp)
TRACE(nc_inq_var_chunking);
return ncp->dispatch->inq_var_all(ncid, varid, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, storagep,
chunksizesp, NULL, NULL, NULL, NULL, NULL);
chunksizesp, NULL, NULL, NULL,
NULL, NULL, NULL);
}
/** \ingroup variables
@ -520,8 +471,7 @@ nc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
no_fill, /*nofillp*/
fill_valuep, /*fillvaluep*/
NULL, /*endianp*/
NULL, /*optionsmaskp*/
NULL /*pixelsp*/
NULL, NULL, NULL
);
}
@ -569,9 +519,7 @@ nc_inq_var_endian(int ncid, int varid, int *endianp)
NULL, /*nofillp*/
NULL, /*fillvaluep*/
endianp, /*endianp*/
NULL, /*optionsmaskp*/
NULL /*pixelsp*/
);
NULL, NULL, NULL);
}
/*! Return number and list of unlimited dimensions.
@ -599,15 +547,159 @@ This function will return one of the following values.
int
nc_inq_unlimdims(int ncid, int *nunlimdimsp, int *unlimdimidsp)
{
#ifndef USE_NETCDF4
return NC_ENOTNC4;
#else
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_unlimdims);
TRACE(nc_inq_unlimdims);
return ncp->dispatch->inq_unlimdims(ncid, nunlimdimsp,
unlimdimidsp);
#endif
}
#endif /* USE_NETCDF4 */
/** \ingroup variables
Find the filter (if any) associated with a variable.
This is a wrapper for nc_inq_var_all().
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param idp Storage which will get the filter id.
\param nparamsp Storage which will get the number of parameters to the filter
\param params Storage which will get associated parameters. Note
the caller must allocate and free.
\returns ::NC_NOERR No error.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_EFILTER No filter defined.
*/
int
nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsigned int* params)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_filter);
return ncp->dispatch->inq_var_all(
ncid, varid,
NULL, /*name*/
NULL, /*xtypep*/
NULL, /*ndimsp*/
NULL, /*dimidsp*/
NULL, /*nattsp*/
NULL, /*shufflep*/
NULL, /*deflatep*/
NULL, /*deflatelevelp*/
NULL, /*fletcher32p*/
NULL, /*contiguousp*/
NULL, /*chunksizep*/
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
idp, nparamsp, params);
}
/** \ingroup variables
Learn the szip settings of a variable.
Similar to nc_inq_var_deflate.
This function returns the szip settings for a variable.
With the introduction of general filter support,
szip inquiry is converted to use the filter interface.
This is a wrapper for nc_inq_var_filter().
\param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
\param varid Variable ID
\param options_maskp The szip options mask will be copied to this
pointer. \ref ignored_if_null.
\param pixels_per_blockp The szip pixels per block will be copied
here. \ref ignored_if_null.
\returns ::NC_NOERR No error.
\returns ::NC_EBADID Bad ncid.
\returns ::NC_ENOTNC4 Not a netCDF-4 file.
\returns ::NC_ENOTVAR Invalid variable ID.
\returns ::NC_EFILTER Variable is not szip encoded
*/
int
nc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp)
{
NC* ncp;
unsigned int id;
size_t nparams;
unsigned int params[2];
int stat = NC_check_id(ncid,&ncp);
if(stat != NC_NOERR) return stat;
TRACE(nc_inq_var_szip);
/* Verify id and nparams */
stat = ncp->dispatch->inq_var_all(
ncid, varid,
NULL, /*name*/
NULL, /*xtypep*/
NULL, /*ndimsp*/
NULL, /*dimidsp*/
NULL, /*nattsp*/
NULL, /*shufflep*/
NULL, /*deflatep*/
NULL, /*deflatelevelp*/
NULL, /*fletcher32p*/
NULL, /*contiguousp*/
NULL, /*chunksizep*/
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
&id,
&nparams,
NULL
);
if(stat != NC_NOERR) return stat;
if(id != H5Z_FILTER_SZIP || nparams != 2)
return NC_EFILTER; /* not szip or bad # params */
/* Get params */
stat = ncp->dispatch->inq_var_all(
ncid, varid,
NULL, /*name*/
NULL, /*xtypep*/
NULL, /*ndimsp*/
NULL, /*dimidsp*/
NULL, /*nattsp*/
NULL, /*shufflep*/
NULL, /*deflatep*/
NULL, /*deflatelevelp*/
NULL, /*fletcher32p*/
NULL, /*contiguousp*/
NULL, /*chunksizep*/
NULL, /*nofillp*/
NULL, /*fillvaluep*/
NULL, /*endianp*/
&id,
&nparams,
params
);
if(stat != NC_NOERR) return stat;
/* Param[0] should be options_mask, Param[1] should be pixels_per_block */
if(options_maskp) *options_maskp = (int)params[0];
if(pixels_per_blockp) *pixels_per_blockp = (int)params[1];
return NC_NOERR;
}
/*!
@ -629,9 +721,10 @@ Used in libdap2 and libdap4.
@param[out] no_fill Pointer to memory to store whether or not there is a fill value associated with the variable.
@param[out] fill_valuep Pointer to memory to store the fill value (if one exists) for the variable.
@param[out] endiannessp Pointer to memory to store endianness value. One of ::NC_ENDIAN_BIG ::NC_ENDIAN_LITTLE ::NC_ENDIAN_NATIVE
@param[out] options_maskp Pointer to memory to store mask options information.
@param[out] pixels_per_blockp Pointer to memory to store pixels-per-block information for chunked data.
@param[out] idp Pointer to memory to store filter id.
@param[out] nparamsp Pointer to memory to store filter parameter count.
@param[out] params Pointer to vector of unsigned integers into which
to store filter parameters.
\note Expose access to nc_inq_var_all().
\internal
@ -645,7 +738,8 @@ NC_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
NC* ncp;
int stat = NC_check_id(ncid,&ncp);
@ -657,8 +751,7 @@ NC_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
contiguousp, chunksizesp,
no_fill, fill_valuep,
endiannessp,
options_maskp,
pixels_per_blockp);
idp,nparamsp,params);
}
/*! \} */ /* End of named group ...*/

19
libsrc/nc3dispatch.c
View File

@ -35,7 +35,8 @@ static int NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp);
unsigned int* idp, size_t* nparamsp, unsigned int* params
);
static int NC3_var_par_access(int,int,int);
@ -74,6 +75,8 @@ static int NC3_def_var_fletcher32(int,int,int);
static int NC3_def_var_chunking(int,int,int,const size_t*);
static int NC3_def_var_fill(int,int,int,const void*);
static int NC3_def_var_endian(int,int,int);
static int NC3_def_var_filter(int, int, unsigned int, size_t, const unsigned int*);
static int NC3_set_var_chunk_cache(int,int,size_t,size_t,float);
static int NC3_get_var_chunk_cache(int,int,size_t*,size_t*,float*);
#endif /*USE_NETCDF4*/
@ -163,6 +166,7 @@ NC3_def_var_fletcher32,
NC3_def_var_chunking,
NC3_def_var_fill,
NC3_def_var_endian,
NC3_def_var_filter,
NC3_set_var_chunk_cache,
NC3_get_var_chunk_cache,
@ -191,7 +195,8 @@ NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
int stat = NC3_inq_var(ncid,varid,name,xtypep,ndimsp,dimidsp,nattsp);
if(stat) return stat;
@ -201,7 +206,9 @@ NC3_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
if(contiguousp) *contiguousp = NC_CONTIGUOUS;
if(no_fill) *no_fill = 1;
if(endiannessp) return NC_ENOTNC4;
if(options_maskp) return NC_ENOTNC4;
if(idp) return NC_ENOTNC4;
if(nparamsp) return NC_ENOTNC4;
if(params) return NC_ENOTNC4;
return NC_NOERR;
}
@ -511,5 +518,11 @@ NC3_def_var_endian(int ncid, int varid, int endianness)
return NC_ENOTNC4;
}
static int
NC3_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms)
{
return NC_ENOTNC4;
}
#endif /*USE_NETCDF4*/

1
libsrc4/nc4dispatch.c
View File

@ -94,6 +94,7 @@ NC4_def_var_fletcher32,
NC4_def_var_chunking,
NC4_def_var_fill,
NC4_def_var_endian,
NC4_def_var_filter,
NC4_set_var_chunk_cache,
NC4_get_var_chunk_cache,

37
libsrc4/nc4file.c
View File

@ -1441,11 +1441,10 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
int d;
att_iter_info att_info; /* Custom iteration information */
#define CD_NELEMS_ZLIB 1
#define CD_NELEMS_SZIP 4
H5Z_filter_t filter;
int num_filters;
unsigned int cd_values[CD_NELEMS_SZIP];
size_t cd_nelems = CD_NELEMS_SZIP;
unsigned int cd_values_zip[CD_NELEMS_ZLIB];
size_t cd_nelems = CD_NELEMS_ZLIB;
hid_t propid = 0;
H5D_fill_value_t fill_status;
H5D_layout_t layout;
@ -1519,7 +1518,7 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
var->hash = hash_fast(var->name, strlen(var->name));
/* Find out what filters are applied to this HDF5 dataset,
* fletcher32, deflate, and/or shuffle. All other filters are
* ignored. */
* just dumped */
if ((propid = H5Dget_create_plist(datasetid)) < 0)
BAIL(NC_EHDFERR);
@ -1545,7 +1544,7 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
for (f = 0; f < num_filters; f++)
{
if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
cd_values, 0, NULL, NULL)) < 0)
cd_values_zip, 0, NULL, NULL)) < 0)
BAIL(NC_EHDFERR);
switch (filter)
{
@ -1559,21 +1558,25 @@ read_var(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name,
case H5Z_FILTER_DEFLATE:
var->deflate = NC_TRUE;
if (cd_nelems != CD_NELEMS_ZLIB || cd_values[0] > MAX_DEFLATE_LEVEL)
if (cd_nelems != CD_NELEMS_ZLIB || cd_values_zip[0] > MAX_DEFLATE_LEVEL)
BAIL(NC_EHDFERR);
var->deflate_level = cd_values[0];
var->deflate_level = cd_values_zip[0];
break;
case H5Z_FILTER_SZIP:
var->szip = NC_TRUE;
if (cd_nelems != CD_NELEMS_SZIP)
BAIL(NC_EHDFERR);
var->options_mask = cd_values[0];
var->pixels_per_block = cd_values[1];
break;
default:
LOG((1, "Yikes! Unknown filter type found on dataset!"));
default:
var->filterid = filter;
var->nparams = cd_nelems;
if(cd_nelems == 0)
var->params = NULL;
else {
/* We have to re-read the parameters based on actual nparams */
var->params = (unsigned int*)calloc(1,sizeof(unsigned int)*var->nparams);
if(var->params == NULL)
BAIL(NC_ENOMEM);
if((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
var->params, 0, NULL, NULL)) < 0)
BAIL(NC_EHDFERR);
}
break;
}
}

30
libsrc4/nc4hdf.c
View File

@ -1528,21 +1528,31 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
}
/* If the user wants to shuffle the data, set that up now. */
if (var->shuffle)
if (var->shuffle) {
if (H5Pset_shuffle(plistid) < 0)
BAIL(NC_EHDFERR);
}
/* If the user wants to deflate the data, set that up now. */
if (var->deflate)
if (var->deflate) {
if (H5Pset_deflate(plistid, var->deflate_level) < 0)
BAIL(NC_EHDFERR);
/* Szip? NO! We don't want anyone to produce szipped netCDF files! */
/* #ifdef USE_SZIP */
/* if (var->options_mask) */
/* if (H5Pset_szip(plistid, var->options_mask, var->bits_per_pixel) < 0) */
/* BAIL(NC_EHDFERR); */
/* #endif */
} else if(var->filterid) {
/* Handle szip case here */
if(var->filterid == H5Z_FILTER_SZIP) {
int options_mask;
int bits_per_pixel;
if(var->nparams != 2)
BAIL(NC_EFILTER);
options_mask = (int)var->params[0];
bits_per_pixel = (int)var->params[1];
if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
BAIL(NC_EFILTER);
} else {
if(H5Pset_filter(plistid, var->filterid, H5Z_FLAG_MANDATORY, var->nparams, var->params) < 0)
BAIL(NC_EFILTER);
}
}
/* If the user wants to fletcher error correcton, set that up now. */
if (var->fletcher32)
@ -1569,7 +1579,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid
* has not specified chunksizes, use contiguous variable for
* better performance. */
if(!var->shuffle && !var->deflate && !var->options_mask &&
if(!var->shuffle && !var->deflate &&
!var->fletcher32 && (var->chunksizes == NULL || !var->chunksizes[0])) {
#ifdef USE_HDF4
NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;

4
libsrc4/nc4internal.c
View File

@ -1071,6 +1071,10 @@ nc4_var_del(NC_VAR_INFO_T *var)
if (var->dimscale_attached)
free(var->dimscale_attached);
/* Release parameter information. */
if (var->params)
free(var->params);
/* Delete the var. */
free(var);

87
libsrc4/nc4var.c
View File

@ -664,7 +664,8 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params
)
{
NC *nc;
NC_GRP_INFO_T *grp;
@ -742,13 +743,13 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
*shufflep = (int)var->shuffle;
if (fletcher32p)
*fletcher32p = (int)var->fletcher32;
/* NOTE: No interface for returning szip flag currently (but it should never
* be set).
*/
if (options_maskp)
*options_maskp = var->options_mask;
if (pixels_per_blockp)
*pixels_per_blockp = var->pixels_per_block;
if (idp)
*idp = var->filterid;
if (nparamsp)
*nparamsp = (var->params == NULL ? 0 : var->nparams);
if (params && var->params != NULL)
memcpy(params,var->params,var->nparams*sizeof(unsigned int));
/* Fill value stuff. */
if (no_fill)
@ -1064,7 +1065,7 @@ nc_inq_var_chunking_ints(int ncid, int varid, int *contiguousp, int *chunksizesp
retval = NC4_inq_var_all(ncid, varid, NULL, NULL, NULL, NULL, NULL,
NULL, NULL, NULL, NULL, contiguousp, cs, NULL,
NULL, NULL, NULL, NULL);
NULL, NULL, NULL, NULL, NULL);
/* Copy from size_t array. */
if (*contiguousp == NC_CHUNKED)
@ -1143,6 +1144,74 @@ NC4_def_var_endian(int ncid, int varid, int endianness)
NULL, NULL, NULL, &endianness);
}
int
NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int* parms)
{
int retval = NC_NOERR;
NC *nc;
NC_GRP_INFO_T *grp;
NC_HDF5_FILE_INFO_T *h5;
NC_VAR_INFO_T *var;
NC_DIM_INFO_T *dim;
LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid));
/* Find info for this file and group, and set pointer to each. */
if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
return retval;
assert(nc && grp && h5);
/* Find the var. */
if (varid < 0 || varid >= grp->vars.nelems)
return NC_ENOTVAR;
var = grp->vars.value[varid];
if (!var) return NC_ENOTVAR;
assert(var->varid == varid);
/* Can't turn on parallel and filters */
if (nc->mode & (NC_MPIIO | NC_MPIPOSIX)) {
return NC_EINVAL;
}
/* If the HDF5 dataset has already been created, then it is too
* late to set all the extra stuff. */
if (var->created)
return NC_ELATEDEF;
#ifdef HAVE_H5Z_SZIP
if(id == H5Z_FILTER_SZIP) {
if(nparams != 2)
return NC_EFILTER; /* incorrect no. of parameters */
}
#else /*!HAVE_H5Z_SZIP*/
if(id == H5Z_FILTER_SZIP)
return NC_EFILTER; /* Not allowed */
#endif
#if 0
{
unsigned int fcfg = 0;
herr_t herr = H5Zget_filter_info(id,&fcfg);
if(herr < 0)
return NC_EFILTER;
if((H5Z_FILTER_CONFIG_ENCODE_ENABLED & fcfg) == 0
|| (H5Z_FILTER_CONFIG_DECODE_ENABLED & fcfg) == 0)
return NC_EFILTER;
}
#endif /*0*/
var->filterid = id;
var->nparams = nparams;
var->params = NULL;
if(parms != NULL) {
var->params = (unsigned int*)calloc(nparams,sizeof(unsigned int));
if(var->params == NULL) return NC_ENOMEM;
memcpy(var->params,parms,sizeof(unsigned int)*var->nparams);
}
return NC_NOERR;
}
/* Get var id from name. */
int
NC4_inq_varid(int ncid, const char *name, int *varidp)

2
libsrcp/ncpdispatch.c
View File

@ -1157,7 +1157,7 @@ NCP_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
int *shufflep, int *deflatep, int *deflate_levelp,
int *fletcher32p, int *contiguousp, size_t *chunksizesp,
int *no_fill, void *fill_valuep, int *endiannessp,
int *options_maskp, int *pixels_per_blockp)
unsigned int* idp, size_t* nparamsp, unsigned int* params)
{
int status;
NC* nc;

16
nc_test4/CMakeLists.txt
View File

@ -9,7 +9,8 @@ SET(NC4_TESTS tst_dims tst_dims2 tst_dims3 tst_files tst_files4
cdm_sea_soundings tst_vl tst_atts1 tst_atts2 tst_vars2 tst_files5
tst_files6 tst_sync tst_h_strbug tst_h_refs tst_h_scalar tst_rename
tst_h5_endians tst_atts_string_rewrite tst_put_vars_two_unlim_dim
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash)
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_filterparser)
# Note, renamegroup needs to be compiled before run_grp_rename
@ -45,6 +46,14 @@ IF(USE_HDF4)
SET(NC4_TESTS ${NC4_TESTS} tst_interops2)
ENDIF()
IF(USE_SZIP)
BUILD_BIN_TEST(test_szip)
BUILD_BIN_TEST(h5testszip)
IF(BUILD_UTILITIES)
add_sh_test(nc_test4 tst_szip)
ENDIF()
ENDIF()
IF(BUILD_BENCHMARKS)
add_sh_test(nc_test4 run_bm_test1)
add_sh_test(nc_test4 run_bm_elena)
@ -60,7 +69,7 @@ IF(BUILD_BENCHMARKS)
ENDIF()
# Copy some test files from current source dir to out-of-tree build dir.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.hdf4)
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.hdf4 ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
IF(MSVC)
FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
@ -83,7 +92,6 @@ IF(USE_HDF4_FILE_TESTS AND NOT MSVC)
add_sh_test(nc_test4 tst_hdf4_read_var)
ENDIF()
IF(TEST_PARALLEL4)
build_bin_test(tst_mpi_parallel)
build_bin_test(tst_parallel)
@ -94,3 +102,5 @@ IF(TEST_PARALLEL4)
build_bin_test(tst_simplerw_coll_r)
add_sh_test(nc_test4 run_par_test)
ENDIF()
ADD_SUBDIRECTORY(filter_test)

25
nc_test4/Make0
View File

@ -1,23 +1,25 @@
# Test c output
T=tst_misc
T=tst_filterparser
#CMD=valgrind --leak-check=full
CMD=gdb --args
#PAR=1
SZIP=1
CFLAGS=-Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm
ifdef PAR
CC=mpicc
#CC=/usr/local/bin/mpicc
LDFLAGS=../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lpnetcdf -lmpich -lm
LDFLAGS += -lmpich
else
CC=gcc
#LDFLAGS=../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -lm -lcurl
LDFLAGS=../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lm -lcurl
endif
# cd .. ; ${MAKE} all
ifdef SZIP
LDFLAGS += -lsz -laec
endif
LLP=/usr/local/lib:${LD_LIBRARY_PATH}
@ -27,7 +29,14 @@ all:: cmp
cmp::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}
cpp::
${CC} -E ${CFLAGS} ${T}.c > ${T}.txt
H5=h5testszip
EXT=testszip.nc
h5::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o h5 ${CFLAGS} ${H5}.c ${SRC} ${LDFLAGS}; \
${CMD} ./h5 ${EXT}

25
nc_test4/Makefile.am
View File

@ -20,9 +20,9 @@ tst_interops6 tst_enums tst_coords tst_coords2 tst_coords3 tst_vars3 \
tst_vars4 tst_chunks tst_chunks2 tst_utf8 tst_fills tst_fills2 \
tst_fillbug tst_xplatform tst_xplatform2 tst_endian_fill tst_atts \
t_type cdm_sea_soundings tst_camrun tst_vl tst_atts1 tst_atts2 \
tst_vars2 tst_files5 tst_files6 tst_sync tst_h_scalar tst_rename \
tst_h5_endians tst_atts_string_rewrite tst_hdf5_file_compat \
tst_fill_attr_vanish tst_rehash
tst_vars2 tst_files5 tst_files6 tst_sync \
tst_h_scalar tst_rename tst_h5_endians tst_atts_string_rewrite \
tst_hdf5_file_compat tst_fill_attr_vanish tst_rehash tst_filterparser
# Temporary I hope
if !ISCYGWIN
@ -106,6 +106,15 @@ endif # USE_HDF4_FILE_TESTS
#-lhdf5 -lz
endif # USE_HDF4
# Szip Tests (requires ncdump)
if USE_SZIP
if BUILD_UTILITIES
check_PROGRAMS += test_szip h5testszip
TESTS += tst_szip.sh
endif
endif
# This will run a bunch of the test programs with valgrind, the memory
# checking tool. (Valgrind must be present for this to work.)
if USE_VALGRIND_TESTS
@ -135,8 +144,8 @@ run_grp_rename.sh tst_formatx_hdf4.sh run_chunk_hdf4.sh \
tst_h5_endians.c tst_h4_lendian.c tst_atts_string_rewrite.c \
tst_put_vars_two_unlim_dim.c tst_empty_vlen_unlim.c run_empty_vlen_test.sh \
ref_hdf5_compat1.nc ref_hdf5_compat2.nc ref_hdf5_compat3.nc tst_misc.sh \
tdset.h5 tst_hdf4_read_var.sh ref_contiguous.hdf4 ref_chunked.hdf4
tdset.h5 tst_hdf4_read_var.sh ref_contiguous.hdf4 ref_chunked.hdf4 \
tst_szip.sh ref_szip.h5 ref_szip.cdl
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt \
@ -148,7 +157,8 @@ usi_01.* thetau_01.* tst_*.nc tst_*.h5 \
tst_grp_rename.cdl tst_grp_rename.nc tst_grp_rename.dmp ref_grp_rename.cdl \
foo1.nc tst_interops2.h4 tst_h5_endians.nc tst_h4_lendian.h4 test.nc \
tst_atts_string_rewrite.nc tst_empty_vlen_unlim.nc tst_empty_vlen_lim.nc \
tst_parallel4_simplerw_coll.nc tst_fill_attr_vanish.nc tst_rehash.nc
tst_parallel4_simplerw_coll.nc tst_fill_attr_vanish.nc tst_rehash.nc \
testszip.nc test.h5 szip_dump.cdl
if USE_HDF4_FILE_TESTS
DISTCLEANFILES = AMSR_E_L2_Rain_V10_200905312326_A.hdf \
@ -157,5 +167,6 @@ MYD29.A2009152.0000.005.2009153124331.hdf \
MYD29.A2002185.0000.005.2007160150627.hdf \
MOD29.A2000055.0005.005.2006267200024.hdf
endif # HDF4_FILE_TESTS
SUBDIRS=filter_test

34
nc_test4/filter_test/CMakeLists.txt Normal file
View File

@ -0,0 +1,34 @@
# This cmake file is here as a placeholder.
# It cannot yet be used to run the filter test.
if(ENABLE_FILTER_TEST)
SET(BZIP2SRC blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c)
ADD_EXECUTABLE(fake fake.c H5Zbzip2.c ${BZIP2SRC})
SET(BZIP2OBJ blocksort.o huffman.o crctable.o randtable.o compress.o decompress.o bzlib.o)
SET(LIBNAME "lib${FILTERNAME}.so")
SET(bzip__LIBS H5Zbzip2.o ${BZIP2OBJ})
FOREACH(LIBS ${bzip_LIBS})
SET(LARGS ${LARGS} $<TARGET_OBJECTS:${LIBS}>)
ENDFOREACH()
ADD_LIBRARY(bzip2 ${LARGS} )
SET(TLL_LIBS "")
SET(TLL_LIBS ${TLL_LIBS} ${HDF5_hdf5_hl_LIBRARY} ${HDF5_hdf5_LIBRARY} ${ZLIB_LIBRARY})
TARGET_LINK_LIBRARIES(bzip2 ${TLL_LIBS})
SET_TARGET_PROPERTIES(bzip2 PROPERTIES LINK_FLAGS -shared)
build_bin_test(test_filter)
IF(BUILD_UTILITIES)
ADD_SH_TEST(nc_test4 tst_filter)
ENDIF(BUILD_UTILITIES)
ENDIF(ENABLE_FILTER_TEST)
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

171
nc_test4/filter_test/H5Zbzip2.c Normal file
View File

@ -0,0 +1,171 @@
#include <sys/types.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "h5bzip2.h"
const H5Z_class2_t H5Z_BZIP2[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)H5Z_FILTER_BZIP2, /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"bzip2", /* Filter name for debugging */
NULL, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_bzip2, /* The actual filter function */
}};
/* External Discovery Functions */
H5PL_type_t
H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
const void*
H5PLget_plugin_info(void)
{
return H5Z_BZIP2;
}
size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
size_t H5Z_filter_bzip2(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
char *outbuf = NULL;
size_t outbuflen, outdatalen;
int ret;
if (flags & H5Z_FLAG_REVERSE) {
/** Decompress data.
**
** This process is troublesome since the size of uncompressed data
** is unknown, so the low-level interface must be used.
** Data is decompressed to the output buffer (which is sized
** for the average case); if it gets full, its size is doubled
** and decompression continues. This avoids repeatedly trying to
** decompress the whole block, which could be really inefficient.
**/
bz_stream stream;
char *newbuf = NULL;
size_t newbuflen;
/* Prepare the output buffer. */
outbuflen = nbytes * 3 + 1; /* average bzip2 compression ratio is 3:1 */
outbuf = malloc(outbuflen);
if (outbuf == NULL) {
fprintf(stderr, "memory allocation failed for bzip2 decompression\n");
goto cleanupAndFail;
}
/* Use standard malloc()/free() for internal memory handling. */
stream.bzalloc = NULL;
stream.bzfree = NULL;
stream.opaque = NULL;
/* Start decompression. */
ret = BZ2_bzDecompressInit(&stream, 0, 0);
if (ret != BZ_OK) {
fprintf(stderr, "bzip2 decompression start failed with error %d\n", ret);
goto cleanupAndFail;
}
/* Feed data to the decompression process and get decompressed data. */
stream.next_out = outbuf;
stream.avail_out = outbuflen;
stream.next_in = *buf;
stream.avail_in = nbytes;
do {
ret = BZ2_bzDecompress(&stream);
if (ret < 0) {
fprintf(stderr, "BUG: bzip2 decompression failed with error %d\n", ret);
goto cleanupAndFail;
}
if (ret != BZ_STREAM_END && stream.avail_out == 0) {
/* Grow the output buffer. */
newbuflen = outbuflen * 2;
newbuf = realloc(outbuf, newbuflen);
if (newbuf == NULL) {
fprintf(stderr, "memory allocation failed for bzip2 decompression\n");
goto cleanupAndFail;
}
stream.next_out = newbuf + outbuflen; /* half the new buffer behind */
stream.avail_out = outbuflen; /* half the new buffer ahead */
outbuf = newbuf;
outbuflen = newbuflen;
}
} while (ret != BZ_STREAM_END);
/* End compression. */
outdatalen = stream.total_out_lo32;
ret = BZ2_bzDecompressEnd(&stream);
if (ret != BZ_OK) {
fprintf(stderr, "bzip2 compression end failed with error %d\n", ret);
goto cleanupAndFail;
}
} else {
/** Compress data.
**
** This is quite simple, since the size of compressed data in the worst
** case is known and it is not much bigger than the size of uncompressed
** data. This allows us to use the simplified one-shot interface to
** compression.
**/
unsigned int odatalen; /* maybe not the same size as outdatalen */
int blockSize100k = 9;
/* Get compression block size if present. */
if (cd_nelmts > 0) {
blockSize100k = cd_values[0];
if (blockSize100k < 1 || blockSize100k > 9) {
fprintf(stderr, "invalid compression block size: %d\n", blockSize100k);
goto cleanupAndFail;
}
}
/* Prepare the output buffer. */
outbuflen = nbytes + nbytes / 100 + 600; /* worst case (bzip2 docs) */
outbuf = malloc(outbuflen);
if (outbuf == NULL) {
fprintf(stderr, "memory allocation failed for bzip2 compression\n");
goto cleanupAndFail;
}
/* Compress data. */
odatalen = outbuflen;
ret = BZ2_bzBuffToBuffCompress(outbuf, &odatalen, *buf, nbytes,
blockSize100k, 0, 0);
outdatalen = odatalen;
if (ret != BZ_OK) {
fprintf(stderr, "bzip2 compression failed with error %d\n", ret);
goto cleanupAndFail;
}
}
/* Always replace the input buffer with the output buffer. */
free(*buf);
*buf = outbuf;
*buf_size = outbuflen;
return outdatalen;
cleanupAndFail:
if (outbuf)
free(outbuf);
return 0;
}

104
nc_test4/filter_test/H5Ztemplate.c Normal file
View File

@ -0,0 +1,104 @@
#include <sys/types.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "xxxx.h"
/*
Provide a textual template (not a C++ template)
from which one can construct a new filter.
The filter "name is marked with "XXXX" or "xxxx"
*/
const H5Z_class2_t H5Z_XXXX[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)H5Z_FILTER_XXXX, /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"xxxx", /* Filter name for debugging */
NULL, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_xxxx, /* The actual filter function */
}};
/* External Discovery Functions */
H5PL_type_t
H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
const void*
H5PLget_plugin_info(void)
{
return H5Z_XXXX;
}
size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
size_t H5Z_filter_xxxx(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
char *outbuf = NULL;
size_t outbuflen, outdatalen;
int ret;
if (flags & H5Z_FLAG_REVERSE) {
/** Decompress data.
**
** This process is troublesome since the size of uncompressed data
** is unknown, so the low-level interface must be used.
** Data is decompressed to the output buffer (which is sized
** for the average case); if it gets full, its size is doubled
** and decompression continues. This avoids repeatedly trying to
** decompress the whole block, which could be really inefficient.
**/
char *newbuf = NULL;
size_t newbuflen;
} else {
/** Compress data.
**
** This is quite simple, since the size of compressed data in the worst
** case is known and it is not much bigger than the size of uncompressed
** data. This allows us to use the simplified one-shot interface to
** compression.
**/
unsigned int odatalen; /* maybe not the same size as outdatalen */
/* Prepare the output buffer. */
outbuflen = M; /* worst case */
outbuf = malloc(outbuflen);
if (outbuf == NULL) {
fprintf(stderr, "memory allocation failed for xxxx compression\n");
goto cleanupAndFail;
}
/* Compress data. */
}
/* Always replace the input buffer with the output buffer. */
free(*buf);
*buf = outbuf;
*buf_size = outbuflen;
return outdatalen;
cleanupAndFail:
if (outbuf)
free(outbuf);
return 0;
}

207
nc_test4/filter_test/H5Ztest.c Normal file
View File

@ -0,0 +1,207 @@
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <sys/types.h>
#include <hdf5.h>
/* Older versions of the hdf library may define H5PL_type_t here */
#include <H5PLextern.h>
#include "h5test.h"
#undef DEBUG
static int paramcheck(size_t nparams, const unsigned int* params);
static void byteswap8(unsigned char* mem);
static void mismatch(size_t i, const char* which);
const H5Z_class2_t H5Z_TEST[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)(H5Z_FILTER_TEST), /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"test", /* Filter name for debugging */
NULL, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_test, /* The actual filter function */
}};
/* External Discovery Functions */
H5PL_type_t
H5PLget_plugin_type(void)
{
return H5PL_TYPE_FILTER;
}
const void*
H5PLget_plugin_info(void)
{
return H5Z_TEST;
}
/*
This filter does some verification
that the parameters passed to the filter
are correct. Specifically, that endian-ness
is correct. As a filter, it is the identify
function, passing input to output unchanged.
Test cases format:
1.The first param is the test index i.e. which test to execute.
2. The remaining parameters are those for the test chosen in #1
*/
size_t
H5Z_filter_test(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
int ret;
void* newbuf;
unsigned int testcase = 0;
if(cd_nelmts == 0)
goto fail;
testcase = cd_values[0];
if(testcase == TC_ENDIAN) {
if(!paramcheck(cd_nelmts,cd_values))
goto fail;
}
if (flags & H5Z_FLAG_REVERSE) {
/* Replace buffer */
newbuf = malloc(*buf_size);
if(newbuf == NULL) abort();
memcpy(newbuf,*buf,*buf_size);
*buf = newbuf;
} else {
/* Replace buffer */
newbuf = malloc(*buf_size);
if(newbuf == NULL) abort();
memcpy(newbuf,*buf,*buf_size);
*buf = newbuf;
}
return *buf_size;
fail:
return 0;
}
static int
paramcheck(size_t nparams, const unsigned int* params)
{
size_t i;
/* Test endianness of this machine */
const unsigned char b[4] = {0x0,0x0,0x0,0x1}; /* value 1 in big-endian*/
int endianness = (1 == *(unsigned int*)b); /* 1=>big 0=>little*/
if(nparams != 14) {
fprintf(stderr,"Too few parameters: need=16 sent=%ld\n",nparams);
return 0;
}
for(i=0;i<nparams;i++) {
switch (i) {
case 0: break; /* this is the testcase # */
case 1: if(((signed char)-17) != (signed int)(params[i]))
{mismatch(i,"signed byte"); return 0; };
break;
case 2: if(((unsigned char)23) != (unsigned int)(params[i]))
{mismatch(i,"unsigned byte"); return 0; };
break;
case 3: if(((signed short)-25) != (signed int)(params[i]))
{mismatch(i,"signed short"); return 0; };
break;
case 4: if(((unsigned short)27) != (unsigned int)(params[i]))
{mismatch(i,"unsigned short"); return 0; };
break;
case 5: if(77 != (signed int)(params[i]))
{mismatch(i,"signed int"); return 0; };
break;
case 6: if(93u != (unsigned int)(params[i]))
{mismatch(i,"unsigned int"); return 0; };
break;
case 7: if(789.0f != *(float*)(&params[i]))
{mismatch(i,"float"); return 0; };
break;
case 8: {/*double*/
double x = *(double*)&params[i];
i++; /* takes two parameters */
if(endianness == 1)
byteswap8((unsigned char*)&x);
if(12345678.12345678d != x) {
mismatch(i,"double");
return 0;
}
}; break;
case 10: {/*signed long long*/
signed long long x = *(signed long long*)&params[i];
i++; /* takes two parameters */
if(endianness == 1)
byteswap8((unsigned char*)&x);
if(-9223372036854775807L != x) {
mismatch(i,"signed long long");
return 0;
}
}; break;
case 12: {/*unsigned long long*/
unsigned long long x = *(unsigned long long*)&params[i];
i++; /* takes two parameters */
if(endianness == 1)
byteswap8((unsigned char*)&x);
if(18446744073709551615UL != x) {
mismatch(i,"unsigned long long");
return 0;
}
}; break;
default:
mismatch(i,"unexpected parameter");
return 0;
break;
}
}
#ifdef DEBUG
{
size_t i;
fprintf(stderr,"endianness=%d nparams=%d params=\n",endianness,nparams);
for(i=0;i<nparams;i++) {
fprintf(stderr,"[%d] %ud %d %f\n", (unsigned int)i, params[i],(signed int)params[i],*(float*)&params[i]);
}
fflush(stderr);
}
#endif
}
static void
byteswap8(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
static void
mismatch(size_t i, const char* which)
{
fprintf(stderr,"mismatch: [%ld] %s\n",i,which);
fflush(stderr);
}

38
nc_test4/filter_test/Make0 Normal file
View File

@ -0,0 +1,38 @@
# Test c output
T=test_misc
#CMD=valgrind --leak-check=full
CMD=gdb --args
EXECDIR=../..
SRCDIR=../..
#PAR=1
SZIP=1
CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I${SRCDIR} -I${SRCDIR}/include
LDFLAGS = ${EXECDIR}/liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm
ifdef PAR
CC=mpicc
LDFLAGS += -lmpich
else
CC=gcc
endif
ifdef SZIP
LDFLAGS += -lsz -laec
endif
LLP=/usr/local/lib:${LD_LIBRARY_PATH}
all:: cmp
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CMD} ./t
cmp::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}
cpp::
${CC} -E ${CFLAGS} ${T}.c > ${T}.txt

77
nc_test4/filter_test/Makefile.am Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
# See the COPYRIGHT file for more information.
# Put together AM_CPPFLAGS and AM_LDFLAGS.
include $(top_srcdir)/lib_flags.am
FILTERNAME=bzip2
BZIP2HDRS=bzlib.h bzlib_private.h
BZIP2SRC=\
blocksort.c \
huffman.c \
crctable.c \
randtable.c \
compress.c \
decompress.c \
bzlib.c
PLUGINSRC=H5Zbzip2.c
PLUGINHDRS=h5bzip2.h
#bin_PROGRAMS = fake
#fake_SOURCES = fake.c H5Zbzip2.c ${BZIP2SRC}
# Link to our assembled library.
test_filter_LDFLAGS = ${top_builddir}/liblib/libnetcdf.la
test_filter_SOURCE = test_filter.c
test_misc_LDFLAGS = ${top_builddir}/liblib/libnetcdf.la
test_misc_SOURCE = test_misc.c
BZIP2OBJ=${BZIP2SRC:%.o=%.c}
if ISCYGWIN
LIBNAME=cyg${FILTERNAME}.dll
TESTLIB=cygtest.dll
else
LIBNAME=lib${FILTERNAME}.so
TESTLIB=libtest.so
endif
EXTRA_DIST=${PLUGINSRC} ${BZIP2SRC} ${PLUGINHDRS} ${BZIP2HDRS} \
tst_filter.sh bzip2.cdl unfiltered.cdl filtered.cdl ref_bzip2.c \
H5Ztemplate.c h5test.h H5Ztest.c \
CMakeLists.txt
CLEANFILES = ${LIBNAME} ${TESTLIB} *.nc *.dump tmp
if ENABLE_FILTER_TEST
check_PROGRAMS = test_filter test_misc
TESTS = tst_filter.sh
# In order to create the bzip shared object,
# we provide an explicit target for it that
# must be invoked before 'make check'
# You may need to change this if HDF5 is installed elsewhere
HDF5LIBDIR = /usr/local/lib
# Ditto for zlib
ZLIBDIR = ${HDF5LIBDIR}
BUILT_SOURCES = ${LIBNAME} ${TESTLIB}
${LIBNAME}: ${PLUGINSRC:${srcdir}%.c=%.c} ${BZIP2SRC:${srcdir}%.c=%.c}
SDIR=${srcdir} ;\
for f in ${PLUGINSRC} ${BZIP2SRC} ; do \
DLLSRC="$${DLLSRC} $${SDIR}/$$f" ; \
done ; \
gcc -g -O0 -shared -o ${LIBNAME} $${DLLSRC} \
-L${HDF5LIBDIR} -lhdf5_hl -lhdf5 -L${ZLIBDIR} -lz
${TESTLIB}: ${srcdir}/H5Ztest.c
SDIR=${srcdir} ;\
DLLSRC="$${DLLSRC} $${SDIR}/H5Ztest.c" ; \
gcc -g -O0 -shared -o ${TESTLIB} $${DLLSRC} \
-L${HDF5LIBDIR} -lhdf5_hl -lhdf5 -L${ZLIBDIR} -lz
endif

1094
nc_test4/filter_test/blocksort.c Normal file
View File

File diff suppressed because it is too large Load Diff

83
nc_test4/filter_test/bzip2.cdl Normal file
View File

@ -0,0 +1,83 @@
netcdf bzip2 {
dimensions:
dim0 = 4 ;
dim1 = 4 ;
dim2 = 4 ;
dim3 = 4 ;
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
var:_Filter = "307,9" ;
var:_NoFill = "true" ;
// global attributes:
:_Format = "netCDF-4" ;
data:
var =
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23,
24, 25, 26, 27,
28, 29, 30, 31,
32, 33, 34, 35,
36, 37, 38, 39,
40, 41, 42, 43,
44, 45, 46, 47,
48, 49, 50, 51,
52, 53, 54, 55,
56, 57, 58, 59,
60, 61, 62, 63,
64, 65, 66, 67,
68, 69, 70, 71,
72, 73, 74, 75,
76, 77, 78, 79,
80, 81, 82, 83,
84, 85, 86, 87,
88, 89, 90, 91,
92, 93, 94, 95,
96, 97, 98, 99,
100, 101, 102, 103,
104, 105, 106, 107,
108, 109, 110, 111,
112, 113, 114, 115,
116, 117, 118, 119,
120, 121, 122, 123,
124, 125, 126, 127,
128, 129, 130, 131,
132, 133, 134, 135,
136, 137, 138, 139,
140, 141, 142, 143,
144, 145, 146, 147,
148, 149, 150, 151,
152, 153, 154, 155,
156, 157, 158, 159,
160, 161, 162, 163,
164, 165, 166, 167,
168, 169, 170, 171,
172, 173, 174, 175,
176, 177, 178, 179,
180, 181, 182, 183,
184, 185, 186, 187,
188, 189, 190, 191,
192, 193, 194, 195,
196, 197, 198, 199,
200, 201, 202, 203,
204, 205, 206, 207,
208, 209, 210, 211,
212, 213, 214, 215,
216, 217, 218, 219,
220, 221, 222, 223,
224, 225, 226, 227,
228, 229, 230, 231,
232, 233, 234, 235,
236, 237, 238, 239,
240, 241, 242, 243,
244, 245, 246, 247,
248, 249, 250, 251,
252, 253, 254, 255 ;
}

1572
nc_test4/filter_test/bzlib.c Normal file
View File

File diff suppressed because it is too large Load Diff

282
nc_test4/filter_test/bzlib.h Normal file
View File

@ -0,0 +1,282 @@
/*-------------------------------------------------------------*/
/*--- Public header file for the library. ---*/
/*--- bzlib.h ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#ifndef _BZLIB_H
#define _BZLIB_H
#ifdef __cplusplus
extern "C" {
#endif
#define BZ_RUN 0
#define BZ_FLUSH 1
#define BZ_FINISH 2
#define BZ_OK 0
#define BZ_RUN_OK 1
#define BZ_FLUSH_OK 2
#define BZ_FINISH_OK 3
#define BZ_STREAM_END 4
#define BZ_SEQUENCE_ERROR (-1)
#define BZ_PARAM_ERROR (-2)
#define BZ_MEM_ERROR (-3)
#define BZ_DATA_ERROR (-4)
#define BZ_DATA_ERROR_MAGIC (-5)
#define BZ_IO_ERROR (-6)
#define BZ_UNEXPECTED_EOF (-7)
#define BZ_OUTBUFF_FULL (-8)
#define BZ_CONFIG_ERROR (-9)
typedef
struct {
char *next_in;
unsigned int avail_in;
unsigned int total_in_lo32;
unsigned int total_in_hi32;
char *next_out;
unsigned int avail_out;
unsigned int total_out_lo32;
unsigned int total_out_hi32;
void *state;
void *(*bzalloc)(void *,int,int);
void (*bzfree)(void *,void *);
void *opaque;
}
bz_stream;
#ifndef BZ_IMPORT
#define BZ_EXPORT
#endif
#ifndef BZ_NO_STDIO
/* Need a definitition for FILE */
#include <stdio.h>
#endif
#ifdef _WIN32
# include <windows.h>
# ifdef small
/* windows.h define small to char */
# undef small
# endif
# ifdef BZ_EXPORT
# define BZ_API(func) WINAPI func
# define BZ_EXTERN extern
# else
/* import windows dll dynamically */
# define BZ_API(func) (WINAPI * func)
# define BZ_EXTERN
# endif
#else
# define BZ_API(func) func
# define BZ_EXTERN extern
#endif
/*-- Core (low-level) library functions --*/
BZ_EXTERN int BZ_API(BZ2_bzCompressInit) (
bz_stream* strm,
int blockSize100k,
int verbosity,
int workFactor
);
BZ_EXTERN int BZ_API(BZ2_bzCompress) (
bz_stream* strm,
int action
);
BZ_EXTERN int BZ_API(BZ2_bzCompressEnd) (
bz_stream* strm
);
BZ_EXTERN int BZ_API(BZ2_bzDecompressInit) (
bz_stream *strm,
int verbosity,
int small
);
BZ_EXTERN int BZ_API(BZ2_bzDecompress) (
bz_stream* strm
);
BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) (
bz_stream *strm
);
/*-- High(er) level library functions --*/
#ifndef BZ_NO_STDIO
#define BZ_MAX_UNUSED 5000
typedef void BZFILE;
BZ_EXTERN BZFILE* BZ_API(BZ2_bzReadOpen) (
int* bzerror,
FILE* f,
int verbosity,
int small,
void* unused,
int nUnused
);
BZ_EXTERN void BZ_API(BZ2_bzReadClose) (
int* bzerror,
BZFILE* b
);
BZ_EXTERN void BZ_API(BZ2_bzReadGetUnused) (
int* bzerror,
BZFILE* b,
void** unused,
int* nUnused
);
BZ_EXTERN int BZ_API(BZ2_bzRead) (
int* bzerror,
BZFILE* b,
void* buf,
int len
);
BZ_EXTERN BZFILE* BZ_API(BZ2_bzWriteOpen) (
int* bzerror,
FILE* f,
int blockSize100k,
int verbosity,
int workFactor
);
BZ_EXTERN void BZ_API(BZ2_bzWrite) (
int* bzerror,
BZFILE* b,
void* buf,
int len
);
BZ_EXTERN void BZ_API(BZ2_bzWriteClose) (
int* bzerror,
BZFILE* b,
int abandon,
unsigned int* nbytes_in,
unsigned int* nbytes_out
);
BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) (
int* bzerror,
BZFILE* b,
int abandon,
unsigned int* nbytes_in_lo32,
unsigned int* nbytes_in_hi32,
unsigned int* nbytes_out_lo32,
unsigned int* nbytes_out_hi32
);
#endif
/*-- Utility functions --*/
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffCompress) (
char* dest,
unsigned int* destLen,
char* source,
unsigned int sourceLen,
int blockSize100k,
int verbosity,
int workFactor
);
BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffDecompress) (
char* dest,
unsigned int* destLen,
char* source,
unsigned int sourceLen,
int small,
int verbosity
);
/*--
Code contributed by Yoshioka Tsuneo (tsuneo@rr.iij4u.or.jp)
to support better zlib compatibility.
This code is not _officially_ part of libbzip2 (yet);
I haven't tested it, documented it, or considered the
threading-safeness of it.
If this code breaks, please contact both Yoshioka and me.
--*/
BZ_EXTERN const char * BZ_API(BZ2_bzlibVersion) (
void
);
#ifndef BZ_NO_STDIO
BZ_EXTERN BZFILE * BZ_API(BZ2_bzopen) (
const char *path,
const char *mode
);
BZ_EXTERN BZFILE * BZ_API(BZ2_bzdopen) (
int fd,
const char *mode
);
BZ_EXTERN int BZ_API(BZ2_bzread) (
BZFILE* b,
void* buf,
int len
);
BZ_EXTERN int BZ_API(BZ2_bzwrite) (
BZFILE* b,
void* buf,
int len
);
BZ_EXTERN int BZ_API(BZ2_bzflush) (
BZFILE* b
);
BZ_EXTERN void BZ_API(BZ2_bzclose) (
BZFILE* b
);
BZ_EXTERN const char * BZ_API(BZ2_bzerror) (
BZFILE *b,
int *errnum
);
#endif
#ifdef __cplusplus
}
#endif
#endif
/*-------------------------------------------------------------*/
/*--- end bzlib.h ---*/
/*-------------------------------------------------------------*/

509
nc_test4/filter_test/bzlib_private.h Normal file
View File

@ -0,0 +1,509 @@
/*-------------------------------------------------------------*/
/*--- Private header file for the library. ---*/
/*--- bzlib_private.h ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#ifndef _BZLIB_PRIVATE_H
#define _BZLIB_PRIVATE_H
#include <stdlib.h>
#ifndef BZ_NO_STDIO
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#endif
#include "bzlib.h"
/*-- General stuff. --*/
#define BZ_VERSION "1.0.6, 6-Sept-2010"
typedef char Char;
typedef unsigned char Bool;
typedef unsigned char UChar;
typedef int Int32;
typedef unsigned int UInt32;
typedef short Int16;
typedef unsigned short UInt16;
#define True ((Bool)1)
#define False ((Bool)0)
#ifndef __GNUC__
#define __inline__ /* */
#endif
#ifndef BZ_NO_STDIO
extern void BZ2_bz__AssertH__fail ( int errcode );
#define AssertH(cond,errcode) \
{ if (!(cond)) BZ2_bz__AssertH__fail ( errcode ); }
#if BZ_DEBUG
#define AssertD(cond,msg) \
{ if (!(cond)) { \
fprintf ( stderr, \
"\n\nlibbzip2(debug build): internal error\n\t%s\n", msg );\
exit(1); \
}}
#else
#define AssertD(cond,msg) /* */
#endif
#define VPrintf0(zf) \
fprintf(stderr,zf)
#define VPrintf1(zf,za1) \
fprintf(stderr,zf,za1)
#define VPrintf2(zf,za1,za2) \
fprintf(stderr,zf,za1,za2)
#define VPrintf3(zf,za1,za2,za3) \
fprintf(stderr,zf,za1,za2,za3)
#define VPrintf4(zf,za1,za2,za3,za4) \
fprintf(stderr,zf,za1,za2,za3,za4)
#define VPrintf5(zf,za1,za2,za3,za4,za5) \
fprintf(stderr,zf,za1,za2,za3,za4,za5)
#else
extern void bz_internal_error ( int errcode );
#define AssertH(cond,errcode) \
{ if (!(cond)) bz_internal_error ( errcode ); }
#define AssertD(cond,msg) do { } while (0)
#define VPrintf0(zf) do { } while (0)
#define VPrintf1(zf,za1) do { } while (0)
#define VPrintf2(zf,za1,za2) do { } while (0)
#define VPrintf3(zf,za1,za2,za3) do { } while (0)
#define VPrintf4(zf,za1,za2,za3,za4) do { } while (0)
#define VPrintf5(zf,za1,za2,za3,za4,za5) do { } while (0)
#endif
#define BZALLOC(nnn) (strm->bzalloc)(strm->opaque,(nnn),1)
#define BZFREE(ppp) (strm->bzfree)(strm->opaque,(ppp))
/*-- Header bytes. --*/
#define BZ_HDR_B 0x42 /* 'B' */
#define BZ_HDR_Z 0x5a /* 'Z' */
#define BZ_HDR_h 0x68 /* 'h' */
#define BZ_HDR_0 0x30 /* '0' */
/*-- Constants for the back end. --*/
#define BZ_MAX_ALPHA_SIZE 258
#define BZ_MAX_CODE_LEN 23
#define BZ_RUNA 0
#define BZ_RUNB 1
#define BZ_N_GROUPS 6
#define BZ_G_SIZE 50
#define BZ_N_ITERS 4
#define BZ_MAX_SELECTORS (2 + (900000 / BZ_G_SIZE))
/*-- Stuff for randomising repetitive blocks. --*/
extern Int32 BZ2_rNums[512];
#define BZ_RAND_DECLS \
Int32 rNToGo; \
Int32 rTPos \
#define BZ_RAND_INIT_MASK \
s->rNToGo = 0; \
s->rTPos = 0 \
#define BZ_RAND_MASK ((s->rNToGo == 1) ? 1 : 0)
#define BZ_RAND_UPD_MASK \
if (s->rNToGo == 0) { \
s->rNToGo = BZ2_rNums[s->rTPos]; \
s->rTPos++; \
if (s->rTPos == 512) s->rTPos = 0; \
} \
s->rNToGo--;
/*-- Stuff for doing CRCs. --*/
extern UInt32 BZ2_crc32Table[256];
#define BZ_INITIALISE_CRC(crcVar) \
{ \
crcVar = 0xffffffffL; \
}
#define BZ_FINALISE_CRC(crcVar) \
{ \
crcVar = ~(crcVar); \
}
#define BZ_UPDATE_CRC(crcVar,cha) \
{ \
crcVar = (crcVar << 8) ^ \
BZ2_crc32Table[(crcVar >> 24) ^ \
((UChar)cha)]; \
}
/*-- States and modes for compression. --*/
#define BZ_M_IDLE 1
#define BZ_M_RUNNING 2
#define BZ_M_FLUSHING 3
#define BZ_M_FINISHING 4
#define BZ_S_OUTPUT 1
#define BZ_S_INPUT 2
#define BZ_N_RADIX 2
#define BZ_N_QSORT 12
#define BZ_N_SHELL 18
#define BZ_N_OVERSHOOT (BZ_N_RADIX + BZ_N_QSORT + BZ_N_SHELL + 2)
/*-- Structure holding all the compression-side stuff. --*/
typedef
struct {
/* pointer back to the struct bz_stream */
bz_stream* strm;
/* mode this stream is in, and whether inputting */
/* or outputting data */
Int32 mode;
Int32 state;
/* remembers avail_in when flush/finish requested */
UInt32 avail_in_expect;
/* for doing the block sorting */
UInt32* arr1;
UInt32* arr2;
UInt32* ftab;
Int32 origPtr;
/* aliases for arr1 and arr2 */
UInt32* ptr;
UChar* block;
UInt16* mtfv;
UChar* zbits;
/* for deciding when to use the fallback sorting algorithm */
Int32 workFactor;
/* run-length-encoding of the input */
UInt32 state_in_ch;
Int32 state_in_len;
BZ_RAND_DECLS;
/* input and output limits and current posns */
Int32 nblock;
Int32 nblockMAX;
Int32 numZ;
Int32 state_out_pos;
/* map of bytes used in block */
Int32 nInUse;
Bool inUse[256];
UChar unseqToSeq[256];
/* the buffer for bit stream creation */
UInt32 bsBuff;
Int32 bsLive;
/* block and combined CRCs */
UInt32 blockCRC;
UInt32 combinedCRC;
/* misc administratium */
Int32 verbosity;
Int32 blockNo;
Int32 blockSize100k;
/* stuff for coding the MTF values */
Int32 nMTF;
Int32 mtfFreq [BZ_MAX_ALPHA_SIZE];
UChar selector [BZ_MAX_SELECTORS];
UChar selectorMtf[BZ_MAX_SELECTORS];
UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 code [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 rfreq [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
/* second dimension: only 3 needed; 4 makes index calculations faster */
UInt32 len_pack[BZ_MAX_ALPHA_SIZE][4];
}
EState;
/*-- externs for compression. --*/
extern void
BZ2_blockSort ( EState* );
extern void
BZ2_compressBlock ( EState*, Bool );
extern void
BZ2_bsInitWrite ( EState* );
extern void
BZ2_hbAssignCodes ( Int32*, UChar*, Int32, Int32, Int32 );
extern void
BZ2_hbMakeCodeLengths ( UChar*, Int32*, Int32, Int32 );
/*-- states for decompression. --*/
#define BZ_X_IDLE 1
#define BZ_X_OUTPUT 2
#define BZ_X_MAGIC_1 10
#define BZ_X_MAGIC_2 11
#define BZ_X_MAGIC_3 12
#define BZ_X_MAGIC_4 13
#define BZ_X_BLKHDR_1 14
#define BZ_X_BLKHDR_2 15
#define BZ_X_BLKHDR_3 16
#define BZ_X_BLKHDR_4 17
#define BZ_X_BLKHDR_5 18
#define BZ_X_BLKHDR_6 19
#define BZ_X_BCRC_1 20
#define BZ_X_BCRC_2 21
#define BZ_X_BCRC_3 22
#define BZ_X_BCRC_4 23
#define BZ_X_RANDBIT 24
#define BZ_X_ORIGPTR_1 25
#define BZ_X_ORIGPTR_2 26
#define BZ_X_ORIGPTR_3 27
#define BZ_X_MAPPING_1 28
#define BZ_X_MAPPING_2 29
#define BZ_X_SELECTOR_1 30
#define BZ_X_SELECTOR_2 31
#define BZ_X_SELECTOR_3 32
#define BZ_X_CODING_1 33
#define BZ_X_CODING_2 34
#define BZ_X_CODING_3 35
#define BZ_X_MTF_1 36
#define BZ_X_MTF_2 37
#define BZ_X_MTF_3 38
#define BZ_X_MTF_4 39
#define BZ_X_MTF_5 40
#define BZ_X_MTF_6 41
#define BZ_X_ENDHDR_2 42
#define BZ_X_ENDHDR_3 43
#define BZ_X_ENDHDR_4 44
#define BZ_X_ENDHDR_5 45
#define BZ_X_ENDHDR_6 46
#define BZ_X_CCRC_1 47
#define BZ_X_CCRC_2 48
#define BZ_X_CCRC_3 49
#define BZ_X_CCRC_4 50
/*-- Constants for the fast MTF decoder. --*/
#define MTFA_SIZE 4096
#define MTFL_SIZE 16
/*-- Structure holding all the decompression-side stuff. --*/
typedef
struct {
/* pointer back to the struct bz_stream */
bz_stream* strm;
/* state indicator for this stream */
Int32 state;
/* for doing the final run-length decoding */
UChar state_out_ch;
Int32 state_out_len;
Bool blockRandomised;
BZ_RAND_DECLS;
/* the buffer for bit stream reading */
UInt32 bsBuff;
Int32 bsLive;
/* misc administratium */
Int32 blockSize100k;
Bool smallDecompress;
Int32 currBlockNo;
Int32 verbosity;
/* for undoing the Burrows-Wheeler transform */
Int32 origPtr;
UInt32 tPos;
Int32 k0;
Int32 unzftab[256];
Int32 nblock_used;
Int32 cftab[257];
Int32 cftabCopy[257];
/* for undoing the Burrows-Wheeler transform (FAST) */
UInt32 *tt;
/* for undoing the Burrows-Wheeler transform (SMALL) */
UInt16 *ll16;
UChar *ll4;
/* stored and calculated CRCs */
UInt32 storedBlockCRC;
UInt32 storedCombinedCRC;
UInt32 calculatedBlockCRC;
UInt32 calculatedCombinedCRC;
/* map of bytes used in block */
Int32 nInUse;
Bool inUse[256];
Bool inUse16[16];
UChar seqToUnseq[256];
/* for decoding the MTF values */
UChar mtfa [MTFA_SIZE];
Int32 mtfbase[256 / MTFL_SIZE];
UChar selector [BZ_MAX_SELECTORS];
UChar selectorMtf[BZ_MAX_SELECTORS];
UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 limit [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 base [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 perm [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 minLens[BZ_N_GROUPS];
/* save area for scalars in the main decompress code */
Int32 save_i;
Int32 save_j;
Int32 save_t;
Int32 save_alphaSize;
Int32 save_nGroups;
Int32 save_nSelectors;
Int32 save_EOB;
Int32 save_groupNo;
Int32 save_groupPos;
Int32 save_nextSym;
Int32 save_nblockMAX;
Int32 save_nblock;
Int32 save_es;
Int32 save_N;
Int32 save_curr;
Int32 save_zt;
Int32 save_zn;
Int32 save_zvec;
Int32 save_zj;
Int32 save_gSel;
Int32 save_gMinlen;
Int32* save_gLimit;
Int32* save_gBase;
Int32* save_gPerm;
}
DState;
/*-- Macros for decompression. --*/
#define BZ_GET_FAST(cccc) \
/* c_tPos is unsigned, hence test < 0 is pointless. */ \
if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
s->tPos = s->tt[s->tPos]; \
cccc = (UChar)(s->tPos & 0xff); \
s->tPos >>= 8;
#define BZ_GET_FAST_C(cccc) \
/* c_tPos is unsigned, hence test < 0 is pointless. */ \
if (c_tPos >= (UInt32)100000 * (UInt32)ro_blockSize100k) return True; \
c_tPos = c_tt[c_tPos]; \
cccc = (UChar)(c_tPos & 0xff); \
c_tPos >>= 8;
#define SET_LL4(i,n) \
{ if (((i) & 0x1) == 0) \
s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0xf0) | (n); else \
s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0x0f) | ((n) << 4); \
}
#define GET_LL4(i) \
((((UInt32)(s->ll4[(i) >> 1])) >> (((i) << 2) & 0x4)) & 0xF)
#define SET_LL(i,n) \
{ s->ll16[i] = (UInt16)(n & 0x0000ffff); \
SET_LL4(i, n >> 16); \
}
#define GET_LL(i) \
(((UInt32)s->ll16[i]) | (GET_LL4(i) << 16))
#define BZ_GET_SMALL(cccc) \
/* c_tPos is unsigned, hence test < 0 is pointless. */ \
if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \
cccc = BZ2_indexIntoF ( s->tPos, s->cftab ); \
s->tPos = GET_LL(s->tPos);
/*-- externs for decompression. --*/
extern Int32
BZ2_indexIntoF ( Int32, Int32* );
extern Int32
BZ2_decompress ( DState* );
extern void
BZ2_hbCreateDecodeTables ( Int32*, Int32*, Int32*, UChar*,
Int32, Int32, Int32 );
#endif
/*-- BZ_NO_STDIO seems to make NULL disappear on some platforms. --*/
#ifdef BZ_NO_STDIO
#ifndef NULL
#define NULL 0
#endif
#endif
/*-------------------------------------------------------------*/
/*--- end bzlib_private.h ---*/
/*-------------------------------------------------------------*/

672
nc_test4/filter_test/compress.c Normal file
View File

@ -0,0 +1,672 @@
/*-------------------------------------------------------------*/
/*--- Compression machinery (not incl block sorting) ---*/
/*--- compress.c ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
/* CHANGES
0.9.0 -- original version.
0.9.0a/b -- no changes in this file.
0.9.0c -- changed setting of nGroups in sendMTFValues()
so as to do a bit better on small files
*/
#include "bzlib_private.h"
/*---------------------------------------------------*/
/*--- Bit stream I/O ---*/
/*---------------------------------------------------*/
/*---------------------------------------------------*/
void BZ2_bsInitWrite ( EState* s )
{
s->bsLive = 0;
s->bsBuff = 0;
}
/*---------------------------------------------------*/
static
void bsFinishWrite ( EState* s )
{
while (s->bsLive > 0) {
s->zbits[s->numZ] = (UChar)(s->bsBuff >> 24);
s->numZ++;
s->bsBuff <<= 8;
s->bsLive -= 8;
}
}
/*---------------------------------------------------*/
#define bsNEEDW(nz) \
{ \
while (s->bsLive >= 8) { \
s->zbits[s->numZ] \
= (UChar)(s->bsBuff >> 24); \
s->numZ++; \
s->bsBuff <<= 8; \
s->bsLive -= 8; \
} \
}
/*---------------------------------------------------*/
static
__inline__
void bsW ( EState* s, Int32 n, UInt32 v )
{
bsNEEDW ( n );
s->bsBuff |= (v << (32 - s->bsLive - n));
s->bsLive += n;
}
/*---------------------------------------------------*/
static
void bsPutUInt32 ( EState* s, UInt32 u )
{
bsW ( s, 8, (u >> 24) & 0xffL );
bsW ( s, 8, (u >> 16) & 0xffL );
bsW ( s, 8, (u >> 8) & 0xffL );
bsW ( s, 8, u & 0xffL );
}
/*---------------------------------------------------*/
static
void bsPutUChar ( EState* s, UChar c )
{
bsW( s, 8, (UInt32)c );
}
/*---------------------------------------------------*/
/*--- The back end proper ---*/
/*---------------------------------------------------*/
/*---------------------------------------------------*/
static
void makeMaps_e ( EState* s )
{
Int32 i;
s->nInUse = 0;
for (i = 0; i < 256; i++)
if (s->inUse[i]) {
s->unseqToSeq[i] = s->nInUse;
s->nInUse++;
}
}
/*---------------------------------------------------*/
static
void generateMTFValues ( EState* s )
{
UChar yy[256];
Int32 i, j;
Int32 zPend;
Int32 wr;
Int32 EOB;
/*
After sorting (eg, here),
s->arr1 [ 0 .. s->nblock-1 ] holds sorted order,
and
((UChar*)s->arr2) [ 0 .. s->nblock-1 ]
holds the original block data.
The first thing to do is generate the MTF values,
and put them in
((UInt16*)s->arr1) [ 0 .. s->nblock-1 ].
Because there are strictly fewer or equal MTF values
than block values, ptr values in this area are overwritten
with MTF values only when they are no longer needed.
The final compressed bitstream is generated into the
area starting at
(UChar*) (&((UChar*)s->arr2)[s->nblock])
These storage aliases are set up in bzCompressInit(),
except for the last one, which is arranged in
compressBlock().
*/
UInt32* ptr = s->ptr;
UChar* block = s->block;
UInt16* mtfv = s->mtfv;
makeMaps_e ( s );
EOB = s->nInUse+1;
for (i = 0; i <= EOB; i++) s->mtfFreq[i] = 0;
wr = 0;
zPend = 0;
for (i = 0; i < s->nInUse; i++) yy[i] = (UChar) i;
for (i = 0; i < s->nblock; i++) {
UChar ll_i;
AssertD ( wr <= i, "generateMTFValues(1)" );
j = ptr[i]-1; if (j < 0) j += s->nblock;
ll_i = s->unseqToSeq[block[j]];
AssertD ( ll_i < s->nInUse, "generateMTFValues(2a)" );
if (yy[0] == ll_i) {
zPend++;
} else {
if (zPend > 0) {
zPend--;
while (True) {
if (zPend & 1) {
mtfv[wr] = BZ_RUNB; wr++;
s->mtfFreq[BZ_RUNB]++;
} else {
mtfv[wr] = BZ_RUNA; wr++;
s->mtfFreq[BZ_RUNA]++;
}
if (zPend < 2) break;
zPend = (zPend - 2) / 2;
};
zPend = 0;
}
{
register UChar rtmp;
register UChar* ryy_j;
register UChar rll_i;
rtmp = yy[1];
yy[1] = yy[0];
ryy_j = &(yy[1]);
rll_i = ll_i;
while ( rll_i != rtmp ) {
register UChar rtmp2;
ryy_j++;
rtmp2 = rtmp;
rtmp = *ryy_j;
*ryy_j = rtmp2;
};
yy[0] = rtmp;
j = ryy_j - &(yy[0]);
mtfv[wr] = j+1; wr++; s->mtfFreq[j+1]++;
}
}
}
if (zPend > 0) {
zPend--;
while (True) {
if (zPend & 1) {
mtfv[wr] = BZ_RUNB; wr++;
s->mtfFreq[BZ_RUNB]++;
} else {
mtfv[wr] = BZ_RUNA; wr++;
s->mtfFreq[BZ_RUNA]++;
}
if (zPend < 2) break;
zPend = (zPend - 2) / 2;
};
zPend = 0;
}
mtfv[wr] = EOB; wr++; s->mtfFreq[EOB]++;
s->nMTF = wr;
}
/*---------------------------------------------------*/
#define BZ_LESSER_ICOST 0
#define BZ_GREATER_ICOST 15
static
void sendMTFValues ( EState* s )
{
Int32 v, t, i, j, gs, ge, totc, bt, bc, iter;
Int32 nSelectors, alphaSize, minLen, maxLen, selCtr;
Int32 nGroups, nBytes;
/*--
UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
is a global since the decoder also needs it.
Int32 code[BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
Int32 rfreq[BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE];
are also globals only used in this proc.
Made global to keep stack frame size small.
--*/
UInt16 cost[BZ_N_GROUPS];
Int32 fave[BZ_N_GROUPS];
UInt16* mtfv = s->mtfv;
if (s->verbosity >= 3)
VPrintf3( " %d in block, %d after MTF & 1-2 coding, "
"%d+2 syms in use\n",
s->nblock, s->nMTF, s->nInUse );
alphaSize = s->nInUse+2;
for (t = 0; t < BZ_N_GROUPS; t++)
for (v = 0; v < alphaSize; v++)
s->len[t][v] = BZ_GREATER_ICOST;
/*--- Decide how many coding tables to use ---*/
AssertH ( s->nMTF > 0, 3001 );
if (s->nMTF < 200) nGroups = 2; else
if (s->nMTF < 600) nGroups = 3; else
if (s->nMTF < 1200) nGroups = 4; else
if (s->nMTF < 2400) nGroups = 5; else
nGroups = 6;
/*--- Generate an initial set of coding tables ---*/
{
Int32 nPart, remF, tFreq, aFreq;
nPart = nGroups;
remF = s->nMTF;
gs = 0;
while (nPart > 0) {
tFreq = remF / nPart;
ge = gs-1;
aFreq = 0;
while (aFreq < tFreq && ge < alphaSize-1) {
ge++;
aFreq += s->mtfFreq[ge];
}
if (ge > gs
&& nPart != nGroups && nPart != 1
&& ((nGroups-nPart) % 2 == 1)) {
aFreq -= s->mtfFreq[ge];
ge--;
}
if (s->verbosity >= 3)
VPrintf5( " initial group %d, [%d .. %d], "
"has %d syms (%4.1f%%)\n",
nPart, gs, ge, aFreq,
(100.0 * (float)aFreq) / (float)(s->nMTF) );
for (v = 0; v < alphaSize; v++)
if (v >= gs && v <= ge)
s->len[nPart-1][v] = BZ_LESSER_ICOST; else
s->len[nPart-1][v] = BZ_GREATER_ICOST;
nPart--;
gs = ge+1;
remF -= aFreq;
}
}
/*---
Iterate up to BZ_N_ITERS times to improve the tables.
---*/
for (iter = 0; iter < BZ_N_ITERS; iter++) {
for (t = 0; t < nGroups; t++) fave[t] = 0;
for (t = 0; t < nGroups; t++)
for (v = 0; v < alphaSize; v++)
s->rfreq[t][v] = 0;
/*---
Set up an auxiliary length table which is used to fast-track
the common case (nGroups == 6).
---*/
if (nGroups == 6) {
for (v = 0; v < alphaSize; v++) {
s->len_pack[v][0] = (s->len[1][v] << 16) | s->len[0][v];
s->len_pack[v][1] = (s->len[3][v] << 16) | s->len[2][v];
s->len_pack[v][2] = (s->len[5][v] << 16) | s->len[4][v];
}
}
nSelectors = 0;
totc = 0;
gs = 0;
while (True) {
/*--- Set group start & end marks. --*/
if (gs >= s->nMTF) break;
ge = gs + BZ_G_SIZE - 1;
if (ge >= s->nMTF) ge = s->nMTF-1;
/*--
Calculate the cost of this group as coded
by each of the coding tables.
--*/
for (t = 0; t < nGroups; t++) cost[t] = 0;
if (nGroups == 6 && 50 == ge-gs+1) {
/*--- fast track the common case ---*/
register UInt32 cost01, cost23, cost45;
register UInt16 icv;
cost01 = cost23 = cost45 = 0;
# define BZ_ITER(nn) \
icv = mtfv[gs+(nn)]; \
cost01 += s->len_pack[icv][0]; \
cost23 += s->len_pack[icv][1]; \
cost45 += s->len_pack[icv][2]; \
BZ_ITER(0); BZ_ITER(1); BZ_ITER(2); BZ_ITER(3); BZ_ITER(4);
BZ_ITER(5); BZ_ITER(6); BZ_ITER(7); BZ_ITER(8); BZ_ITER(9);
BZ_ITER(10); BZ_ITER(11); BZ_ITER(12); BZ_ITER(13); BZ_ITER(14);
BZ_ITER(15); BZ_ITER(16); BZ_ITER(17); BZ_ITER(18); BZ_ITER(19);
BZ_ITER(20); BZ_ITER(21); BZ_ITER(22); BZ_ITER(23); BZ_ITER(24);
BZ_ITER(25); BZ_ITER(26); BZ_ITER(27); BZ_ITER(28); BZ_ITER(29);
BZ_ITER(30); BZ_ITER(31); BZ_ITER(32); BZ_ITER(33); BZ_ITER(34);
BZ_ITER(35); BZ_ITER(36); BZ_ITER(37); BZ_ITER(38); BZ_ITER(39);
BZ_ITER(40); BZ_ITER(41); BZ_ITER(42); BZ_ITER(43); BZ_ITER(44);
BZ_ITER(45); BZ_ITER(46); BZ_ITER(47); BZ_ITER(48); BZ_ITER(49);
# undef BZ_ITER
cost[0] = cost01 & 0xffff; cost[1] = cost01 >> 16;
cost[2] = cost23 & 0xffff; cost[3] = cost23 >> 16;
cost[4] = cost45 & 0xffff; cost[5] = cost45 >> 16;
} else {
/*--- slow version which correctly handles all situations ---*/
for (i = gs; i <= ge; i++) {
UInt16 icv = mtfv[i];
for (t = 0; t < nGroups; t++) cost[t] += s->len[t][icv];
}
}
/*--
Find the coding table which is best for this group,
and record its identity in the selector table.
--*/
bc = 999999999; bt = -1;
for (t = 0; t < nGroups; t++)
if (cost[t] < bc) { bc = cost[t]; bt = t; };
totc += bc;
fave[bt]++;
s->selector[nSelectors] = bt;
nSelectors++;
/*--
Increment the symbol frequencies for the selected table.
--*/
if (nGroups == 6 && 50 == ge-gs+1) {
/*--- fast track the common case ---*/
# define BZ_ITUR(nn) s->rfreq[bt][ mtfv[gs+(nn)] ]++
BZ_ITUR(0); BZ_ITUR(1); BZ_ITUR(2); BZ_ITUR(3); BZ_ITUR(4);
BZ_ITUR(5); BZ_ITUR(6); BZ_ITUR(7); BZ_ITUR(8); BZ_ITUR(9);
BZ_ITUR(10); BZ_ITUR(11); BZ_ITUR(12); BZ_ITUR(13); BZ_ITUR(14);
BZ_ITUR(15); BZ_ITUR(16); BZ_ITUR(17); BZ_ITUR(18); BZ_ITUR(19);
BZ_ITUR(20); BZ_ITUR(21); BZ_ITUR(22); BZ_ITUR(23); BZ_ITUR(24);
BZ_ITUR(25); BZ_ITUR(26); BZ_ITUR(27); BZ_ITUR(28); BZ_ITUR(29);
BZ_ITUR(30); BZ_ITUR(31); BZ_ITUR(32); BZ_ITUR(33); BZ_ITUR(34);
BZ_ITUR(35); BZ_ITUR(36); BZ_ITUR(37); BZ_ITUR(38); BZ_ITUR(39);
BZ_ITUR(40); BZ_ITUR(41); BZ_ITUR(42); BZ_ITUR(43); BZ_ITUR(44);
BZ_ITUR(45); BZ_ITUR(46); BZ_ITUR(47); BZ_ITUR(48); BZ_ITUR(49);
# undef BZ_ITUR
} else {
/*--- slow version which correctly handles all situations ---*/
for (i = gs; i <= ge; i++)
s->rfreq[bt][ mtfv[i] ]++;
}
gs = ge+1;
}
if (s->verbosity >= 3) {
VPrintf2 ( " pass %d: size is %d, grp uses are ",
iter+1, totc/8 );
for (t = 0; t < nGroups; t++)
VPrintf1 ( "%d ", fave[t] );
VPrintf0 ( "\n" );
}
/*--
Recompute the tables based on the accumulated frequencies.
--*/
/* maxLen was changed from 20 to 17 in bzip2-1.0.3. See
comment in huffman.c for details. */
for (t = 0; t < nGroups; t++)
BZ2_hbMakeCodeLengths ( &(s->len[t][0]), &(s->rfreq[t][0]),
alphaSize, 17 /*20*/ );
}
AssertH( nGroups < 8, 3002 );
AssertH( nSelectors < 32768 &&
nSelectors <= (2 + (900000 / BZ_G_SIZE)),
3003 );
/*--- Compute MTF values for the selectors. ---*/
{
UChar pos[BZ_N_GROUPS], ll_i, tmp2, tmp;
for (i = 0; i < nGroups; i++) pos[i] = i;
for (i = 0; i < nSelectors; i++) {
ll_i = s->selector[i];
j = 0;
tmp = pos[j];
while ( ll_i != tmp ) {
j++;
tmp2 = tmp;
tmp = pos[j];
pos[j] = tmp2;
};
pos[0] = tmp;
s->selectorMtf[i] = j;
}
};
/*--- Assign actual codes for the tables. --*/
for (t = 0; t < nGroups; t++) {
minLen = 32;
maxLen = 0;
for (i = 0; i < alphaSize; i++) {
if (s->len[t][i] > maxLen) maxLen = s->len[t][i];
if (s->len[t][i] < minLen) minLen = s->len[t][i];
}
AssertH ( !(maxLen > 17 /*20*/ ), 3004 );
AssertH ( !(minLen < 1), 3005 );
BZ2_hbAssignCodes ( &(s->code[t][0]), &(s->len[t][0]),
minLen, maxLen, alphaSize );
}
/*--- Transmit the mapping table. ---*/
{
Bool inUse16[16];
for (i = 0; i < 16; i++) {
inUse16[i] = False;
for (j = 0; j < 16; j++)
if (s->inUse[i * 16 + j]) inUse16[i] = True;
}
nBytes = s->numZ;
for (i = 0; i < 16; i++)
if (inUse16[i]) bsW(s,1,1); else bsW(s,1,0);
for (i = 0; i < 16; i++)
if (inUse16[i])
for (j = 0; j < 16; j++) {
if (s->inUse[i * 16 + j]) bsW(s,1,1); else bsW(s,1,0);
}
if (s->verbosity >= 3)
VPrintf1( " bytes: mapping %d, ", s->numZ-nBytes );
}
/*--- Now the selectors. ---*/
nBytes = s->numZ;
bsW ( s, 3, nGroups );
bsW ( s, 15, nSelectors );
for (i = 0; i < nSelectors; i++) {
for (j = 0; j < s->selectorMtf[i]; j++) bsW(s,1,1);
bsW(s,1,0);
}
if (s->verbosity >= 3)
VPrintf1( "selectors %d, ", s->numZ-nBytes );
/*--- Now the coding tables. ---*/
nBytes = s->numZ;
for (t = 0; t < nGroups; t++) {
Int32 curr = s->len[t][0];
bsW ( s, 5, curr );
for (i = 0; i < alphaSize; i++) {
while (curr < s->len[t][i]) { bsW(s,2,2); curr++; /* 10 */ };
while (curr > s->len[t][i]) { bsW(s,2,3); curr--; /* 11 */ };
bsW ( s, 1, 0 );
}
}
if (s->verbosity >= 3)
VPrintf1 ( "code lengths %d, ", s->numZ-nBytes );
/*--- And finally, the block data proper ---*/
nBytes = s->numZ;
selCtr = 0;
gs = 0;
while (True) {
if (gs >= s->nMTF) break;
ge = gs + BZ_G_SIZE - 1;
if (ge >= s->nMTF) ge = s->nMTF-1;
AssertH ( s->selector[selCtr] < nGroups, 3006 );
if (nGroups == 6 && 50 == ge-gs+1) {
/*--- fast track the common case ---*/
UInt16 mtfv_i;
UChar* s_len_sel_selCtr
= &(s->len[s->selector[selCtr]][0]);
Int32* s_code_sel_selCtr
= &(s->code[s->selector[selCtr]][0]);
# define BZ_ITAH(nn) \
mtfv_i = mtfv[gs+(nn)]; \
bsW ( s, \
s_len_sel_selCtr[mtfv_i], \
s_code_sel_selCtr[mtfv_i] )
BZ_ITAH(0); BZ_ITAH(1); BZ_ITAH(2); BZ_ITAH(3); BZ_ITAH(4);
BZ_ITAH(5); BZ_ITAH(6); BZ_ITAH(7); BZ_ITAH(8); BZ_ITAH(9);
BZ_ITAH(10); BZ_ITAH(11); BZ_ITAH(12); BZ_ITAH(13); BZ_ITAH(14);
BZ_ITAH(15); BZ_ITAH(16); BZ_ITAH(17); BZ_ITAH(18); BZ_ITAH(19);
BZ_ITAH(20); BZ_ITAH(21); BZ_ITAH(22); BZ_ITAH(23); BZ_ITAH(24);
BZ_ITAH(25); BZ_ITAH(26); BZ_ITAH(27); BZ_ITAH(28); BZ_ITAH(29);
BZ_ITAH(30); BZ_ITAH(31); BZ_ITAH(32); BZ_ITAH(33); BZ_ITAH(34);
BZ_ITAH(35); BZ_ITAH(36); BZ_ITAH(37); BZ_ITAH(38); BZ_ITAH(39);
BZ_ITAH(40); BZ_ITAH(41); BZ_ITAH(42); BZ_ITAH(43); BZ_ITAH(44);
BZ_ITAH(45); BZ_ITAH(46); BZ_ITAH(47); BZ_ITAH(48); BZ_ITAH(49);
# undef BZ_ITAH
} else {
/*--- slow version which correctly handles all situations ---*/
for (i = gs; i <= ge; i++) {
bsW ( s,
s->len [s->selector[selCtr]] [mtfv[i]],
s->code [s->selector[selCtr]] [mtfv[i]] );
}
}
gs = ge+1;
selCtr++;
}
AssertH( selCtr == nSelectors, 3007 );
if (s->verbosity >= 3)
VPrintf1( "codes %d\n", s->numZ-nBytes );
}
/*---------------------------------------------------*/
void BZ2_compressBlock ( EState* s, Bool is_last_block )
{
if (s->nblock > 0) {
BZ_FINALISE_CRC ( s->blockCRC );
s->combinedCRC = (s->combinedCRC << 1) | (s->combinedCRC >> 31);
s->combinedCRC ^= s->blockCRC;
if (s->blockNo > 1) s->numZ = 0;
if (s->verbosity >= 2)
VPrintf4( " block %d: crc = 0x%08x, "
"combined CRC = 0x%08x, size = %d\n",
s->blockNo, s->blockCRC, s->combinedCRC, s->nblock );
BZ2_blockSort ( s );
}
s->zbits = (UChar*) (&((UChar*)s->arr2)[s->nblock]);
/*-- If this is the first block, create the stream header. --*/
if (s->blockNo == 1) {
BZ2_bsInitWrite ( s );
bsPutUChar ( s, BZ_HDR_B );
bsPutUChar ( s, BZ_HDR_Z );
bsPutUChar ( s, BZ_HDR_h );
bsPutUChar ( s, (UChar)(BZ_HDR_0 + s->blockSize100k) );
}
if (s->nblock > 0) {
bsPutUChar ( s, 0x31 ); bsPutUChar ( s, 0x41 );
bsPutUChar ( s, 0x59 ); bsPutUChar ( s, 0x26 );
bsPutUChar ( s, 0x53 ); bsPutUChar ( s, 0x59 );
/*-- Now the block's CRC, so it is in a known place. --*/
bsPutUInt32 ( s, s->blockCRC );
/*--
Now a single bit indicating (non-)randomisation.
As of version 0.9.5, we use a better sorting algorithm
which makes randomisation unnecessary. So always set
the randomised bit to 'no'. Of course, the decoder
still needs to be able to handle randomised blocks
so as to maintain backwards compatibility with
older versions of bzip2.
--*/
bsW(s,1,0);
bsW ( s, 24, s->origPtr );
generateMTFValues ( s );
sendMTFValues ( s );
}
/*-- If this is the last block, add the stream trailer. --*/
if (is_last_block) {
bsPutUChar ( s, 0x17 ); bsPutUChar ( s, 0x72 );
bsPutUChar ( s, 0x45 ); bsPutUChar ( s, 0x38 );
bsPutUChar ( s, 0x50 ); bsPutUChar ( s, 0x90 );
bsPutUInt32 ( s, s->combinedCRC );
if (s->verbosity >= 2)
VPrintf1( " final combined CRC = 0x%08x\n ", s->combinedCRC );
bsFinishWrite ( s );
}
}
/*-------------------------------------------------------------*/
/*--- end compress.c ---*/
/*-------------------------------------------------------------*/

104
nc_test4/filter_test/crctable.c Normal file
View File

@ -0,0 +1,104 @@
/*-------------------------------------------------------------*/
/*--- Table for doing CRCs ---*/
/*--- crctable.c ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#include "bzlib_private.h"
/*--
I think this is an implementation of the AUTODIN-II,
Ethernet & FDDI 32-bit CRC standard. Vaguely derived
from code by Rob Warnock, in Section 51 of the
comp.compression FAQ.
--*/
UInt32 BZ2_crc32Table[256] = {
/*-- Ugly, innit? --*/
0x00000000L, 0x04c11db7L, 0x09823b6eL, 0x0d4326d9L,
0x130476dcL, 0x17c56b6bL, 0x1a864db2L, 0x1e475005L,
0x2608edb8L, 0x22c9f00fL, 0x2f8ad6d6L, 0x2b4bcb61L,
0x350c9b64L, 0x31cd86d3L, 0x3c8ea00aL, 0x384fbdbdL,
0x4c11db70L, 0x48d0c6c7L, 0x4593e01eL, 0x4152fda9L,
0x5f15adacL, 0x5bd4b01bL, 0x569796c2L, 0x52568b75L,
0x6a1936c8L, 0x6ed82b7fL, 0x639b0da6L, 0x675a1011L,
0x791d4014L, 0x7ddc5da3L, 0x709f7b7aL, 0x745e66cdL,
0x9823b6e0L, 0x9ce2ab57L, 0x91a18d8eL, 0x95609039L,
0x8b27c03cL, 0x8fe6dd8bL, 0x82a5fb52L, 0x8664e6e5L,
0xbe2b5b58L, 0xbaea46efL, 0xb7a96036L, 0xb3687d81L,
0xad2f2d84L, 0xa9ee3033L, 0xa4ad16eaL, 0xa06c0b5dL,
0xd4326d90L, 0xd0f37027L, 0xddb056feL, 0xd9714b49L,
0xc7361b4cL, 0xc3f706fbL, 0xceb42022L, 0xca753d95L,
0xf23a8028L, 0xf6fb9d9fL, 0xfbb8bb46L, 0xff79a6f1L,
0xe13ef6f4L, 0xe5ffeb43L, 0xe8bccd9aL, 0xec7dd02dL,
0x34867077L, 0x30476dc0L, 0x3d044b19L, 0x39c556aeL,
0x278206abL, 0x23431b1cL, 0x2e003dc5L, 0x2ac12072L,
0x128e9dcfL, 0x164f8078L, 0x1b0ca6a1L, 0x1fcdbb16L,
0x018aeb13L, 0x054bf6a4L, 0x0808d07dL, 0x0cc9cdcaL,
0x7897ab07L, 0x7c56b6b0L, 0x71159069L, 0x75d48ddeL,
0x6b93dddbL, 0x6f52c06cL, 0x6211e6b5L, 0x66d0fb02L,
0x5e9f46bfL, 0x5a5e5b08L, 0x571d7dd1L, 0x53dc6066L,
0x4d9b3063L, 0x495a2dd4L, 0x44190b0dL, 0x40d816baL,
0xaca5c697L, 0xa864db20L, 0xa527fdf9L, 0xa1e6e04eL,
0xbfa1b04bL, 0xbb60adfcL, 0xb6238b25L, 0xb2e29692L,
0x8aad2b2fL, 0x8e6c3698L, 0x832f1041L, 0x87ee0df6L,
0x99a95df3L, 0x9d684044L, 0x902b669dL, 0x94ea7b2aL,
0xe0b41de7L, 0xe4750050L, 0xe9362689L, 0xedf73b3eL,
0xf3b06b3bL, 0xf771768cL, 0xfa325055L, 0xfef34de2L,
0xc6bcf05fL, 0xc27dede8L, 0xcf3ecb31L, 0xcbffd686L,
0xd5b88683L, 0xd1799b34L, 0xdc3abdedL, 0xd8fba05aL,
0x690ce0eeL, 0x6dcdfd59L, 0x608edb80L, 0x644fc637L,
0x7a089632L, 0x7ec98b85L, 0x738aad5cL, 0x774bb0ebL,
0x4f040d56L, 0x4bc510e1L, 0x46863638L, 0x42472b8fL,
0x5c007b8aL, 0x58c1663dL, 0x558240e4L, 0x51435d53L,
0x251d3b9eL, 0x21dc2629L, 0x2c9f00f0L, 0x285e1d47L,
0x36194d42L, 0x32d850f5L, 0x3f9b762cL, 0x3b5a6b9bL,
0x0315d626L, 0x07d4cb91L, 0x0a97ed48L, 0x0e56f0ffL,
0x1011a0faL, 0x14d0bd4dL, 0x19939b94L, 0x1d528623L,
0xf12f560eL, 0xf5ee4bb9L, 0xf8ad6d60L, 0xfc6c70d7L,
0xe22b20d2L, 0xe6ea3d65L, 0xeba91bbcL, 0xef68060bL,
0xd727bbb6L, 0xd3e6a601L, 0xdea580d8L, 0xda649d6fL,
0xc423cd6aL, 0xc0e2d0ddL, 0xcda1f604L, 0xc960ebb3L,
0xbd3e8d7eL, 0xb9ff90c9L, 0xb4bcb610L, 0xb07daba7L,
0xae3afba2L, 0xaafbe615L, 0xa7b8c0ccL, 0xa379dd7bL,
0x9b3660c6L, 0x9ff77d71L, 0x92b45ba8L, 0x9675461fL,
0x8832161aL, 0x8cf30badL, 0x81b02d74L, 0x857130c3L,
0x5d8a9099L, 0x594b8d2eL, 0x5408abf7L, 0x50c9b640L,
0x4e8ee645L, 0x4a4ffbf2L, 0x470cdd2bL, 0x43cdc09cL,
0x7b827d21L, 0x7f436096L, 0x7200464fL, 0x76c15bf8L,
0x68860bfdL, 0x6c47164aL, 0x61043093L, 0x65c52d24L,
0x119b4be9L, 0x155a565eL, 0x18197087L, 0x1cd86d30L,
0x029f3d35L, 0x065e2082L, 0x0b1d065bL, 0x0fdc1becL,
0x3793a651L, 0x3352bbe6L, 0x3e119d3fL, 0x3ad08088L,
0x2497d08dL, 0x2056cd3aL, 0x2d15ebe3L, 0x29d4f654L,
0xc5a92679L, 0xc1683bceL, 0xcc2b1d17L, 0xc8ea00a0L,
0xd6ad50a5L, 0xd26c4d12L, 0xdf2f6bcbL, 0xdbee767cL,
0xe3a1cbc1L, 0xe760d676L, 0xea23f0afL, 0xeee2ed18L,
0xf0a5bd1dL, 0xf464a0aaL, 0xf9278673L, 0xfde69bc4L,
0x89b8fd09L, 0x8d79e0beL, 0x803ac667L, 0x84fbdbd0L,
0x9abc8bd5L, 0x9e7d9662L, 0x933eb0bbL, 0x97ffad0cL,
0xafb010b1L, 0xab710d06L, 0xa6322bdfL, 0xa2f33668L,
0xbcb4666dL, 0xb8757bdaL, 0xb5365d03L, 0xb1f740b4L
};
/*-------------------------------------------------------------*/
/*--- end crctable.c ---*/
/*-------------------------------------------------------------*/

646
nc_test4/filter_test/decompress.c Normal file
View File

@ -0,0 +1,646 @@
/*-------------------------------------------------------------*/
/*--- Decompression machinery ---*/
/*--- decompress.c ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#include "bzlib_private.h"
/*---------------------------------------------------*/
static
void makeMaps_d ( DState* s )
{
Int32 i;
s->nInUse = 0;
for (i = 0; i < 256; i++)
if (s->inUse[i]) {
s->seqToUnseq[s->nInUse] = i;
s->nInUse++;
}
}
/*---------------------------------------------------*/
#define RETURN(rrr) \
{ retVal = rrr; goto save_state_and_return; };
#define GET_BITS(lll,vvv,nnn) \
case lll: s->state = lll; \
while (True) { \
if (s->bsLive >= nnn) { \
UInt32 v; \
v = (s->bsBuff >> \
(s->bsLive-nnn)) & ((1 << nnn)-1); \
s->bsLive -= nnn; \
vvv = v; \
break; \
} \
if (s->strm->avail_in == 0) RETURN(BZ_OK); \
s->bsBuff \
= (s->bsBuff << 8) | \
((UInt32) \
(*((UChar*)(s->strm->next_in)))); \
s->bsLive += 8; \
s->strm->next_in++; \
s->strm->avail_in--; \
s->strm->total_in_lo32++; \
if (s->strm->total_in_lo32 == 0) \
s->strm->total_in_hi32++; \
}
#define GET_UCHAR(lll,uuu) \
GET_BITS(lll,uuu,8)
#define GET_BIT(lll,uuu) \
GET_BITS(lll,uuu,1)
/*---------------------------------------------------*/
#define GET_MTF_VAL(label1,label2,lval) \
{ \
if (groupPos == 0) { \
groupNo++; \
if (groupNo >= nSelectors) \
RETURN(BZ_DATA_ERROR); \
groupPos = BZ_G_SIZE; \
gSel = s->selector[groupNo]; \
gMinlen = s->minLens[gSel]; \
gLimit = &(s->limit[gSel][0]); \
gPerm = &(s->perm[gSel][0]); \
gBase = &(s->base[gSel][0]); \
} \
groupPos--; \
zn = gMinlen; \
GET_BITS(label1, zvec, zn); \
while (1) { \
if (zn > 20 /* the longest code */) \
RETURN(BZ_DATA_ERROR); \
if (zvec <= gLimit[zn]) break; \
zn++; \
GET_BIT(label2, zj); \
zvec = (zvec << 1) | zj; \
}; \
if (zvec - gBase[zn] < 0 \
|| zvec - gBase[zn] >= BZ_MAX_ALPHA_SIZE) \
RETURN(BZ_DATA_ERROR); \
lval = gPerm[zvec - gBase[zn]]; \
}
/*---------------------------------------------------*/
Int32 BZ2_decompress ( DState* s )
{
UChar uc;
Int32 retVal;
Int32 minLen, maxLen;
bz_stream* strm = s->strm;
/* stuff that needs to be saved/restored */
Int32 i;
Int32 j;
Int32 t;
Int32 alphaSize;
Int32 nGroups;
Int32 nSelectors;
Int32 EOB;
Int32 groupNo;
Int32 groupPos;
Int32 nextSym;
Int32 nblockMAX;
Int32 nblock;
Int32 es;
Int32 N;
Int32 curr;
Int32 zt;
Int32 zn;
Int32 zvec;
Int32 zj;
Int32 gSel;
Int32 gMinlen;
Int32* gLimit;
Int32* gBase;
Int32* gPerm;
if (s->state == BZ_X_MAGIC_1) {
/*initialise the save area*/
s->save_i = 0;
s->save_j = 0;
s->save_t = 0;
s->save_alphaSize = 0;
s->save_nGroups = 0;
s->save_nSelectors = 0;
s->save_EOB = 0;
s->save_groupNo = 0;
s->save_groupPos = 0;
s->save_nextSym = 0;
s->save_nblockMAX = 0;
s->save_nblock = 0;
s->save_es = 0;
s->save_N = 0;
s->save_curr = 0;
s->save_zt = 0;
s->save_zn = 0;
s->save_zvec = 0;
s->save_zj = 0;
s->save_gSel = 0;
s->save_gMinlen = 0;
s->save_gLimit = NULL;
s->save_gBase = NULL;
s->save_gPerm = NULL;
}
/*restore from the save area*/
i = s->save_i;
j = s->save_j;
t = s->save_t;
alphaSize = s->save_alphaSize;
nGroups = s->save_nGroups;
nSelectors = s->save_nSelectors;
EOB = s->save_EOB;
groupNo = s->save_groupNo;
groupPos = s->save_groupPos;
nextSym = s->save_nextSym;
nblockMAX = s->save_nblockMAX;
nblock = s->save_nblock;
es = s->save_es;
N = s->save_N;
curr = s->save_curr;
zt = s->save_zt;
zn = s->save_zn;
zvec = s->save_zvec;
zj = s->save_zj;
gSel = s->save_gSel;
gMinlen = s->save_gMinlen;
gLimit = s->save_gLimit;
gBase = s->save_gBase;
gPerm = s->save_gPerm;
retVal = BZ_OK;
switch (s->state) {
GET_UCHAR(BZ_X_MAGIC_1, uc);
if (uc != BZ_HDR_B) RETURN(BZ_DATA_ERROR_MAGIC);
GET_UCHAR(BZ_X_MAGIC_2, uc);
if (uc != BZ_HDR_Z) RETURN(BZ_DATA_ERROR_MAGIC);
GET_UCHAR(BZ_X_MAGIC_3, uc)
if (uc != BZ_HDR_h) RETURN(BZ_DATA_ERROR_MAGIC);
GET_BITS(BZ_X_MAGIC_4, s->blockSize100k, 8)
if (s->blockSize100k < (BZ_HDR_0 + 1) ||
s->blockSize100k > (BZ_HDR_0 + 9)) RETURN(BZ_DATA_ERROR_MAGIC);
s->blockSize100k -= BZ_HDR_0;
if (s->smallDecompress) {
s->ll16 = BZALLOC( s->blockSize100k * 100000 * sizeof(UInt16) );
s->ll4 = BZALLOC(
((1 + s->blockSize100k * 100000) >> 1) * sizeof(UChar)
);
if (s->ll16 == NULL || s->ll4 == NULL) RETURN(BZ_MEM_ERROR);
} else {
s->tt = BZALLOC( s->blockSize100k * 100000 * sizeof(Int32) );
if (s->tt == NULL) RETURN(BZ_MEM_ERROR);
}
GET_UCHAR(BZ_X_BLKHDR_1, uc);
if (uc == 0x17) goto endhdr_2;
if (uc != 0x31) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_BLKHDR_2, uc);
if (uc != 0x41) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_BLKHDR_3, uc);
if (uc != 0x59) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_BLKHDR_4, uc);
if (uc != 0x26) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_BLKHDR_5, uc);
if (uc != 0x53) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_BLKHDR_6, uc);
if (uc != 0x59) RETURN(BZ_DATA_ERROR);
s->currBlockNo++;
if (s->verbosity >= 2)
VPrintf1 ( "\n [%d: huff+mtf ", s->currBlockNo );
s->storedBlockCRC = 0;
GET_UCHAR(BZ_X_BCRC_1, uc);
s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_BCRC_2, uc);
s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_BCRC_3, uc);
s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_BCRC_4, uc);
s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc);
GET_BITS(BZ_X_RANDBIT, s->blockRandomised, 1);
s->origPtr = 0;
GET_UCHAR(BZ_X_ORIGPTR_1, uc);
s->origPtr = (s->origPtr << 8) | ((Int32)uc);
GET_UCHAR(BZ_X_ORIGPTR_2, uc);
s->origPtr = (s->origPtr << 8) | ((Int32)uc);
GET_UCHAR(BZ_X_ORIGPTR_3, uc);
s->origPtr = (s->origPtr << 8) | ((Int32)uc);
if (s->origPtr < 0)
RETURN(BZ_DATA_ERROR);
if (s->origPtr > 10 + 100000*s->blockSize100k)
RETURN(BZ_DATA_ERROR);
/*--- Receive the mapping table ---*/
for (i = 0; i < 16; i++) {
GET_BIT(BZ_X_MAPPING_1, uc);
if (uc == 1)
s->inUse16[i] = True; else
s->inUse16[i] = False;
}
for (i = 0; i < 256; i++) s->inUse[i] = False;
for (i = 0; i < 16; i++)
if (s->inUse16[i])
for (j = 0; j < 16; j++) {
GET_BIT(BZ_X_MAPPING_2, uc);
if (uc == 1) s->inUse[i * 16 + j] = True;
}
makeMaps_d ( s );
if (s->nInUse == 0) RETURN(BZ_DATA_ERROR);
alphaSize = s->nInUse+2;
/*--- Now the selectors ---*/
GET_BITS(BZ_X_SELECTOR_1, nGroups, 3);
if (nGroups < 2 || nGroups > 6) RETURN(BZ_DATA_ERROR);
GET_BITS(BZ_X_SELECTOR_2, nSelectors, 15);
if (nSelectors < 1) RETURN(BZ_DATA_ERROR);
for (i = 0; i < nSelectors; i++) {
j = 0;
while (True) {
GET_BIT(BZ_X_SELECTOR_3, uc);
if (uc == 0) break;
j++;
if (j >= nGroups) RETURN(BZ_DATA_ERROR);
}
s->selectorMtf[i] = j;
}
/*--- Undo the MTF values for the selectors. ---*/
{
UChar pos[BZ_N_GROUPS], tmp, v;
for (v = 0; v < nGroups; v++) pos[v] = v;
for (i = 0; i < nSelectors; i++) {
v = s->selectorMtf[i];
tmp = pos[v];
while (v > 0) { pos[v] = pos[v-1]; v--; }
pos[0] = tmp;
s->selector[i] = tmp;
}
}
/*--- Now the coding tables ---*/
for (t = 0; t < nGroups; t++) {
GET_BITS(BZ_X_CODING_1, curr, 5);
for (i = 0; i < alphaSize; i++) {
while (True) {
if (curr < 1 || curr > 20) RETURN(BZ_DATA_ERROR);
GET_BIT(BZ_X_CODING_2, uc);
if (uc == 0) break;
GET_BIT(BZ_X_CODING_3, uc);
if (uc == 0) curr++; else curr--;
}
s->len[t][i] = curr;
}
}
/*--- Create the Huffman decoding tables ---*/
for (t = 0; t < nGroups; t++) {
minLen = 32;
maxLen = 0;
for (i = 0; i < alphaSize; i++) {
if (s->len[t][i] > maxLen) maxLen = s->len[t][i];
if (s->len[t][i] < minLen) minLen = s->len[t][i];
}
BZ2_hbCreateDecodeTables (
&(s->limit[t][0]),
&(s->base[t][0]),
&(s->perm[t][0]),
&(s->len[t][0]),
minLen, maxLen, alphaSize
);
s->minLens[t] = minLen;
}
/*--- Now the MTF values ---*/
EOB = s->nInUse+1;
nblockMAX = 100000 * s->blockSize100k;
groupNo = -1;
groupPos = 0;
for (i = 0; i <= 255; i++) s->unzftab[i] = 0;
/*-- MTF init --*/
{
Int32 ii, jj, kk;
kk = MTFA_SIZE-1;
for (ii = 256 / MTFL_SIZE - 1; ii >= 0; ii--) {
for (jj = MTFL_SIZE-1; jj >= 0; jj--) {
s->mtfa[kk] = (UChar)(ii * MTFL_SIZE + jj);
kk--;
}
s->mtfbase[ii] = kk + 1;
}
}
/*-- end MTF init --*/
nblock = 0;
GET_MTF_VAL(BZ_X_MTF_1, BZ_X_MTF_2, nextSym);
while (True) {
if (nextSym == EOB) break;
if (nextSym == BZ_RUNA || nextSym == BZ_RUNB) {
es = -1;
N = 1;
do {
/* Check that N doesn't get too big, so that es doesn't
go negative. The maximum value that can be
RUNA/RUNB encoded is equal to the block size (post
the initial RLE), viz, 900k, so bounding N at 2
million should guard against overflow without
rejecting any legitimate inputs. */
if (N >= 2*1024*1024) RETURN(BZ_DATA_ERROR);
if (nextSym == BZ_RUNA) es = es + (0+1) * N; else
if (nextSym == BZ_RUNB) es = es + (1+1) * N;
N = N * 2;
GET_MTF_VAL(BZ_X_MTF_3, BZ_X_MTF_4, nextSym);
}
while (nextSym == BZ_RUNA || nextSym == BZ_RUNB);
es++;
uc = s->seqToUnseq[ s->mtfa[s->mtfbase[0]] ];
s->unzftab[uc] += es;
if (s->smallDecompress)
while (es > 0) {
if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR);
s->ll16[nblock] = (UInt16)uc;
nblock++;
es--;
}
else
while (es > 0) {
if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR);
s->tt[nblock] = (UInt32)uc;
nblock++;
es--;
};
continue;
} else {
if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR);
/*-- uc = MTF ( nextSym-1 ) --*/
{
Int32 ii, jj, kk, pp, lno, off;
UInt32 nn;
nn = (UInt32)(nextSym - 1);
if (nn < MTFL_SIZE) {
/* avoid general-case expense */
pp = s->mtfbase[0];
uc = s->mtfa[pp+nn];
while (nn > 3) {
Int32 z = pp+nn;
s->mtfa[(z) ] = s->mtfa[(z)-1];
s->mtfa[(z)-1] = s->mtfa[(z)-2];
s->mtfa[(z)-2] = s->mtfa[(z)-3];
s->mtfa[(z)-3] = s->mtfa[(z)-4];
nn -= 4;
}
while (nn > 0) {
s->mtfa[(pp+nn)] = s->mtfa[(pp+nn)-1]; nn--;
};
s->mtfa[pp] = uc;
} else {
/* general case */
lno = nn / MTFL_SIZE;
off = nn % MTFL_SIZE;
pp = s->mtfbase[lno] + off;
uc = s->mtfa[pp];
while (pp > s->mtfbase[lno]) {
s->mtfa[pp] = s->mtfa[pp-1]; pp--;
};
s->mtfbase[lno]++;
while (lno > 0) {
s->mtfbase[lno]--;
s->mtfa[s->mtfbase[lno]]
= s->mtfa[s->mtfbase[lno-1] + MTFL_SIZE - 1];
lno--;
}
s->mtfbase[0]--;
s->mtfa[s->mtfbase[0]] = uc;
if (s->mtfbase[0] == 0) {
kk = MTFA_SIZE-1;
for (ii = 256 / MTFL_SIZE-1; ii >= 0; ii--) {
for (jj = MTFL_SIZE-1; jj >= 0; jj--) {
s->mtfa[kk] = s->mtfa[s->mtfbase[ii] + jj];
kk--;
}
s->mtfbase[ii] = kk + 1;
}
}
}
}
/*-- end uc = MTF ( nextSym-1 ) --*/
s->unzftab[s->seqToUnseq[uc]]++;
if (s->smallDecompress)
s->ll16[nblock] = (UInt16)(s->seqToUnseq[uc]); else
s->tt[nblock] = (UInt32)(s->seqToUnseq[uc]);
nblock++;
GET_MTF_VAL(BZ_X_MTF_5, BZ_X_MTF_6, nextSym);
continue;
}
}
/* Now we know what nblock is, we can do a better sanity
check on s->origPtr.
*/
if (s->origPtr < 0 || s->origPtr >= nblock)
RETURN(BZ_DATA_ERROR);
/*-- Set up cftab to facilitate generation of T^(-1) --*/
/* Check: unzftab entries in range. */
for (i = 0; i <= 255; i++) {
if (s->unzftab[i] < 0 || s->unzftab[i] > nblock)
RETURN(BZ_DATA_ERROR);
}
/* Actually generate cftab. */
s->cftab[0] = 0;
for (i = 1; i <= 256; i++) s->cftab[i] = s->unzftab[i-1];
for (i = 1; i <= 256; i++) s->cftab[i] += s->cftab[i-1];
/* Check: cftab entries in range. */
for (i = 0; i <= 256; i++) {
if (s->cftab[i] < 0 || s->cftab[i] > nblock) {
/* s->cftab[i] can legitimately be == nblock */
RETURN(BZ_DATA_ERROR);
}
}
/* Check: cftab entries non-descending. */
for (i = 1; i <= 256; i++) {
if (s->cftab[i-1] > s->cftab[i]) {
RETURN(BZ_DATA_ERROR);
}
}
s->state_out_len = 0;
s->state_out_ch = 0;
BZ_INITIALISE_CRC ( s->calculatedBlockCRC );
s->state = BZ_X_OUTPUT;
if (s->verbosity >= 2) VPrintf0 ( "rt+rld" );
if (s->smallDecompress) {
/*-- Make a copy of cftab, used in generation of T --*/
for (i = 0; i <= 256; i++) s->cftabCopy[i] = s->cftab[i];
/*-- compute the T vector --*/
for (i = 0; i < nblock; i++) {
uc = (UChar)(s->ll16[i]);
SET_LL(i, s->cftabCopy[uc]);
s->cftabCopy[uc]++;
}
/*-- Compute T^(-1) by pointer reversal on T --*/
i = s->origPtr;
j = GET_LL(i);
do {
Int32 tmp = GET_LL(j);
SET_LL(j, i);
i = j;
j = tmp;
}
while (i != s->origPtr);
s->tPos = s->origPtr;
s->nblock_used = 0;
if (s->blockRandomised) {
BZ_RAND_INIT_MASK;
BZ_GET_SMALL(s->k0); s->nblock_used++;
BZ_RAND_UPD_MASK; s->k0 ^= BZ_RAND_MASK;
} else {
BZ_GET_SMALL(s->k0); s->nblock_used++;
}
} else {
/*-- compute the T^(-1) vector --*/
for (i = 0; i < nblock; i++) {
uc = (UChar)(s->tt[i] & 0xff);
s->tt[s->cftab[uc]] |= (i << 8);
s->cftab[uc]++;
}
s->tPos = s->tt[s->origPtr] >> 8;
s->nblock_used = 0;
if (s->blockRandomised) {
BZ_RAND_INIT_MASK;
BZ_GET_FAST(s->k0); s->nblock_used++;
BZ_RAND_UPD_MASK; s->k0 ^= BZ_RAND_MASK;
} else {
BZ_GET_FAST(s->k0); s->nblock_used++;
}
}
RETURN(BZ_OK);
endhdr_2:
GET_UCHAR(BZ_X_ENDHDR_2, uc);
if (uc != 0x72) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_ENDHDR_3, uc);
if (uc != 0x45) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_ENDHDR_4, uc);
if (uc != 0x38) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_ENDHDR_5, uc);
if (uc != 0x50) RETURN(BZ_DATA_ERROR);
GET_UCHAR(BZ_X_ENDHDR_6, uc);
if (uc != 0x90) RETURN(BZ_DATA_ERROR);
s->storedCombinedCRC = 0;
GET_UCHAR(BZ_X_CCRC_1, uc);
s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_CCRC_2, uc);
s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_CCRC_3, uc);
s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc);
GET_UCHAR(BZ_X_CCRC_4, uc);
s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc);
s->state = BZ_X_IDLE;
RETURN(BZ_STREAM_END);
default: AssertH ( False, 4001 );
}
AssertH ( False, 4002 );
save_state_and_return:
s->save_i = i;
s->save_j = j;
s->save_t = t;
s->save_alphaSize = alphaSize;
s->save_nGroups = nGroups;
s->save_nSelectors = nSelectors;
s->save_EOB = EOB;
s->save_groupNo = groupNo;
s->save_groupPos = groupPos;
s->save_nextSym = nextSym;
s->save_nblockMAX = nblockMAX;
s->save_nblock = nblock;
s->save_es = es;
s->save_N = N;
s->save_curr = curr;
s->save_zt = zt;
s->save_zn = zn;
s->save_zvec = zvec;
s->save_zj = zj;
s->save_gSel = gSel;
s->save_gMinlen = gMinlen;
s->save_gLimit = gLimit;
s->save_gBase = gBase;
s->save_gPerm = gPerm;
return retVal;
}
/*-------------------------------------------------------------*/
/*--- end decompress.c ---*/
/*-------------------------------------------------------------*/

5
nc_test4/filter_test/fake.c Normal file
View File

@ -0,0 +1,5 @@
int
main()
{
return 0;
}

88
nc_test4/filter_test/filtered.cdl Normal file
View File

@ -0,0 +1,88 @@
netcdf filtered {
dimensions:
dim0 = 4 ;
dim1 = 4 ;
dim2 = 4 ;
dim3 = 4 ;
// global attributes:
:_Format = "netCDF-4" ;
group: g {
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
var:_Filter = "307,9,4";
var:_NoFill = "true" ;
// group attributes:
data:
var =
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23,
24, 25, 26, 27,
28, 29, 30, 31,
32, 33, 34, 35,
36, 37, 38, 39,
40, 41, 42, 43,
44, 45, 46, 47,
48, 49, 50, 51,
52, 53, 54, 55,
56, 57, 58, 59,
60, 61, 62, 63,
64, 65, 66, 67,
68, 69, 70, 71,
72, 73, 74, 75,
76, 77, 78, 79,
80, 81, 82, 83,
84, 85, 86, 87,
88, 89, 90, 91,
92, 93, 94, 95,
96, 97, 98, 99,
100, 101, 102, 103,
104, 105, 106, 107,
108, 109, 110, 111,
112, 113, 114, 115,
116, 117, 118, 119,
120, 121, 122, 123,
124, 125, 126, 127,
128, 129, 130, 131,
132, 133, 134, 135,
136, 137, 138, 139,
140, 141, 142, 143,
144, 145, 146, 147,
148, 149, 150, 151,
152, 153, 154, 155,
156, 157, 158, 159,
160, 161, 162, 163,
164, 165, 166, 167,
168, 169, 170, 171,
172, 173, 174, 175,
176, 177, 178, 179,
180, 181, 182, 183,
184, 185, 186, 187,
188, 189, 190, 191,
192, 193, 194, 195,
196, 197, 198, 199,
200, 201, 202, 203,
204, 205, 206, 207,
208, 209, 210, 211,
212, 213, 214, 215,
216, 217, 218, 219,
220, 221, 222, 223,
224, 225, 226, 227,
228, 229, 230, 231,
232, 233, 234, 235,
236, 237, 238, 239,
240, 241, 242, 243,
244, 245, 246, 247,
248, 249, 250, 251,
252, 253, 254, 255 ;
} // group g
}

12
nc_test4/filter_test/h5bzip2.h Normal file
View File

@ -0,0 +1,12 @@
#include "bzlib.h"
/* use an integer greater than 256 to be id of the registered filter. */
#define H5Z_FILTER_BZIP2 307
const H5Z_class2_t H5Z_BZIP2[1];
/* declare a filter function */
size_t H5Z_filter_bzip2(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],
size_t nbytes,size_t *buf_size,void**buf);

18
nc_test4/filter_test/h5test.h Normal file
View File

@ -0,0 +1,18 @@
#include "bzlib.h"
/* use an integer greater than 256 to be id of the registered filter. */
#define H5Z_FILTER_TEST 32768
/* Define the test cases */
typedef enum H5testcase {
TC_NONE = 0,
TC_ENDIAN = 1,
} H5testcase;
const H5Z_class2_t H5Z_TEST[1];
/* declare a filter function */
extern size_t H5Z_filter_test(unsigned flags,size_t cd_nelmts,const unsigned cd_values[],
size_t nbytes,size_t *buf_size,void**buf);

205
nc_test4/filter_test/huffman.c Normal file
View File

@ -0,0 +1,205 @@
/*-------------------------------------------------------------*/
/*--- Huffman coding low-level stuff ---*/
/*--- huffman.c ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#include "bzlib_private.h"
/*---------------------------------------------------*/
#define WEIGHTOF(zz0) ((zz0) & 0xffffff00)
#define DEPTHOF(zz1) ((zz1) & 0x000000ff)
#define MYMAX(zz2,zz3) ((zz2) > (zz3) ? (zz2) : (zz3))
#define ADDWEIGHTS(zw1,zw2) \
(WEIGHTOF(zw1)+WEIGHTOF(zw2)) | \
(1 + MYMAX(DEPTHOF(zw1),DEPTHOF(zw2)))
#define UPHEAP(z) \
{ \
Int32 zz, tmp; \
zz = z; tmp = heap[zz]; \
while (weight[tmp] < weight[heap[zz >> 1]]) { \
heap[zz] = heap[zz >> 1]; \
zz >>= 1; \
} \
heap[zz] = tmp; \
}
#define DOWNHEAP(z) \
{ \
Int32 zz, yy, tmp; \
zz = z; tmp = heap[zz]; \
while (True) { \
yy = zz << 1; \
if (yy > nHeap) break; \
if (yy < nHeap && \
weight[heap[yy+1]] < weight[heap[yy]]) \
yy++; \
if (weight[tmp] < weight[heap[yy]]) break; \
heap[zz] = heap[yy]; \
zz = yy; \
} \
heap[zz] = tmp; \
}
/*---------------------------------------------------*/
void BZ2_hbMakeCodeLengths ( UChar *len,
Int32 *freq,
Int32 alphaSize,
Int32 maxLen )
{
/*--
Nodes and heap entries run from 1. Entry 0
for both the heap and nodes is a sentinel.
--*/
Int32 nNodes, nHeap, n1, n2, i, j, k;
Bool tooLong;
Int32 heap [ BZ_MAX_ALPHA_SIZE + 2 ];
Int32 weight [ BZ_MAX_ALPHA_SIZE * 2 ];
Int32 parent [ BZ_MAX_ALPHA_SIZE * 2 ];
for (i = 0; i < alphaSize; i++)
weight[i+1] = (freq[i] == 0 ? 1 : freq[i]) << 8;
while (True) {
nNodes = alphaSize;
nHeap = 0;
heap[0] = 0;
weight[0] = 0;
parent[0] = -2;
for (i = 1; i <= alphaSize; i++) {
parent[i] = -1;
nHeap++;
heap[nHeap] = i;
UPHEAP(nHeap);
}
AssertH( nHeap < (BZ_MAX_ALPHA_SIZE+2), 2001 );
while (nHeap > 1) {
n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1);
nNodes++;
parent[n1] = parent[n2] = nNodes;
weight[nNodes] = ADDWEIGHTS(weight[n1], weight[n2]);
parent[nNodes] = -1;
nHeap++;
heap[nHeap] = nNodes;
UPHEAP(nHeap);
}
AssertH( nNodes < (BZ_MAX_ALPHA_SIZE * 2), 2002 );
tooLong = False;
for (i = 1; i <= alphaSize; i++) {
j = 0;
k = i;
while (parent[k] >= 0) { k = parent[k]; j++; }
len[i-1] = j;
if (j > maxLen) tooLong = True;
}
if (! tooLong) break;
/* 17 Oct 04: keep-going condition for the following loop used
to be 'i < alphaSize', which missed the last element,
theoretically leading to the possibility of the compressor
looping. However, this count-scaling step is only needed if
one of the generated Huffman code words is longer than
maxLen, which up to and including version 1.0.2 was 20 bits,
which is extremely unlikely. In version 1.0.3 maxLen was
changed to 17 bits, which has minimal effect on compression
ratio, but does mean this scaling step is used from time to
time, enough to verify that it works.
This means that bzip2-1.0.3 and later will only produce
Huffman codes with a maximum length of 17 bits. However, in
order to preserve backwards compatibility with bitstreams
produced by versions pre-1.0.3, the decompressor must still
handle lengths of up to 20. */
for (i = 1; i <= alphaSize; i++) {
j = weight[i] >> 8;
j = 1 + (j / 2);
weight[i] = j << 8;
}
}
}
/*---------------------------------------------------*/
void BZ2_hbAssignCodes ( Int32 *code,
UChar *length,
Int32 minLen,
Int32 maxLen,
Int32 alphaSize )
{
Int32 n, vec, i;
vec = 0;
for (n = minLen; n <= maxLen; n++) {
for (i = 0; i < alphaSize; i++)
if (length[i] == n) { code[i] = vec; vec++; };
vec <<= 1;
}
}
/*---------------------------------------------------*/
void BZ2_hbCreateDecodeTables ( Int32 *limit,
Int32 *base,
Int32 *perm,
UChar *length,
Int32 minLen,
Int32 maxLen,
Int32 alphaSize )
{
Int32 pp, i, j, vec;
pp = 0;
for (i = minLen; i <= maxLen; i++)
for (j = 0; j < alphaSize; j++)
if (length[j] == i) { perm[pp] = j; pp++; };
for (i = 0; i < BZ_MAX_CODE_LEN; i++) base[i] = 0;
for (i = 0; i < alphaSize; i++) base[length[i]+1]++;
for (i = 1; i < BZ_MAX_CODE_LEN; i++) base[i] += base[i-1];
for (i = 0; i < BZ_MAX_CODE_LEN; i++) limit[i] = 0;
vec = 0;
for (i = minLen; i <= maxLen; i++) {
vec += (base[i+1] - base[i]);
limit[i] = vec-1;
vec <<= 1;
}
for (i = minLen + 1; i <= maxLen; i++)
base[i] = ((limit[i-1] + 1) << 1) - base[i];
}
/*-------------------------------------------------------------*/
/*--- end huffman.c ---*/
/*-------------------------------------------------------------*/

84
nc_test4/filter_test/randtable.c Normal file
View File

@ -0,0 +1,84 @@
/*-------------------------------------------------------------*/
/*--- Table for randomising repetitive blocks ---*/
/*--- randtable.c ---*/
/*-------------------------------------------------------------*/
/* ------------------------------------------------------------------
This file is part of bzip2/libbzip2, a program and library for
lossless, block-sorting data compression.
bzip2/libbzip2 version 1.0.6 of 6 September 2010
Copyright (C) 1996-2010 Julian Seward <jseward@bzip.org>
Please read the WARNING, DISCLAIMER and PATENTS sections in the
README file.
This program is released under the terms of the license contained
in the file LICENSE.
------------------------------------------------------------------ */
#include "bzlib_private.h"
/*---------------------------------------------*/
Int32 BZ2_rNums[512] = {
619, 720, 127, 481, 931, 816, 813, 233, 566, 247,
985, 724, 205, 454, 863, 491, 741, 242, 949, 214,
733, 859, 335, 708, 621, 574, 73, 654, 730, 472,
419, 436, 278, 496, 867, 210, 399, 680, 480, 51,
878, 465, 811, 169, 869, 675, 611, 697, 867, 561,
862, 687, 507, 283, 482, 129, 807, 591, 733, 623,
150, 238, 59, 379, 684, 877, 625, 169, 643, 105,
170, 607, 520, 932, 727, 476, 693, 425, 174, 647,
73, 122, 335, 530, 442, 853, 695, 249, 445, 515,
909, 545, 703, 919, 874, 474, 882, 500, 594, 612,
641, 801, 220, 162, 819, 984, 589, 513, 495, 799,
161, 604, 958, 533, 221, 400, 386, 867, 600, 782,
382, 596, 414, 171, 516, 375, 682, 485, 911, 276,
98, 553, 163, 354, 666, 933, 424, 341, 533, 870,
227, 730, 475, 186, 263, 647, 537, 686, 600, 224,
469, 68, 770, 919, 190, 373, 294, 822, 808, 206,
184, 943, 795, 384, 383, 461, 404, 758, 839, 887,
715, 67, 618, 276, 204, 918, 873, 777, 604, 560,
951, 160, 578, 722, 79, 804, 96, 409, 713, 940,
652, 934, 970, 447, 318, 353, 859, 672, 112, 785,
645, 863, 803, 350, 139, 93, 354, 99, 820, 908,
609, 772, 154, 274, 580, 184, 79, 626, 630, 742,
653, 282, 762, 623, 680, 81, 927, 626, 789, 125,
411, 521, 938, 300, 821, 78, 343, 175, 128, 250,
170, 774, 972, 275, 999, 639, 495, 78, 352, 126,
857, 956, 358, 619, 580, 124, 737, 594, 701, 612,
669, 112, 134, 694, 363, 992, 809, 743, 168, 974,
944, 375, 748, 52, 600, 747, 642, 182, 862, 81,
344, 805, 988, 739, 511, 655, 814, 334, 249, 515,
897, 955, 664, 981, 649, 113, 974, 459, 893, 228,
433, 837, 553, 268, 926, 240, 102, 654, 459, 51,
686, 754, 806, 760, 493, 403, 415, 394, 687, 700,
946, 670, 656, 610, 738, 392, 760, 799, 887, 653,
978, 321, 576, 617, 626, 502, 894, 679, 243, 440,
680, 879, 194, 572, 640, 724, 926, 56, 204, 700,
707, 151, 457, 449, 797, 195, 791, 558, 945, 679,
297, 59, 87, 824, 713, 663, 412, 693, 342, 606,
134, 108, 571, 364, 631, 212, 174, 643, 304, 329,
343, 97, 430, 751, 497, 314, 983, 374, 822, 928,
140, 206, 73, 263, 980, 736, 876, 478, 430, 305,
170, 514, 364, 692, 829, 82, 855, 953, 676, 246,
369, 970, 294, 750, 807, 827, 150, 790, 288, 923,
804, 378, 215, 828, 592, 281, 565, 555, 710, 82,
896, 831, 547, 261, 524, 462, 293, 465, 502, 56,
661, 821, 976, 991, 658, 869, 905, 758, 745, 193,
768, 550, 608, 933, 378, 286, 215, 979, 792, 961,
61, 688, 793, 644, 986, 403, 106, 366, 905, 644,
372, 567, 466, 434, 645, 210, 389, 550, 919, 135,
780, 773, 635, 389, 707, 100, 626, 958, 165, 504,
920, 176, 193, 713, 857, 265, 203, 50, 668, 108,
645, 990, 626, 197, 510, 357, 358, 850, 858, 364,
936, 638
};
/*-------------------------------------------------------------*/
/*--- end randtable.c ---*/
/*-------------------------------------------------------------*/

98
nc_test4/filter_test/ref_bzip2.c Normal file
View File

@ -0,0 +1,98 @@
#include <stdio.h>
#include <stdlib.h>
#include <netcdf.h>
static size_t var_chunksizes[4] = {4, 4, 4, 4} ;
static unsigned int var_filterparams[1] = {9U} ;
void
check_err(const int stat, const int line, const char *file) {
if (stat != NC_NOERR) {
(void)fprintf(stderr,"line %d of %s: %s\n", line, file, nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
int
main() {/* create bzip2.nc */
int stat; /* return status */
int ncid; /* netCDF id */
/* group ids */
int bzip2_grp;
/* dimension ids */
int dim0_dim;
int dim1_dim;
int dim2_dim;
int dim3_dim;
/* dimension lengths */
size_t dim0_len = 4;
size_t dim1_len = 4;
size_t dim2_len = 4;
size_t dim3_len = 4;
/* variable ids */
int var_id;
/* rank (number of dimensions) for each variable */
# define RANK_var 4
/* variable shapes */
int var_dims[RANK_var];
/* enter define mode */
stat = nc_create("bzip2.nc", NC_CLOBBER|NC_NETCDF4, &ncid);
check_err(stat,__LINE__,__FILE__);
stat = nc_put_att_text(ncid, NC_GLOBAL, "_Format", 1, "netCDF-4");
check_err(stat,__LINE__,__FILE__);
bzip2_grp = ncid;
/* define dimensions */
stat = nc_def_dim(bzip2_grp, "dim0", dim0_len, &dim0_dim);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_dim(bzip2_grp, "dim1", dim1_len, &dim1_dim);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_dim(bzip2_grp, "dim2", dim2_len, &dim2_dim);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_dim(bzip2_grp, "dim3", dim3_len, &dim3_dim);
check_err(stat,__LINE__,__FILE__);
/* define variables */
var_dims[0] = dim0_dim;
var_dims[1] = dim1_dim;
var_dims[2] = dim2_dim;
var_dims[3] = dim3_dim;
stat = nc_def_var(bzip2_grp, "var", NC_FLOAT, RANK_var, var_dims, &var_id);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_var_chunking(bzip2_grp, var_id, NC_CHUNKED, var_chunksizes);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_var_fill(bzip2_grp, var_id, NC_NOFILL, NULL);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_var_filter(bzip2_grp, var_id, 307, 1, var_filterparams);
check_err(stat,__LINE__,__FILE__);
/* leave define mode */
stat = nc_enddef (bzip2_grp);
check_err(stat,__LINE__,__FILE__);
/* assign variable data */
{
float var_data[256] = {((float)0), ((float)1), ((float)2), ((float)3), ((float)4), ((float)5), ((float)6), ((float)7), ((float)8), ((float)9), ((float)10), ((float)11), ((float)12), ((float)13), ((float)14), ((float)15), ((float)16), ((float)17), ((float)18), ((float)19), ((float)20), ((float)21), ((float)22), ((float)23), ((float)24), ((float)25), ((float)26), ((float)27), ((float)28), ((float)29), ((float)30), ((float)31), ((float)32), ((float)33), ((float)34), ((float)35), ((float)36), ((float)37), ((float)38), ((float)39), ((float)40), ((float)41), ((float)42), ((float)43), ((float)44), ((float)45), ((float)46), ((float)47), ((float)48), ((float)49), ((float)50), ((float)51), ((float)52), ((float)53), ((float)54), ((float)55), ((float)56), ((float)57), ((float)58), ((float)59), ((float)60), ((float)61), ((float)62), ((float)63), ((float)64), ((float)65), ((float)66), ((float)67), ((float)68), ((float)69), ((float)70), ((float)71), ((float)72), ((float)73), ((float)74), ((float)75), ((float)76), ((float)77), ((float)78), ((float)79), ((float)80), ((float)81), ((float)82), ((float)83), ((float)84), ((float)85), ((float)86), ((float)87), ((float)88), ((float)89), ((float)90), ((float)91), ((float)92), ((float)93), ((float)94), ((float)95), ((float)96), ((float)97), ((float)98), ((float)99), ((float)100), ((float)101), ((float)102), ((float)103), ((float)104), ((float)105), ((float)106), ((float)107), ((float)108), ((float)109), ((float)110), ((float)111), ((float)112), ((float)113), ((float)114), ((float)115), ((float)116), ((float)117), ((float)118), ((float)119), ((float)120), ((float)121), ((float)122), ((float)123), ((float)124), ((float)125), ((float)126), ((float)127), ((float)128), ((float)129), ((float)130), ((float)131), ((float)132), ((float)133), ((float)134), ((float)135), ((float)136), ((float)137), ((float)138), ((float)139), ((float)140), ((float)141), ((float)142), ((float)143), ((float)144), ((float)145), ((float)146), ((float)147), ((float)148), ((float)149), ((float)150), ((float)151), ((float)152), ((float)153), ((float)154), ((float)155), ((float)156), ((float)157), ((float)158), ((float)159), ((float)160), ((float)161), ((float)162), ((float)163), ((float)164), ((float)165), ((float)166), ((float)167), ((float)168), ((float)169), ((float)170), ((float)171), ((float)172), ((float)173), ((float)174), ((float)175), ((float)176), ((float)177), ((float)178), ((float)179), ((float)180), ((float)181), ((float)182), ((float)183), ((float)184), ((float)185), ((float)186), ((float)187), ((float)188), ((float)189), ((float)190), ((float)191), ((float)192), ((float)193), ((float)194), ((float)195), ((float)196), ((float)197), ((float)198), ((float)199), ((float)200), ((float)201), ((float)202), ((float)203), ((float)204), ((float)205), ((float)206), ((float)207), ((float)208), ((float)209), ((float)210), ((float)211), ((float)212), ((float)213), ((float)214), ((float)215), ((float)216), ((float)217), ((float)218), ((float)219), ((float)220), ((float)221), ((float)222), ((float)223), ((float)224), ((float)225), ((float)226), ((float)227), ((float)228), ((float)229), ((float)230), ((float)231), ((float)232), ((float)233), ((float)234), ((float)235), ((float)236), ((float)237), ((float)238), ((float)239), ((float)240), ((float)241), ((float)242), ((float)243), ((float)244), ((float)245), ((float)246), ((float)247), ((float)248), ((float)249), ((float)250), ((float)251), ((float)252), ((float)253), ((float)254), ((float)255)} ;
size_t var_startset[4] = {0, 0, 0, 0} ;
size_t var_countset[4] = {4, 4, 4, 4};
stat = nc_put_vara(bzip2_grp, var_id, var_startset, var_countset, var_data);
check_err(stat,__LINE__,__FILE__);
}
stat = nc_close(bzip2_grp);
check_err(stat,__LINE__,__FILE__);
return 0;
}

493
nc_test4/filter_test/test_filter.c Normal file
View File

@ -0,0 +1,493 @@
/*
Copyright 2008, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "netcdf.h"
//#define BASELINE 1
#define USEFLOAT 1
/* Write using var1_T instead of var_T */
#undef VAR1
#define BZIP2_ID 307
#define BZIP2_LEVEL 9
#define MAXERRS 8
#define MAXPARAMS 32
/* Following 3 must be consistent */
#ifdef USEFLOAT
#define T float
#else
#define T int
#endif
#define NC_PUT_VAR1 nc_put_var1_float
#define NC_PUT_VAR nc_put_var_float
typedef enum XZIP { NOZIP=0, BZIP2=1} XZIP;
/* Created Meta-data
netcdf zip {
dimensions:
dim1 = .. ;
dim2 = ... ;
dim3 = ... ;
...
dimn = ... ;
variables:
int var(dim1, dim2, dim3,...dimn) ;
}
*/
#define MAXDIMS 8
#define DEFAULTACTUALDIMS 4
#define DEFAULTDIMSIZE 4
#define DEFAULTCHUNKSIZE 4
static size_t dimsize = DEFAULTDIMSIZE;
static size_t chunksize = DEFAULTCHUNKSIZE;
static size_t actualdims = DEFAULTACTUALDIMS;
static size_t pattern[MAXDIMS];
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static size_t dims[MAXDIMS];
static size_t chunks[MAXDIMS];
static int nerrs = 0;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static T* array = NULL;
static T* expected = NULL;
static unsigned int filterid = 0;
static size_t nparams = 0;
static unsigned int* params = NULL;
/* Forward */
static int test_bzip2(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static T expectedvalue(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
fflush(stderr);
exit(1);
}
return NC_NOERR;
}
#define CHECK(x) check(x,__LINE__)
static char*
filenamefor(XZIP encoder)
{
static char testfile[2048];
snprintf(testfile,sizeof(testfile),"%s.nc",
(encoder == NOZIP?"nozip":"bzip2"));
return testfile;
}
static int
verifychunks(void)
{
int i;
int store = -1;
size_t chunksizes[MAXDIMS];
memset(chunksizes,0,sizeof(chunksizes));
CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<actualdims;i++) {
if(chunksizes[i] != chunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
}
return 1;
}
static int
create(XZIP encoder)
{
int i;
char* testfile = filenamefor(encoder);
/* Create a file with one big variable. */
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<actualdims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
return NC_NOERR;
}
static void
setvarfilter(XZIP encoder)
{
unsigned int level = BZIP2_LEVEL;
unsigned int id=0;
size_t nparams = 0;
if(encoder == BZIP2) {
CHECK(nc_def_var_filter(ncid,varid,BZIP2_ID,1,&level));
level = 0;
CHECK(nc_inq_var_filter(ncid,varid,&id,&nparams,&level));
if(id != BZIP2_ID || nparams != 1 || level != BZIP2_LEVEL)
printf("setvarfilter: def/inq mismatch\n");
}
}
static int
open(XZIP encoder)
{
char* testfile = filenamefor(encoder);
/* Open the file and check it. */
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
if(0 || encoder != NOZIP) {
/* Check the compression algorithm */
filterid = 0;
nparams = 0;
params = NULL;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
if(nparams > 0) {
params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
if(params == NULL)
return NC_ENOMEM;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
}
if(filterid != BZIP2_ID) {
printf("Bzip2 id mismatch: %d\n",filterid);
return NC_EFILTER;
}
if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) {
printf("Compression parameter mismatch\n");
return NC_EFILTER;
}
}
/* Verify chunking */
if(!verifychunks())
return 0;
fflush(stderr);
return 1;
}
static int
setchunking(void)
{
int i;
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunks));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (T)i;
} else {
while(odom_more()) {
int offset = odom_offset();
T expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
write(void)
{
int stat = NC_NOERR;
#ifdef VAR1
odom_reset();
while(odom_more()) {
size_t offset = odom_offset();
CHECK(NC_PUT_VAR1(ncid,varid,odom,&expected[offset]));
odom_next();
}
#else
stat = NC_PUT_VAR(ncid,varid,expected);
#endif
return stat;
}
static int
compare(void)
{
int errs = 0;
printf("data comparison: |array|=%d\n",actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
printf("mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
printf("mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
printf("no data errors\n");
return (errs == 0);
}
static void
showparameters(XZIP encoding)
{
int i;
printf("bzip2:");
for(i=0;i<nparams;i++) {
printf(" %u",params[i]);
}
printf("\n");
for(i=0;i<actualdims;i++)
printf("%s%d",(i==0?" chunks=":","),chunks[i]);
printf("\n");
}
static int
test_bzip2(void)
{
int ok = 1;
unsigned int param = BZIP2_LEVEL;
printf("\n*** Testing API: bzip2 compression.\n");
reset();
create(BZIP2);
setchunking();
setvarfilter(BZIP2); showparameters(BZIP2);
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
/* write array */
CHECK(write());
CHECK(nc_close(ncid));
printf("\n*** Testing API: bzip2 decompression.\n");
reset();
open(BZIP2);
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
#ifdef BASELINE
static int
test_nozip(void)
{
int ok = 1;
printf("\n*** Testing nozip compression.\n");
reset();
create(NOZIP);
setchunking();
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
/* write array */
CHECK(write());
CHECK(nc_close(ncid));
printf("\n*** Testing nozip decompression.\n");
reset();
open(NOZIP);
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
#endif
/**************************************************/
/* Utilities */
static void
reset()
{
memset(array,0,sizeof(T)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dims[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=actualdims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dims[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
offset += odom[i];
}
return offset;
}
static T
expectedvalue(void)
{
int i;
T offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
offset += odom[i];
}
return offset;
}
#if 0
#ifndef USEFLOAT
static size_t
getint(const char* arg)
{
char* p;
long l = strtol(arg,&p,10);
if(*p == '\0')
return (size_t)l;
fprintf(stderr,"expected integer: found %s\n", arg);
exit(1);
}
#else
static double
getdouble(const char* arg)
{
char* p;
double d = strtod(arg,&p);
if(*p == '\0')
return d;
fprintf(stderr,"expected double: found %s\n", arg);
exit(1);
}
#endif
#endif
static void
init(int argc, char** argv)
{
int i;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<MAXDIMS;i++) {
dims[i] = dimsize;
chunks[i] = (pattern[i] == 1 ? 1 : chunksize);
totalproduct *= dims[i];
if(i < actualdims) {
actualproduct *= dims[i];
chunkproduct *= chunks[i];
}
}
/* Allocate max size */
array = (T*)calloc(1,sizeof(T)*actualproduct);
expected = (T*)calloc(1,sizeof(T)*actualproduct);
}
/**************************************************/
int
main(int argc, char **argv)
{
init(argc,argv);
#ifdef BASELINE
if(!test_nozip()) ERRR;
#endif
if(!test_bzip2()) ERRR;
exit(nerrs > 0?1:0);
}

414
nc_test4/filter_test/test_misc.c Normal file
View File

@ -0,0 +1,414 @@
/*
Copyright 2008, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "netcdf.h"
#define TEST_ID 32768
#define MAXERRS 8
#define MAXPARAMS 32
#define NBASELINE 14
static const unsigned int baseline[NBASELINE] = {
1, /* 0 testcase # */
-17, /* 1 signed int*/
23, /* 2 unsigned int*/
-25, /* 3 signed int*/
27, /* 4 unsigned int*/
77, /* 5 signed int*/
93, /* 65 unsigned int*/
1145389056U, /* 7 float*/
697067329, 2723935171, /* 8-9 double*/
128, 16777216, /* 10-11 signed long long*/
4294967295, 4294967295, /* 12-13 unsigned long long*/
};
#define MAXDIMS 8
#define DEFAULTACTUALDIMS 4
#define DEFAULTDIMSIZE 4
#define DEFAULTCHUNKSIZE 4
static size_t dimsize = DEFAULTDIMSIZE;
static size_t chunksize = DEFAULTCHUNKSIZE;
static size_t actualdims = DEFAULTACTUALDIMS;
static size_t pattern[MAXDIMS];
static size_t totalproduct = 1; /* x-product over max dims */
static size_t actualproduct = 1; /* x-product over actualdims */
static size_t chunkproduct = 1; /* x-product over actual chunks */
static size_t dims[MAXDIMS];
static size_t chunks[MAXDIMS];
static int nerrs = 0;
static int ncid, varid;
static int dimids[MAXDIMS];
static size_t odom[MAXDIMS];
static float* array = NULL;
static float* expected = NULL;
static unsigned int filterid = 0;
static size_t nparams = 0;
static unsigned int params[MAXPARAMS];
/* Forward */
static int test_test1(void);
static void init(int argc, char** argv);
static void reset(void);
static void odom_reset(void);
static int odom_more(void);
static int odom_next(void);
static int odom_offset(void);
static float expectedvalue(void);
static void verifyparams(void);
#define ERRR do { \
fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
__FILE__, __LINE__); \
nerrs++;\
} while (0)
static int
check(int err,int line)
{
if(err != NC_NOERR) {
fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
}
return NC_NOERR;
}
static void
report(const char* msg, int lineno)
{
fprintf(stderr,"fail: line=%d %s\n",lineno,msg);
exit(1);
}
#define CHECK(x) check(x,__LINE__)
#define REPORT(x) report(x,__LINE__)
static char*
filenamefor(void)
{
return strdup("testmisc.nc");
}
static int
verifychunks(void)
{
int i;
int store = -1;
size_t chunksizes[MAXDIMS];
memset(chunksizes,0,sizeof(chunksizes));
CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
if(store != NC_CHUNKED) {
fprintf(stderr,"bad chunk store\n");
return 0;
}
for(i=0;i<actualdims;i++) {
if(chunksizes[i] != chunks[i]) {
fprintf(stderr,"bad chunk size: %d\n",i);
return 0;
}
}
return 1;
}
static int
create(void)
{
int i;
char* testfile = filenamefor();
/* Create a file with one big variable. */
CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid));
CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
for(i=0;i<actualdims;i++) {
char dimname[1024];
snprintf(dimname,sizeof(dimname),"dim%d",i);
CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
}
CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
return NC_NOERR;
}
static void
setvarfilter(void)
{
size_t i;
CHECK(nc_def_var_filter(ncid,varid,TEST_ID,NBASELINE,baseline));
verifyparams();
}
static void
verifyparams(void)
{
size_t i;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
if(filterid != TEST_ID) REPORT("id mismatch");
if(nparams != NBASELINE) REPORT("nparams mismatch");
for(i=0;i<nparams;i++) {
if(params[i] != baseline[i])
REPORT("param mismatch");
}
}
static int
open(void)
{
char* testfile = filenamefor();
unsigned int* params;
/* Open the file and check it. */
CHECK(nc_open(testfile, NC_NOWRITE, &ncid));
CHECK(nc_inq_varid(ncid, "var", &varid));
/* Check the compression algorithm */
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
if(nparams > 0) {
params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
if(params == NULL)
return NC_ENOMEM;
CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
}
if(filterid != TEST_ID) {
printf("open: test id mismatch: %d\n",filterid);
return NC_EFILTER;
}
if(nparams != NBASELINE) {
size_t i;
unsigned int inqparams[MAXPARAMS];
printf("nparams mismatch\n");
for(nerrs=0,i=0;i<nparams;i++) {
if(inqparams[i] != baseline[i]) {
printf("open: testparam mismatch: %d\n",i);
nerrs++;
}
}
}
if(nerrs > 0) return NC_EFILTER;
/* Verify chunking */
if(!verifychunks())
return 0;
fflush(stderr);
return 1;
}
static int
setchunking(void)
{
int i;
int store;
store = NC_CHUNKED;
CHECK(nc_def_var_chunking(ncid,varid,store,chunks));
if(!verifychunks())
return NC_EINVAL;
return NC_NOERR;
}
static void
fill(void)
{
odom_reset();
if(1) {
int i;
if(actualproduct <= 1) abort();
for(i=0;i<actualproduct;i++)
expected[i] = (float)i;
} else {
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
expected[offset] = expect;
odom_next();
}
}
}
static int
compare(void)
{
int errs = 0;
fprintf(stderr,"data comparison: |array|=%d\n",actualproduct);
if(1)
{
int i;
for(i=0;i<actualproduct;i++) {
if(expected[i] != array[i]) {
fprintf(stderr,"mismatch: array[%d]=%f expected[%d]=%f\n",
i,array[i],i,expected[i]);
errs++;
if(errs >= MAXERRS)
break;
}
}
} else
{
odom_reset();
while(odom_more()) {
int offset = odom_offset();
float expect = expectedvalue();
if(array[offset] != expect) {
fprintf(stderr,"mismatch: array[%d]=%f expected=%f\n",
offset,array[offset],expect);
errs++;
if(errs >= MAXERRS)
break;
}
odom_next();
}
}
if(errs == 0)
fprintf(stderr,"no data errors\n");
return (errs == 0);
}
static void
showparameters(void)
{
int i;
printf("test: nparams=%ld: params=",(unsigned long)nparams);
for(i=0;i<nparams;i++) {
printf(" %u",params[i]);
}
printf("\n");
for(i=0;i<actualdims;i++)
printf("%s%d",(i==0?" chunks=":","),chunks[i]);
printf("\n");
}
static int
test_test1(void)
{
int ok = 1;
reset();
printf("test1: compression.\n");
create();
setchunking();
setvarfilter();
showparameters();
CHECK(nc_enddef(ncid));
/* Fill in the array */
fill();
/* write array */
CHECK(nc_put_var(ncid,varid,expected));
CHECK(nc_close(ncid));
printf("test1: decompression.\n");
reset();
open();
CHECK(nc_get_var_float(ncid, varid, array));
ok = compare();
CHECK(nc_close(ncid));
return ok;
}
/**************************************************/
/* Utilities */
static void
reset()
{
memset(array,0,sizeof(float)*actualproduct);
}
static void
odom_reset(void)
{
memset(odom,0,sizeof(odom));
}
static int
odom_more(void)
{
return (odom[0] < dims[0]);
}
static int
odom_next(void)
{
int i; /* do not make unsigned */
for(i=actualdims-1;i>=0;i--) {
odom[i] += 1;
if(odom[i] < dims[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows*/
odom[i] = 0; /* reset this position*/
}
return 1;
}
static int
odom_offset(void)
{
int i;
int offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
offset += odom[i];
}
return offset;
}
static float
expectedvalue(void)
{
int i;
float offset = 0;
for(i=0;i<actualdims;i++) {
offset *= dims[i];
offset += odom[i];
}
return offset;
}
static void
init(int argc, char** argv)
{
int i;
/* Setup various variables */
totalproduct = 1;
actualproduct = 1;
chunkproduct = 1;
for(i=0;i<MAXDIMS;i++) {
dims[i] = dimsize;
chunks[i] = (pattern[i] == 1 ? 1 : chunksize);
totalproduct *= dims[i];
if(i < actualdims) {
actualproduct *= dims[i];
chunkproduct *= chunks[i];
}
}
/* Allocate max size */
array = (float*)calloc(1,sizeof(float)*actualproduct);
expected = (float*)calloc(1,sizeof(float)*actualproduct);
}
/**************************************************/
int
main(int argc, char **argv)
{
init(argc,argv);
if(!test_test1()) ERRR;
exit(nerrs > 0?1:0);
}

128
nc_test4/filter_test/tst_filter.sh Normal file
View File

@ -0,0 +1,128 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../../test_common.sh
set -e
export HDF5_PLUGIN_PATH=`pwd`
# Which test cases to exercise
API=1
NG=1
NCP=1
UNK=1
NGC=1
MISC=1
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/var:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| sed -e '/_IsNetcdf4/d' \
| cat > $2
}
# Function to extract _Filter attribute from a file
# These attributes might be platform dependent
getfilterattr() {
sed -e '/var:_Filter/p' -ed <$1 >$2
}
trimleft() {
sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2
}
if test "x$API" = x1 ; then
echo "*** Testing dynamic filters using API"
rm -f ./bzip2.nc ./bzip2.dump ./tmp
${execdir}/test_filter
$NCDUMP -s ./bzip2.nc > ./tmp
# Remove irrelevant -s output
sclean ./tmp ./bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
echo "*** Pass: API dynamic filter"
fi
if test "x$MISC" = x1 ; then
echo
echo "*** Testing dynamic filters parameter passing"
rm -f ./testmisc.nc tmp tmp2
${execdir}/test_misc
# Verify the parameters via ncdump
$NCDUMP -s ./testmisc.nc > ./tmp
# Extract the parameters
getfilterattr ./tmp ./tmp2
rm -f ./tmp
trimleft ./tmp2 ./tmp
rm -f ./tmp2
cat >./tmp2 <<EOF
var:_Filter = "32768,1,4294967279,23,4294967271,27,77,93,1145389056,697067329,2723935171,128,16777216,4294967295,4294967295" ;
EOF
diff -b -w ./tmp ./tmp2
echo "*** Pass: parameter passing"
fi
if test "x$NG" = x1 ; then
echo "*** Testing dynamic filters using ncgen"
rm -f ./bzip2.nc ./bzip2.dump ./tmp
$NCGEN -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
$NCDUMP -s ./bzip2.nc > ./tmp
# Remove irrelevant -s output
sclean ./tmp ./bzip2.dump
diff -b -w ${srcdir}/bzip2.cdl ./bzip2.dump
echo "*** Pass: ncgen dynamic filter"
fi
if test "x$NCP" = x1 ; then
echo "*** Testing dynamic filters using nccopy"
rm -f ./unfiltered.nc ./filtered.nc ./filtered.dump ./tmp
$NCGEN -4 -lb -o unfiltered.nc ${srcdir}/unfiltered.cdl
$NCCOPY -F "/g/var,307,9,4" unfiltered.nc filtered.nc
$NCDUMP -s ./filtered.nc > ./tmp
# Remove irrelevant -s output
sclean ./tmp ./filtered.dump
diff -b -w ${srcdir}/filtered.cdl ./filtered.dump
echo "*** Pass: nccopy dynamic filter"
fi
if test "x$UNK" = x1 ; then
echo "*** Testing access to filter info when filter dll is not available"
rm -f bzip2.nc ./tmp
# build bzip2.nc
$NCGEN -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl
for f in cygbzip2.dll libbzip2.so ; do
if test -f ${execdir}/$f ; then LIBNAME=${execdir}/$f; fi;
done
# dump and clean bzip2.nc header only when filter is avail
$NCDUMP -hs ./bzip2.nc > ./tmp
# Remove irrelevant -s output
sclean ./tmp bzip2.dump
# Now hide the filter code
mv ${LIBNAME} ${LIBNAME}.save
# dump and clean bzip2.nc header only when filter is not avail
rm -f ./tmp
$NCDUMP -hs ./bzip2.nc > ./tmp
# Remove irrelevant -s output
sclean ./tmp bzip2x.dump
# Restore the filter code
mv ${LIBNAME}.save ${LIBNAME}
diff -b -w ./bzip2.dump ./bzip2x.dump
echo "*** Pass: ncgen dynamic filter"
fi
if test "x$NGC" = x1 ; then
rm -f ./test_bzip2.c
echo "*** Testing dynamic filters using ncgen with -lc"
$NCGEN -lc -4 ${srcdir}/bzip2.cdl > test_bzip2.c
diff -b -w ${srcdir}/ref_bzip2.c ./test_bzip2.c
echo "*** Pass: ncgen dynamic filter"
fi
#cleanup
rm -f ./bzip*.nc ./unfiltered.nc ./filtered.nc ./tmp ./tmp2 *.dump bzip*hdr.*
rm -fr ./test_bzip2.c
rm -fr ./testmisc.nc
exit 0

87
nc_test4/filter_test/unfiltered.cdl Normal file
View File

@ -0,0 +1,87 @@
netcdf unfiltered {
dimensions:
dim0 = 4 ;
dim1 = 4 ;
dim2 = 4 ;
dim3 = 4 ;
// global attributes:
:_Format = "netCDF-4" ;
group: g {
variables:
float var(dim0, dim1, dim2, dim3) ;
var:_Storage = "chunked" ;
var:_ChunkSizes = 4, 4, 4, 4 ;
var:_Endianness = "little" ;
// group attributes:
data:
var =
0, 1, 2, 3,
4, 5, 6, 7,
8, 9, 10, 11,
12, 13, 14, 15,
16, 17, 18, 19,
20, 21, 22, 23,
24, 25, 26, 27,
28, 29, 30, 31,
32, 33, 34, 35,
36, 37, 38, 39,
40, 41, 42, 43,
44, 45, 46, 47,
48, 49, 50, 51,
52, 53, 54, 55,
56, 57, 58, 59,
60, 61, 62, 63,
64, 65, 66, 67,
68, 69, 70, 71,
72, 73, 74, 75,
76, 77, 78, 79,
80, 81, 82, 83,
84, 85, 86, 87,
88, 89, 90, 91,
92, 93, 94, 95,
96, 97, 98, 99,
100, 101, 102, 103,
104, 105, 106, 107,
108, 109, 110, 111,
112, 113, 114, 115,
116, 117, 118, 119,
120, 121, 122, 123,
124, 125, 126, 127,
128, 129, 130, 131,
132, 133, 134, 135,
136, 137, 138, 139,
140, 141, 142, 143,
144, 145, 146, 147,
148, 149, 150, 151,
152, 153, 154, 155,
156, 157, 158, 159,
160, 161, 162, 163,
164, 165, 166, 167,
168, 169, 170, 171,
172, 173, 174, 175,
176, 177, 178, 179,
180, 181, 182, 183,
184, 185, 186, 187,
188, 189, 190, 191,
192, 193, 194, 195,
196, 197, 198, 199,
200, 201, 202, 203,
204, 205, 206, 207,
208, 209, 210, 211,
212, 213, 214, 215,
216, 217, 218, 219,
220, 221, 222, 223,
224, 225, 226, 227,
228, 229, 230, 231,
232, 233, 234, 235,
236, 237, 238, 239,
240, 241, 242, 243,
244, 245, 246, 247,
248, 249, 250, 251,
252, 253, 254, 255 ;
} // group g
}

182
nc_test4/h5testszip.c Executable file
View File

@ -0,0 +1,182 @@
/*
* Example illustrates the use of SZIP compression in HDF5
*/
#include <stdio.h>
#include "hdf5.h"
#define NX 500
#define NY 600
#define CH_NX 100
#define CH_NY 25
static void initialize(void);
static int compare(void);
static float buf[NX][NY];
static float buf_r[NX][NY];
static const char* filename = "test.h5";
static int
writeszip()
{
hid_t file;
hid_t dataset32;
hid_t properties;
hid_t lcpl_id, dapl_id;
hid_t data_space;
hsize_t dims[2], chunk_size[2];
unsigned szip_options_mask;
unsigned szip_pixels_per_block;
/*
* Create a new file using read/write access, default file
* creation properties, and default file access properties.
*/
file = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
/* Describe the size of the array. */
dims[0] = NX;
dims[1] = NY;
data_space = H5Screate_simple (2, dims, NULL);
/*
* Set the dataset creation property list to specify that
* the raw data is to be partitioned into 100x100 element
* chunks and that each chunk is to be compressed.
*/
chunk_size[0] = CH_NX;
chunk_size[1] = CH_NY;
properties = H5Pcreate (H5P_DATASET_CREATE);
H5Pset_chunk (properties, 2, chunk_size);
/*
* Set parameters for SZIP compression; check the description of
* the H5Pset_szip function in the HDF5 Reference Manual for more
* information.
*/
szip_options_mask=H5_SZIP_NN_OPTION_MASK;
szip_pixels_per_block=32;
H5Pset_szip (properties, szip_options_mask, szip_pixels_per_block);
/*
* Create a new dataset within the file. The datatype
* and data space describe the data on disk, which may
* be different from the format used in the application's
* memory.
*/
lcpl_id = H5Pcreate (H5P_LINK_CREATE);
dapl_id = H5Pcreate (H5P_DATASET_ACCESS);
dataset32 = H5Dcreate (file, "datasetF32", H5T_NATIVE_FLOAT, data_space, lcpl_id, properties, dapl_id);
/*
* Write the array to the file. The datatype and dataspace
* describe the format of the data in the `buf' buffer.
* The raw data is translated to the format required on disk,
* as defined above. We use default raw data transfer properties.
*/
H5Dwrite (dataset32, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL,
H5P_DEFAULT, buf);
H5Sclose (data_space);
H5Pclose(lcpl_id);
H5Pclose(dapl_id);
H5Pclose (properties);
H5Dclose (dataset32);
H5Fclose (file);
return 1;
}
static int
readszip()
{
hid_t file;
hid_t dataset32;
hid_t properties;
int errcnt = 0;
file = H5Fopen (filename, H5F_ACC_RDONLY, H5P_DEFAULT);
properties = H5Pcreate(H5P_DATASET_ACCESS);
dataset32 = H5Dopen(file, "datasetF32", properties);
/*
* Read the array. This is similar to writing data,
* except the data flows in the opposite direction.
* Note: Decompression is automatic.
*/
H5Dread(dataset32, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_r);
errcnt = compare();
H5Pclose(properties);
H5Dclose (dataset32);
H5Fclose (file);
return (errcnt==0 ? 1 : 0);
}
static int
compare(void)
{
int i,j;
int errs = 0;
/* Do comparison */
for (i=0; i < NX; i++) {
for (j=0; j < NY; j++) {
if(buf[i][j] != buf_r[i][j]) {
errs++;
printf("mismatch: [%d][%d]: write = %f read=%f\n",
i,j,buf[i][j],buf_r[i][j]);
}
}
}
return errs;
}
static void
initialize(void)
{
int i, j;
/* Initialize data buffer with some bogus data. */
for(i=0; i < NX; i++) {
for(j=0; j < NY; j++) {
buf[i][j] = (float)(i + j);
}
}
}
int
main(int argc, char** argv)
{
int extfile = 0;
if(argc > 1) {
filename = argv[1];
extfile = 1;
}
initialize();
if(!extfile) {
if(!writeszip()) {
fprintf(stderr,"writeszip failed.\n");
goto fail;
}
}
if(!readszip()) {
fprintf(stderr,"openfile failed.\n");
goto fail;
}
fprintf(stderr,"***PASS\n");
return 0;
fail:
fprintf(stderr,"***FAIL\n");
return 1;
}

85
nc_test4/ref_szip.cdl Normal file
View File

@ -0,0 +1,85 @@
netcdf ref_szip {
dimensions:
phony_dim_0 = 40 ;
phony_dim_1 = 20 ;
variables:
int dset_szip(phony_dim_0, phony_dim_1) ;
data:
dset_szip =
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39,
40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59,
60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,
80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99,
100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 119,
120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134,
135, 136, 137, 138, 139,
140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154,
155, 156, 157, 158, 159,
160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174,
175, 176, 177, 178, 179,
180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194,
195, 196, 197, 198, 199,
200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214,
215, 216, 217, 218, 219,
220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234,
235, 236, 237, 238, 239,
240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254,
255, 256, 257, 258, 259,
260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274,
275, 276, 277, 278, 279,
280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294,
295, 296, 297, 298, 299,
300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314,
315, 316, 317, 318, 319,
320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334,
335, 336, 337, 338, 339,
340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354,
355, 356, 357, 358, 359,
360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374,
375, 376, 377, 378, 379,
380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394,
395, 396, 397, 398, 399,
400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414,
415, 416, 417, 418, 419,
420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 434,
435, 436, 437, 438, 439,
440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453, 454,
455, 456, 457, 458, 459,
460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474,
475, 476, 477, 478, 479,
480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 494,
495, 496, 497, 498, 499,
500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514,
515, 516, 517, 518, 519,
520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 530, 531, 532, 533, 534,
535, 536, 537, 538, 539,
540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554,
555, 556, 557, 558, 559,
560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 573, 574,
575, 576, 577, 578, 579,
580, 581, 582, 583, 584, 585, 586, 587, 588, 589, 590, 591, 592, 593, 594,
595, 596, 597, 598, 599,
600, 601, 602, 603, 604, 605, 606, 607, 608, 609, 610, 611, 612, 613, 614,
615, 616, 617, 618, 619,
620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634,
635, 636, 637, 638, 639,
640, 641, 642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654,
655, 656, 657, 658, 659,
660, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 672, 673, 674,
675, 676, 677, 678, 679,
680, 681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693, 694,
695, 696, 697, 698, 699,
700, 701, 702, 703, 704, 705, 706, 707, 708, 709, 710, 711, 712, 713, 714,
715, 716, 717, 718, 719,
720, 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, 731, 732, 733, 734,
735, 736, 737, 738, 739,
740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754,
755, 756, 757, 758, 759,
760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774,
775, 776, 777, 778, 779,
780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794,
795, 796, 797, 798, 799 ;
}

BIN
nc_test4/ref_szip.h5 Normal file
View File

Binary file not shown.

155
nc_test4/test_szip.c Normal file
View File

@ -0,0 +1,155 @@
/* This is part of the netCDF package.
Copyright 2005 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
*/
/*
* Example illustrates the use of SZIP compression in netCDF5
* Taken from HDF5 example.
*/
#include <config.h>
#include <stdlib.h>
#include <stdio.h>
#include "nc_tests.h"
#include "err_macros.h"
#include "netcdf.h"
#undef PLAIN
#define USECLOSE
/* Szip Constants. */
#define HDF5_FILTER_SZIP 4
#define H5_SZIP_MAX_PIXELS_PER_BLOCK 32
/* Option Mask Flags (Refere to HDF5 szip documentation) */
#define H5_SZIP_ALLOW_K13_OPTION_MASK 1 /*Allows k split = 13 compression mode. (Default)*/
#define H5_SZIP_CHIP_OPTION_MASK 2 /*Compresses exactly as in hardware*/
#define H5_SZIP_EC_OPTION_MASK 4 /*Selects entropy coding method. (Default)*/
#define H5_SZIP_NN_OPTION_MASK 32 /*Selects nearest neighbor coding method*/
#define NX 500
#define NY 600
#define CH_NX 100
#define CH_NY 25
static void initialize(void);
static int compare(void);
static float buf[NX][NY];
static float buf_r[NX][NY];
int
main(void)
{
int ncid, varid, dimids[2];
size_t dims[2], chunk_size[2];
unsigned int szip_params[2]; /* [0]=options_mask [1]=pixels_per_block */
int errcnt = 0;
/* Create a new file using read/write access. */
if(nc_create("testszip.nc", NC_CLOBBER|NC_NETCDF4, &ncid)) ERR;
/* Create dims */
dims[0] = NX;
dims[1] = NY;
if(nc_def_dim(ncid, "x", dims[0], &dimids[0])) ERR;
if(nc_def_dim(ncid, "y", dims[1], &dimids[1])) ERR;
/* Create a dimensioned variable */
if(nc_def_var(ncid, "datasetF32", NC_FLOAT, 2, dimids, &varid)) ERR;
/* no fill */
if(nc_def_var_fill(ncid, varid, 1, NULL)) ERR;
/* Define chunking for the variable:
* the raw data is to be partitioned into 100x100 element chunks.
*/
chunk_size[0] = CH_NX;
chunk_size[1] = CH_NY;
if(nc_def_var_chunking(ncid, varid, NC_CHUNKED, chunk_size)) ERR;
#ifndef PLAIN
/*
* Set parameters for SZIP compression; check the description of
* the H5Pset_szip function in the HDF5 Reference Manual for more
* information.
*/
szip_params[0] = H5_SZIP_NN_OPTION_MASK;
szip_params[1] = H5_SZIP_MAX_PIXELS_PER_BLOCK;
{ int stat = nc_def_var_filter(ncid, varid, HDF5_FILTER_SZIP, 2, szip_params);
if(stat) {
fprintf(stderr,"XXX: %d %s\b",stat,nc_strerror(stat));
ERR;
}
}
#endif
if(nc_enddef(ncid)) ERR;
initialize();
/* Write the array to the file */
if(nc_put_var_float(ncid, varid, &buf[0][0])) ERR;
#ifdef USECLOSE
/* Close and re-open the file */
if(nc_close(ncid)) ERR;
if(nc_open("testszip.nc", NC_NETCDF4, &ncid)) ERR;
if(nc_inq_varid(ncid, "datasetF32", &varid)) ERR;
#endif
/*
* Read the array. This is similar to writing data,
* except the data flows in the opposite direction.
* Note: Decompression should be automatic.
*/
memset(buf_r,0,sizeof(buf_r));
if(nc_get_var_float(ncid, varid, &buf_r[0][0])) ERR;
/* Do comparison */
errcnt = compare();
if(nc_close(ncid)) ERR;
if(errcnt) ERR;
SUMMARIZE_ERR;
FINAL_RESULTS;
return (errcnt==0?0:1);
}
static int
compare()
{
int i,j;
int errs = 0;
/* Do comparison */
for (i=0; i < NX; i++) {
for (j=0; j < NY; j++) {
if(buf[i][j] != buf_r[i][j]) {
errs++;
printf("mismatch: [%d][%d]: write = %f read=%f\n",
i,j,buf[i][j],buf_r[i][j]);
}
}
}
return errs;
}
static void
initialize(void)
{
int i, j;
/* Initialize data buffer with some bogus data. */
for(i=0; i < NX; i++) {
for(j=0; j < NY; j++) {
buf[i][j] = (float)(i + j);
}
}
}

303
nc_test4/tst_filterparser.c Normal file
View File

@ -0,0 +1,303 @@
/*
Copyright 2008, UCAR/Unidata
See COPYRIGHT file for copying and redistribution conditions.
*/
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "netcdf.h"
#include "ncfilter.h"
#undef USE_INTERNAL
#define PARAMS_ID 32768
static const unsigned int baseline[] = {
-17, /* 0 signed int*/
23, /* 1 unsigned int*/
-25, /* 2 signed int*/
27, /* 3 unsigned int*/
77, /* 4 signed int*/
93, /* 5 unsigned int*/
1145389056U, /* 6 float*/
3287505826, 1097305129, /* 7-8 double*/
1, 2147483648, /* 9-10 signed long long*/
4294967295, 4294967295, /* 11-12 unsigned long long*/
/* Edge cases */
2147483647, /* 13 max signed int*/
-2147483648, /* 14 min signed int*/
4294967295 /* 15 max unsigned int with no trailing U*/
};
static const char* spec =
"32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL, 2147483647, -2147483648, 4294967295";
/* Test support for the conversions */
/* Not sure if this kind of casting via union is legal C99 */
static union {
unsigned int ui;
float f;
} uf;
static union {
unsigned int ui[2];
double d;
} ud;
static union {
unsigned int ui[2];
unsigned long long ull;
long long ll;
} ul;
static int nerrs = 0;
#ifdef USE_INTERNAL
static int parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp);
#endif
#ifdef WORD_BIGENDIAN
static void byteswap8(unsigned char* mem);
#endif
static void
report(const char* which)
{
fprintf(stderr,"mismatch: %s\n",which);
fflush(stderr);
nerrs++;
}
static void
mismatch(size_t i, unsigned int baseline, unsigned int params)
{
fprintf(stderr,"mismatch: [%d] baseline=%ud spec=%ud\n",(int)i,baseline,params);
fflush(stderr);
nerrs++;
}
/**************************************************/
int
main(int argc, char **argv)
{
int stat = 0;
unsigned int id = 0;
size_t i,nparams = 0;
unsigned int* params = NULL;
printf("\nTesting filter parser.\n");
#ifdef USE_INTERNAL
stat = parsefilterspec(spec,&id,&nparams,&params);
#else
stat = NC_parsefilterspec(spec,&id,&nparams,&params);
#endif
if(!stat) {
report("NC_parsefilterspec failed");
exit(1);
}
for(i=0;i<nparams;i++) {
if(baseline[i] != params[i])
mismatch(i,baseline[i],params[i]);
}
/* Now some specialized tests */
uf.ui = params[6];
if(uf.f != (float)789.0)
report("uf.f");
ud.ui[0] = params[7];
ud.ui[1] = params[8];
#ifdef WORD_BIGENDIAN
byteswap8((unsigned char*)&ud.d);
#endif
if(ud.d != (double)12345678.12345678)
report("ud.d");
ul.ui[0] = params[9];
ul.ui[1] = params[10];
#ifdef WORD_BIGENDIAN
byteswap8((unsigned char*)&ul.ll);
#endif
if(ul.ll != -9223372036854775807LL)
report("ul.ll");
ul.ui[0] = params[11];
ul.ui[1] = params[12];
#ifdef WORD_BIGENDIAN
byteswap8((unsigned char*)&ul.ull);
#endif
if(ul.ull != 18446744073709551615ULL)
report("ul.ull");
if (params)
free(params);
if (!nerrs)
printf("SUCCESS!!\n");
return (nerrs > 0 ? 1 : 0);
}
#ifdef WORD_BIGENDIAN
/* Byte swap an 8-byte integer in place */
static void
byteswap8(unsigned char* mem)
{
unsigned char c;
c = mem[0];
mem[0] = mem[7];
mem[7] = c;
c = mem[1];
mem[1] = mem[6];
mem[6] = c;
c = mem[2];
mem[2] = mem[5];
mem[5] = c;
c = mem[3];
mem[3] = mem[4];
mem[4] = c;
}
#endif
#ifdef USE_INTERNAL
static int
parsefilterspec(const char* spec, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp)
{
char* p;
char* sdata = NULL;
int stat;
unsigned int uval, id;
size_t count; /* no. of comma delimited params */
size_t nparams; /* final no. of unsigned ints */
size_t i;
unsigned int* ulist = NULL;
unsigned char mem[8]; /* to convert to network byte order */
if(spec == NULL || strlen(spec) == 0) goto fail;
sdata = strdup(spec);
/* Count number of parameters + id and delimit */
p=sdata;
for(count=0;;count++) {
char* q = strchr(p,',');
if(q == NULL) break;
*q++ = '\0';
p = q;
}
count++; /* for final piece */
if(count == 0)
goto fail; /* no id and no parameters */
/* Extract the filter id */
p = sdata;
stat = sscanf(p,"%u",&id);
if(stat != 1) goto fail;
p = p + strlen(p) + 1; /* skip the filter id */
count--;
/* Allocate the max needed space; *2 in case the params are all doubles */
ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count)*2);
if(ulist == NULL) goto fail;
/* walk and convert */
nparams = 0;
for(i=0;i<count;i++) {
char* q;
unsigned long long val64u;
unsigned int val32u;
double vald;
float valf;
unsigned int *vector;
int isunsigned;
int isnegative;
int type;
/* Get trailing discrimination characters */
isunsigned = 0;
isnegative = 0;
type = 0;
if(strchr(p,'-') != NULL) isnegative = 1;
q = p+strlen(p)-2;
if(*q == 'U' || *q == 'u') isunsigned = 1;
q++;
switch (*q) {
case 'f': case 'F': type = 'f'; break; /* short */
case 'd': case 'D': type = 'd'; break; /* double */
case 'b': case 'B': type = 'b'; break; /* byte */
case 's': case 'S': type = 's'; break; /* short */
case 'l': case 'L': type = 'l'; break; /* long long */
case 'u': case 'U': type = 'i'; isunsigned = 1; break; /* integer */
case '.':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': type = 'i'; break;
default:
if(*q == '\0')
type = 'i';
else goto fail;
}
/* Now parse */
switch (type) {
case 'b':
case 's':
case 'i':
/* special case for a positive integer;for back compatibility.*/
if(!isnegative)
stat = sscanf(p,"%u",&val32u);
else
stat = sscanf(p,"%d",(int*)&val32u);
if(stat != 1) goto fail;
ulist[nparams++] = val32u;
break;
case 'f':
stat = sscanf(p,"%lf",&vald);
if(stat != 1) goto fail;
valf = (float)vald;
ulist[nparams++] = *(unsigned int*)&valf;
break;
case 'd':
stat = sscanf(p,"%lf",&vald);
if(stat != 1) goto fail;
/* convert to network byte order */
memcpy(mem,&vald,sizeof(mem));
#ifdef WORDS_BIGENDIAN
byteswap8(mem); /* convert little endian to big endian */
#endif
vector = (unsigned int*)mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
case 'l': /* long long */
if(isunsigned)
stat = sscanf(p,"%llu",&val64u);
else
stat = sscanf(p,"%lld",(long long*)&val64u);
if(stat != 1) goto fail;
/* convert to network byte order */
memcpy(mem,&val64u,sizeof(mem));
#ifdef WORDS_BIGENDIAN
byteswap8(mem); /* convert little endian to big endian */
#endif
vector = (unsigned int*)mem;
ulist[nparams++] = vector[0];
ulist[nparams++] = vector[1];
break;
default:
goto fail;
}
p = p + strlen(p) + 1; /* move to next param */
}
/* Now return results */
if(idp) *idp = id;
if(nparamsp) *nparamsp = nparams;
if(paramsp) *paramsp = ulist;
ulist = NULL; /* avoid duplicate free */
if(sdata) free(sdata);
if(ulist) free(ulist);
return 1;
fail:
if(sdata) free(sdata);
if(ulist) free(ulist);
return 0;
}
#endif /*USE_INTERNAL*/

29
nc_test4/tst_szip.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
set -e
rm -f testnc.h5 testszip.nc szip_dump.cdl
echo "*** Test read of known szip file"
${NCDUMP} ${srcdir}/ref_szip.h5 >szip_dump.cdl
diff -w ${srcdir}/ref_szip.cdl ./szip_dump.cdl
echo "*** Testing tst_szip "
${execdir}/test_szip
echo "***Passed"
echo "*** Testing h5testszip "
${execdir}/h5testszip
echo "***Passed"
echo "*** Testing h5testszip on testszip.nc"
${execdir}/h5testszip ./testszip.nc
echo "***Passed"
rm -f testnc.h5 testszip.nc
rm -f szip_dump.cdl
exit

1
ncdap_test/tst_ber.sh
View File

@ -1,6 +1,5 @@
#!/bin/sh
export SETX=1
#export NCPATHDEBUG=1
if test "x$srcdir" = x ; then srcdir=`pwd`; fi

1
ncdump/cdl.h
View File

@ -18,5 +18,6 @@
#define NC_ATT_STORAGE "_Storage"
#define NC_ATT_NOFILL "_NoFill"
#define NC_ATT_NETCDF4 "_NetCDF4"
#define NC_ATT_FILTER "_Filter"
#endif /*_CDL_H_ */

25
ncdump/nccopy.1
View File

@ -20,6 +20,7 @@ nccopy
\%[\-h \fI chunk_cache \fP]
\%[\-e \fI cache_elems \fP]
\%[\-r]
\%[\-F \fI filterspec \fP]
\%\fI infile \fP
\%\fI outfile \fP
.hy
@ -251,6 +252,30 @@ Read netCDF classic or 64-bit offset input file into a diskless netCDF
file in memory before copying. Requires that input file be small
enough to fit into memory. For \fBnccopy\fP, this doesn't seem to provide
any significant speedup, so may not be a useful option.
.IP "\fB \-F \fP \fIfilterspec\fP"
For netCDF-4 output, including netCDF-4 classic model, specify a filter
to apply to an specified variable in the output. As a rule, the filter
is a compression/decompression algorithm with a unique numeric identifier
assigned by the HDF Group (see https://support.hdfgroup.org/services/filters.html).
.IP
The \fIfilterspec\fP argument has this general form.
.RS
fqn,filterid,param1,param2...paramn
.RE
The fqn (fully qualified name) is the name
of a variable prefixed by its containing
groups with the group names separated by forward slash ('/').
An example might be \FI/g1/g2/var\fP. Alternatively,
just the variable name can be given if it is in the root group:
e.g. \FIvar\fP. Backslash escapes may be used as needed.
The filterid is an unsigned positive integer representing the id
assigned by the HDFgroup to the filter. Following the id is a sequence of
parameters defining the operation of the filter. Each parameter
is a 32-bit unsigned integer.
.IP
This parameter may be repeated multiple times with different
variable names.
.SH EXAMPLES
.LP
Make a copy of foo1.nc, a netCDF file of any type, to foo2.nc, a

232
ncdump/nccopy.c
View File

@ -8,6 +8,7 @@
#include "config.h" /* for USE_NETCDF4 macro */
#include <stdlib.h>
#include <stdio.h>
#ifdef HAVE_GETOPT_H
#include <getopt.h>
#endif
@ -15,12 +16,13 @@
#include <unistd.h>
#endif
#include <string.h>
#include <netcdf.h>
#include "netcdf.h"
#include "nciter.h"
#include "utils.h"
#include "chunkspec.h"
#include "dimmap.h"
#include "nccomps.h"
#include "ncfilter.h"
#ifdef _MSC_VER
#include "XGetopt.h"
@ -40,6 +42,33 @@ int optind;
#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model if netCDF-4 not available. */
#endif
/* Ascii characters requiring escaping as lead*/
#define ESCAPESD "0123456789"
#define ESCAPES " !\"#$%&'()*,:;<=>?[]\\^`{|}~"
#ifdef USE_NETCDF4
/* The unique id for a variable requires also the enclosing group id */
typedef struct VarID {
int grpid;
int varid;
} VarID;
#define MAX_FILTER_SPECS 64
#define MAX_FILTER_PARAMS 256
struct FilterSpec {
char* fqn;
unsigned int filterid;
size_t nparams;
unsigned int* params;
};
static int nfilterspecs = 0; /* Number of defined filter specs */
static struct FilterSpec filterspecs[MAX_FILTER_SPECS];
#endif
/* Global variables for command-line requests */
char *progname; /* for error messages */
static int option_kind = SAME_AS_INPUT;
@ -127,6 +156,122 @@ nc_inq_parid(int ncid, const char *fullname, int *locidp) {
return NC_NOERR;
}
/* Compute the fully qualified name of a (grpid,varid) pair; caller must free */
static int
computeFQN(VarID vid, char** fqnp)
{
int stat = NC_NOERR;
size_t len;
char* fqn = NULL;
char vname[NC_MAX_NAME+1];
char escname[(2*NC_MAX_NAME)+1];
int first;
char *p, *q;
if((stat = nc_inq_grpname_full(vid.grpid,&len,NULL))) goto done;
fqn = (char*)malloc(len+1+(2*NC_MAX_NAME)+1);
if(fqn == NULL) {stat = NC_ENOMEM; goto done;}
if((stat=nc_inq_grpname_full(vid.grpid,&len,fqn))) goto done;
fqn[len] = '\0'; /* guarantee */
if((stat=nc_inq_varname(vid.grpid,vid.varid,vname))) goto done;
vname[NC_MAX_NAME] = '\0';
if(strlen(fqn) > 1) strcat(fqn,"/");
p = vname;
q = escname;
for(first=1;*p;first=0) {
if((first && strchr(ESCAPESD,*p) != NULL)
|| strchr(ESCAPES,*p) != NULL) *q++ = '\\';
*q++ = *p++;
}
*q++ = '\0'; /* guarantee */
strcat(fqn,escname);
done:
if(stat == NC_NOERR && fqnp != NULL) *fqnp = fqn;
return stat;
}
#if 0
static int
parseFQN(int ncid, const char* fqn0, VarID* idp)
{
int stat = NC_NOERR;
char* fqn;
VarID vid;
char* p;
char* q;
char* segment;
vid.grpid = ncid;
if(fqn0 == NULL || fqn0[1] != '/')
{stat = NC_EBADNAME; goto done;}
fqn = strdup(fqn0+1); /* skip leading '/'*/
p = fqn;
for(;;) {
int newgrp;
segment = p;
q = p;
while(*p != '\0' && *p != '/') {
if(*p == '\\') p++;
*q++ = *p++;
}
if(*p == '\0') break;
*p++ = '\0';
if((stat=nc_inq_grp_ncid(vid.grpid,segment,&newgrp))) goto done;
vid.grpid = newgrp;
}
/* Segment should point to the varname */
if((stat=nc_inq_varid(vid.grpid,segment,&vid.varid))) goto done;
done:
if(fqn) free(fqn);
if(stat == NC_NOERR && idp != NULL) *idp = vid;
return stat;
}
#endif
static int
parsefilterspec(const char* optarg0, struct FilterSpec* spec)
{
char* optarg = NULL;
unsigned int* params = NULL;
size_t nparams;
unsigned int id;
char* p = NULL;
char* remainder = NULL;
if(optarg0 == NULL || strlen(optarg0) == 0 || spec == NULL) return 0;
optarg = strdup(optarg0);
/* Collect the fqn, taking escapes into account */
p = optarg;
remainder = NULL;
for(;*p;p++) {
if(*p == '\\') p++;
else if(*p == ',') {*p = '\0'; remainder = p+1; break;}
else if(*p == '\0') {remainder = p; break;}
/* else continue */
}
if(strlen(optarg) == 0) return 0; /* fqn does not exist */
/* Make sure leading '/' is in place */
if(optarg[0]=='/')
spec->fqn = strdup(optarg);
else {
spec->fqn = (char*)malloc(1+strlen(optarg)+1);
strcpy(spec->fqn,"/");
strcat(spec->fqn,optarg);
}
/* Collect the id+parameters */
if(!NC_parsefilterspec(remainder,&id,&nparams,&params))
return 0;
if(spec != NULL) {
spec->filterid = id;
spec->nparams = nparams;
spec->params = params;
}
return 1;
}
/* Return size of chunk in bytes for a variable varid in a group igrp, or 0 if
* layout is contiguous */
static int
@ -567,6 +712,69 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
return stat;
}
/* Copy netCDF-4 specific variable filter properties */
/* Watch out if input is netcdf-3 */
static int
copy_var_filter(int igrp, int varid, int ogrp, int o_varid)
{
int stat = NC_NOERR;
#ifdef USE_NETCDF4
VarID vid = {igrp,varid};
VarID ovid = {ogrp,o_varid};
/* handle filter parameters, copying from input, overriding with command-line options */
unsigned int filterid = 0;
size_t nparams = 0;
unsigned int* params = NULL;
struct FilterSpec spec;
int i, found;
char* ofqn = NULL;
int format, oformat;
/* Get file format of the input and output */
if((stat=nc_inq_format(vid.grpid,&format))) goto done;
if((stat=nc_inq_format(ovid.grpid,&oformat))) goto done;
if(oformat != NC_FORMAT_NETCDF4 && oformat != NC_FORMAT_NETCDF4_CLASSIC)
goto done; /* Can only use filter when output is netcdf4 */
/* Compute the output vid's FQN */
if((stat = computeFQN(ovid,&ofqn))) goto done;
/* See if any filter spec is defined for this output variable */
for(found=0,i=0;i<nfilterspecs;i++) {
if(strcmp(filterspecs[i].fqn,ofqn)==0) {spec = filterspecs[i]; found = 1; break;}
}
if(!found) {
spec.filterid = 0; /* marker to indicate not filter to apply */
if((oformat == NC_FORMAT_NETCDF4
|| oformat != NC_FORMAT_NETCDF4_CLASSIC)
&& (format == NC_FORMAT_NETCDF4
|| format != NC_FORMAT_NETCDF4_CLASSIC)
) {
/* Only bother to look if both input and output are netcdf-4 */
if((stat=nc_inq_var_filter(vid.grpid,vid.varid,&spec.filterid,&spec.nparams,NULL)))
goto done;
if(spec.filterid > 0) {/* input has a filter */
spec.params = (unsigned int*)malloc(sizeof(unsigned int)*spec.nparams);
if((stat=nc_inq_var_filter(vid.grpid,vid.varid,&spec.filterid,&spec.nparams,spec.params)))
goto done;
}
}
}
/* Apply filter spec if any */
if(spec.filterid > 0) {/* Apply filter */
#ifdef USE_NETCDF4
if((stat=nc_def_var_filter(ovid.grpid,ovid.varid,spec.filterid,spec.nparams,spec.params)))
goto done;
#endif
}
done:
/* Cleanup */
if(spec.filterid > 0 && spec.nparams > 0 && spec.params != NULL)
free(spec.params);
#endif /*USE_NETCDF4*/
return stat;
}
/* Set output variable o_varid (in group ogrp) to use chunking
* specified on command line, only called for classic format input and
* netCDF-4 format output, so no existing chunk lengths to override. */
@ -849,6 +1057,7 @@ copy_var(int igrp, int varid, int ogrp)
/* Set compression if specified in command line option */
NC_CHECK(set_var_compressed(ogrp, o_varid));
}
NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid));
}
}
#endif /* USE_NETCDF4 */
@ -1560,6 +1769,7 @@ usage(void)
[-h n] set size in bytes of chunk_cache for chunked variables\n\
[-e n] set number of elements that chunk_cache can hold\n\
[-r] read whole input file into diskless file on open (classic or 64-bit offset or cdf5 formats only)\n\
[-F filterspec] specify the compression algorithm to apply to an output variable.\n\
infile name of netCDF input file\n\
outfile name for netCDF output file\n"
@ -1576,6 +1786,9 @@ main(int argc, char**argv)
char* inputfile = NULL;
char* outputfile = NULL;
int c;
#ifdef USE_NETCDF4
struct FilterSpec filterspec;
#endif
/* table of formats for legal -k values */
struct Kvalues {
@ -1630,7 +1843,7 @@ main(int argc, char**argv)
usage();
}
while ((c = getopt(argc, argv, "k:3467d:sum:c:h:e:rwxg:G:v:V:")) != -1) {
while ((c = getopt(argc, argv, "k:3467d:sum:c:h:e:rwxg:G:v:V:F:")) != -1) {
switch(c) {
case 'k': /* for specifying variant of netCDF format to be generated
Format names:
@ -1745,7 +1958,20 @@ main(int argc, char**argv)
make_lvars (optarg, &option_nlvars, &option_lvars);
option_varstruct = false;
break;
default:
case 'F': /* optional filter spec for a specified variable */
#ifdef USE_NETCDF4
if(!parsefilterspec(optarg,&filterspec)) usage();
if(nfilterspecs >= (MAX_FILTER_SPECS-1))
error("too many -F filterspecs\n");
filterspecs[nfilterspecs] = filterspec;
nfilterspecs++;
// Force output to be netcdf-4
option_kind = NC_FORMAT_NETCDF4;
#else
error("-F requires netcdf-4");
#endif
break;
default:
usage();
}
}

23
ncdump/ncdump.c
View File

@ -34,8 +34,8 @@ int optind;
#include <locale.h>
#endif /* HAVE_LOCALE_H */
#include "netcdf_mem.h"
#include "netcdf.h"
#include "netcdf_mem.h"
#include "utils.h"
#include "nccomps.h"
#include "nctime0.h" /* new iso time and calendar stuff */
@ -1052,6 +1052,27 @@ pr_att_specials(
printf(" ;\n");
}
}
/* _Filter */
{
unsigned int id;
size_t nparams;
unsigned int* params = NULL;
if(nc_inq_var_filter(ncid, varid, &id, &nparams, NULL) == NC_NOERR
&& id > 0) {
if(nparams > 0) {
params = (unsigned int*)calloc(1,sizeof(unsigned int)*nparams);
NC_CHECK(nc_inq_var_filter(ncid, varid, &id, &nparams, params));
}
pr_att_name(ncid,varp->name,NC_ATT_FILTER);
printf(" = \"%u",id);
if(nparams > 0) {
int i;
for(i=0;i<nparams;i++)
printf(",%u",params[i]);
}
printf("\" ;\n");
}
}
{
int no_fill = 0;
/* Don't get the fill_value, it's set explicitly with

2
ncdump/tst_output.sh
View File

@ -2,7 +2,7 @@
set -x
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# This shell script tests the output from several previous tests.

2
ncgen/Makefile.am
View File

@ -52,6 +52,8 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc ref_camrun.c \
# autoconf will forcibly delete files of the name *.tab.*
# Note also that this should be built under linux or cygwin
# using bison version 3 or later.
# Note also that this code is in a shell script if you do
# not want to other to ./configure.
makeparser::
flex -Pncg -8 ncgen.l

15
ncgen/cvt.c
View File

@ -642,3 +642,18 @@ convertstringtochars(NCConstant* str)
}
return dl;
}
unsigned int
convertFilterID(const char* id)
{
unsigned int nid = 0;
int ok = 0;
struct FilterID* f;
/* for now, must be an integer */
ok = sscanf(id,"%u",&nid);
if(ok == 1)
return nid;
return 0; /* Not a recognizable id */
}

27
ncgen/genbin.c
View File

@ -241,6 +241,33 @@ genbin_definespecialattributes(Symbol* var)
NULL);
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _FILTER_FLAG) {
/* Special check for alternate way to specify _Deflate */
if(special->_FilterID == ZIP_ID) {
unsigned int level;
if(special->nparams == 0 || special->_FilterParams == NULL)
level = 9; /* default */
else
level = special->_FilterParams[0];
if(level < 0 || level > 9)
derror("Illegal deflate level");
else {
stat = nc_def_var_deflate(var->container->ncid,
var->ncid,
(special->_Shuffle == 1?1:0),
(level >= 0?1:0),
level);
}
} else {
stat = nc_def_var_filter(var->container->ncid,
var->ncid,
special->_FilterID,
special->nparams,
special->_FilterParams
);
}
check_err(stat,__LINE__,__FILE__);
}
}
#endif /*USE_NETCDF4*/

56
ncgen/genc.c
View File

@ -111,6 +111,21 @@ gen_ncc(const char *filename)
codedump(tmp);
codeline("} ;");
}
if(special->flags & _FILTER_FLAG) {
int i;
unsigned int* params = special->_FilterParams;
if(special->nparams == 0 || params == NULL) continue;
bbClear(tmp);
for(i=0;i<special->nparams;i++) {
bbprintf(tmp,"%s%luU",
(i == 0?"":", "),params[i]);
}
bbprintf0(stmt,"static unsigned int %s_filterparams[%d] = {",
cname(var),special->nparams);
codedump(stmt);
codedump(tmp);
codeline("} ;");
}
bbFree(tmp);
}
codeline("");
@ -494,6 +509,47 @@ genc_definespecialattributes(Symbol* vsym)
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
}
if(special->flags & _FILTER_FLAG) {
/* Special check for alternate way to specify _Deflate */
if(special->_FilterID == ZIP_ID) {
unsigned int level;
if(special->nparams == 0 || special->_FilterParams == NULL)
level = 9; /* default */
else
level = special->_FilterParams[0];
if(level < 0 || level > 9)
derror("Illegal deflate level");
else {
bbprintf0(stmt,
" stat = nc_def_var_deflate(%s, %s, %s, %d, %d);\n",
groupncid(vsym->container),
varncid(vsym),
(special->_Shuffle == 1?"NC_SHUFFLE":"NC_NOSHUFFLE"),
(level >= 0?1:0),
level);
codedump(stmt);
}
} else {
bbprintf0(stmt,
" stat = nc_def_var_filter(%s, %s, %u, %lu, ",
groupncid(vsym->container),
varncid(vsym),
special->_FilterID,
special->nparams
);
codedump(stmt);
if(special->nparams == 0 || special->_FilterParams == NULL)
codepartial("NULL");
else {
bbprintf0(stmt,"%s_filterparams",cname(vsym));
codedump(stmt);
}
codeline(");");
}
codelined(1,"check_err(stat,__LINE__,__FILE__);");
}
}
#endif /*USE_NETCDF4*/

20
ncgen/genlib.c
View File

@ -19,33 +19,33 @@ define_netcdf(void)
char filename[2048+1];
/* Rule for specifying the dataset name:
1. use explicit datasetname
2. use the datasetname from the .cdl file (see ncgen.l)
3. use -o name ?implemented?
4. use input cdl file name (with .cdl removed) ?implemented?
*/
/* Rule for specifying the output file name:
1. use -o name
2. use input cdl file name (with .cdl removed)
3. use the datasetname
2. use the datasetname from the .cdl file
3. use input cdl file name (with .cdl removed)
It would be better if there was some way
to specify the datasetname independently of the
file name, but oh well.
*/
if(netcdf_name) { /* -o flag name */
strncpy(filename,netcdf_name,2048);
} else { /* construct a usable output file name */
if (cdlname != NULL && strcmp(cdlname,"-") != 0) {/* cmd line name */
char* p;
strncpy(filename,cdlname,2048);
/* remove any suffix and prefix => create in cwd */
/* remove any suffix and prefix*/
p = strrchr(filename,'.');
if(p != NULL) {*p= '\0';}
p = strrchr(filename,'/');
if(p != NULL) {memmove(filename,(p+1),2048);}
} else {/* construct name from dataset name */
} else {/* construct name from dataset name */
strncpy(filename,datasetname,2048); /* Reserve space for extension, terminating '\0' */
}
/* Append the proper extension */
strncat(filename,binary_ext,2048-(strlen(filename) + strlen(binary_ext)));
}
/* Execute exactly one of these */
#ifdef ENABLE_C
if (l_flag == L_C) gen_ncc(filename); else /* create C code to create netcdf */

2
ncgen/genlib.h
View File

@ -88,7 +88,7 @@ extern Symbol* makearraytype(Symbol*, Dimset*);
extern void convert1(NCConstant*,NCConstant*); /* Convert an arbitrary value to another */
extern void setprimlength(NCConstant* prim, unsigned long len);
extern struct Datalist* convertstringtochars(NCConstant* str);
extern unsigned int convertFilterID(const char* id);
/* from: semantic.c */
extern void processsemantics(void);

1
ncgen/includes.h
View File

@ -48,7 +48,6 @@
#include "util.h"
#include "debug.h"
#include "nc.h"
#ifdef USE_NETCDF4
#include "nc4internal.h"
#endif

2
ncgen/main.c
View File

@ -67,7 +67,7 @@ void usage( void );
int main( int argc, char** argv );
/* Define tables vs modes for legal -k values*/
struct Kvalues legalkinds[NKVALUES] = {
struct Kvalues legalkinds[] = {
/* NetCDF-3 classic format (32-bit offsets) */
{"classic", NC_FORMAT_CLASSIC}, /* canonical format name */
{"nc3", NC_FORMAT_CLASSIC}, /* short format name */

8
ncgen/makeparser.sh Normal file
View File

@ -0,0 +1,8 @@
#!/bin/bash
flex -Pncg -8 ncgen.l
rm -f ncgenl.c
sed -e s/lex.ncg.c/ncgenl.c/g <lex.ncg.c >ncgenl.c
bison -pncg -t -d ncgen.y
rm -f ncgeny.c ncgeny.h
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.c >ncgeny.c
sed -e s/ncgen.tab.c/ncgeny.c/g -e s/ncgen.tab.h/ncgeny.h/g <ncgen.tab.h >ncgeny.h

19
ncgen/ncgen.h
View File

@ -93,7 +93,7 @@ various C global variables
#define _ISNETCDF4_FLAG 0x200
#define _SUPERBLOCK_FLAG 0x400
#define _FORMAT_FLAG 0x800
#define _FILTER_FLAG 0x1000
extern struct Specialtoken {
char* name;
@ -115,8 +115,18 @@ char* name;
int k_flag;
};
#define NKVALUES 100
extern struct Kvalues legalkinds[NKVALUES];
extern struct Kvalues legalkinds[];
struct FilterID {
char* name;
unsigned int id;
};
#define ZIP_ID 0xFFFFFFFF
#define SZIP_ID 0xFFFFFFFE
#define BZIP2_ID 307U
#define ZFP_ID 32013U
#define FPZIP_ID 32014U
/* Note: some non-var specials (i.e. _Format) are not included in this struct*/
typedef struct Specialdata {
@ -130,6 +140,9 @@ typedef struct Specialdata {
int _Shuffle; /* 0 => false, 1 => true*/
int _Endianness; /* 1 =>little, 2 => big*/
int _Fill ; /* 0 => false, 1 => true WATCHOUT: this is inverse of NOFILL*/
unsigned int _FilterID;
unsigned int* _FilterParams; /* NULL => defaults*/
size_t nparams; /* |_FilterParms| ; 0 => not specified*/
} Specialdata;
typedef struct GlobalSpecialdata {

5
ncgen/ncgen.l
View File

@ -132,6 +132,7 @@ struct Specialtoken specials[] = {
{"_NCProperties",_NCPROPS,_NCPROPS_FLAG},
{"_IsNetcdf4",_ISNETCDF4,_ISNETCDF4_FLAG},
{"_SuperblockVersion",_SUPERBLOCK,_SUPERBLOCK_FLAG},
{"_Filter",_FILTER,_FILTER_FLAG},
{NULL,0} /* null terminate */
};
@ -210,7 +211,7 @@ NUMBER [+-]?[0-9][0-9]*[Uu]?([BbSs]|[Ll]|[Ll][Ll])?
DBLNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[LlDd]?|[+-]?[0-9]*{exp}[LlDd]?
FLTNUMBER [+-]?[0-9]*\.[0-9]*{exp}?[Ff]|[+-]?[0-9]*{exp}[Ff]
SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"
SPECIAL "_FillValue"|"_Format"|"_Storage"|"_ChunkSizes"|"_Fletcher32"|"_DeflateLevel"|"_Shuffle"|"_Endianness"|"_NoFill"|"_NCProperties"|"_IsNetcdf4"|"_SuperblockVersion"|"_Filter"
USASCII [\x01-\x7F]
@ -274,6 +275,8 @@ ulong|uint|uinteger {return lexdebug(UINT_K);}
int64 {return lexdebug(INT64_K);}
uint64 {return lexdebug(UINT64_K);}
double {return lexdebug(DOUBLE_K);}
string {return lexdebug(STRING_K);}
unlimited|UNLIMITED {int32_val = -1;
return lexdebug(NC_UNLIMITED_K);}

87
ncgen/ncgen.y
View File

@ -73,7 +73,10 @@ char* primtypenames[PRIMNO] = {
"string"
};
static int GLOBAL_SPECIAL = _NCPROPS_FLAG | _ISNETCDF4_FLAG | _SUPERBLOCK_FLAG | _FORMAT_FLAG ;
static int GLOBAL_SPECIAL = _NCPROPS_FLAG
| _ISNETCDF4_FLAG
| _SUPERBLOCK_FLAG
| _FORMAT_FLAG ;
/*Defined in ncgen.l*/
extern int lineno; /* line number for error messages */
@ -119,6 +122,7 @@ static int containsfills(Datalist* list);
static void datalistextend(Datalist* dl, NCConstant* con);
static void vercheck(int ncid);
static long long extractint(NCConstant con);
static int parsefilterflag(const char* sdata0, Specialdata* special);
int yylex(void);
@ -141,7 +145,7 @@ unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
long mark; /* track indices into the sequence*/
int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant constant;
NCConstant constant;
}
%token <sym>
@ -152,11 +156,12 @@ NCConstant constant;
INT_K /* keyword for int datatype */
FLOAT_K /* keyword for float datatype */
DOUBLE_K /* keyword for double datatype */
UBYTE_K /* keyword for unsigned byte datatype */
USHORT_K /* keyword for unsigned short datatype */
UINT_K /* keyword for unsigned int datatype */
INT64_K /* keyword for long long datatype */
UINT64_K /* keyword for unsigned long long datatype */
UBYTE_K /* keyword for unsigned byte datatype */
USHORT_K /* keyword for unsigned short datatype */
UINT_K /* keyword for unsigned int datatype */
INT64_K /* keyword for long long datatype */
UINT64_K /* keyword for unsigned long long datatype */
STRING_K /* keyword for string datatype */
IDENT /* name for a dimension, variable, or attribute */
TERMSTRING /* terminal string */
CHAR_CONST /* char constant (not ever generated by ncgen.l) */
@ -195,6 +200,7 @@ NCConstant constant;
_NCPROPS
_ISNETCDF4
_SUPERBLOCK
_FILTER
DATASETID
%type <sym> ident typename primtype dimd varspec
@ -430,6 +436,7 @@ primtype: CHAR_K { $$ = primsymbols[NC_CHAR]; }
| UINT_K { vercheck(NC_UINT); $$ = primsymbols[NC_UINT]; }
| INT64_K { vercheck(NC_INT64); $$ = primsymbols[NC_INT64]; }
| UINT64_K { vercheck(NC_UINT64); $$ = primsymbols[NC_UINT64]; }
| STRING_K { vercheck(NC_STRING); $$ = primsymbols[NC_STRING]; }
;
dimsection: /* empty */
@ -742,6 +749,8 @@ attrdecl:
{$$ = makespecial(_SHUFFLE_FLAG,$1,NULL,(void*)&$5,1);}
| type_var_ref ':' _ENDIANNESS '=' conststring
{$$ = makespecial(_ENDIAN_FLAG,$1,NULL,(void*)&$5,1);}
| type_var_ref ':' _FILTER '=' conststring
{$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)&$5,1);}
| type_var_ref ':' _NOFILL '=' constbool
{$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)&$5,1);}
| ':' _FORMAT '=' conststring
@ -1151,6 +1160,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
int tf = 0;
char* sdata = NULL;
int idata = -1;
unsigned int udata = 0;
if((GLOBAL_SPECIAL & tag) != 0) {
if(vsym != NULL) {
@ -1189,6 +1199,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
case _STORAGE_FLAG:
case _NCPROPS_FLAG:
case _ENDIAN_FLAG:
case _FILTER_FLAG:
iconst.nctype = NC_STRING;
convert1(con,&iconst);
if(iconst.nctype == NC_STRING)
@ -1318,6 +1329,11 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
special->flags |= _STORAGE_FLAG;
special->_Storage = NC_CHUNKED;
} break;
case _FILTER_FLAG:
/* Parse the filter spec */
if(parsefilterflag(sdata,special))
special->flags |= _FILTER_FLAG;
break;
default: PANIC1("makespecial: illegal token: %d",tag);
}
}
@ -1427,6 +1443,63 @@ specialname(int tag)
return "<unknown>";
}
/*
Parse a filter spec string and store it in special
*/
static int
parsefilterflag(const char* sdata0, Specialdata* special)
{
char* p;
char* sdata = NULL;
int stat;
size_t count;
unsigned int* ulist = NULL;
if(sdata0 == NULL || strlen(sdata0) == 0) goto fail;
sdata = strdup(sdata0);
/* Count number of unsigned integers and delimit */
p=sdata;
for(count=0;;count++) {
char* q = strchr(p,',');
if(q == NULL) break;
*q++ = '\0'; /* delimit */
p = q;
}
count++; /* for final piece */
/* Start by collecting the filter id */
p = sdata;
stat = sscanf(p,"%u",&special->_FilterID);
if(stat != 1) goto fail;
count--; /* actual param count minus the id */
ulist = (unsigned int*)malloc(sizeof(unsigned int)*(count));
if(ulist == NULL) goto fail;
special->nparams = count;
for(count=0;count < special->nparams ;) {
unsigned int uval;
p = p + strlen(p) + 1; /* move to next param */
stat = sscanf(p,"%u",&uval);
if(stat != 1) goto fail;
ulist[count++] = uval;
}
special->_FilterParams = ulist;
ulist = NULL; /* avoid duplicate free */
if(sdata) free(sdata);
if(ulist) free(ulist);
return 1;
fail:
if(sdata) free(sdata);
if(ulist) free(ulist);
if(special) special->_FilterID = 0;
derror("Malformed filter spec: %s",sdata);
return 0;
}
/*
Since the arguments are all simple constants,
we can evaluate the function immediately

1205
ncgen/ncgenl.c
View File

File diff suppressed because it is too large Load Diff

3021
ncgen/ncgeny.c
View File

File diff suppressed because it is too large Load Diff

222
ncgen/ncgeny.h
View File

@ -1,13 +1,14 @@
/* A Bison parser, made by GNU Bison 3.0.4. */
/* A Bison parser, made by GNU Bison 2.3. */
/* Bison interface for Yacc-like parsers in C
/* Skeleton interface for Bison's Yacc-like parsers in C
Copyright (C) 1984, 1989-1990, 2000-2015 Free Software Foundation, Inc.
Copyright (C) 1984, 1989, 1990, 2000, 2001, 2002, 2003, 2004, 2005, 2006
Free Software Foundation, Inc.
This program is free software: you can redistribute it and/or modify
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
the Free Software Foundation; either version 2, or (at your option)
any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
@ -15,7 +16,9 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>. */
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor,
Boston, MA 02110-1301, USA. */
/* As a special exception, you may create a larger work that contains
part or all of the Bison parser skeleton and distribute that work
@ -30,100 +33,143 @@
This special exception was added by the Free Software Foundation in
version 2.2 of Bison. */
#ifndef YY_NCG_NCGEN_TAB_H_INCLUDED
# define YY_NCG_NCGEN_TAB_H_INCLUDED
/* Debug traces. */
#ifndef YYDEBUG
# define YYDEBUG 1
#endif
#if YYDEBUG
extern int ncgdebug;
#endif
/* Token type. */
/* Tokens. */
#ifndef YYTOKENTYPE
# define YYTOKENTYPE
enum yytokentype
{
NC_UNLIMITED_K = 258,
CHAR_K = 259,
BYTE_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
UBYTE_K = 265,
USHORT_K = 266,
UINT_K = 267,
INT64_K = 268,
UINT64_K = 269,
IDENT = 270,
TERMSTRING = 271,
CHAR_CONST = 272,
BYTE_CONST = 273,
SHORT_CONST = 274,
INT_CONST = 275,
INT64_CONST = 276,
UBYTE_CONST = 277,
USHORT_CONST = 278,
UINT_CONST = 279,
UINT64_CONST = 280,
FLOAT_CONST = 281,
DOUBLE_CONST = 282,
DIMENSIONS = 283,
VARIABLES = 284,
NETCDF = 285,
DATA = 286,
TYPES = 287,
COMPOUND = 288,
ENUM = 289,
OPAQUE_ = 290,
OPAQUESTRING = 291,
GROUP = 292,
PATH = 293,
FILLMARKER = 294,
NIL = 295,
_FILLVALUE = 296,
_FORMAT = 297,
_STORAGE = 298,
_CHUNKSIZES = 299,
_DEFLATELEVEL = 300,
_SHUFFLE = 301,
_ENDIANNESS = 302,
_NOFILL = 303,
_FLETCHER32 = 304,
_NCPROPS = 305,
_ISNETCDF4 = 306,
_SUPERBLOCK = 307,
DATASETID = 308
};
/* Put the tokens into the symbol table, so that GDB and other debuggers
know about them. */
enum yytokentype {
NC_UNLIMITED_K = 258,
CHAR_K = 259,
BYTE_K = 260,
SHORT_K = 261,
INT_K = 262,
FLOAT_K = 263,
DOUBLE_K = 264,
UBYTE_K = 265,
USHORT_K = 266,
UINT_K = 267,
INT64_K = 268,
UINT64_K = 269,
STRING_K = 270,
IDENT = 271,
TERMSTRING = 272,
CHAR_CONST = 273,
BYTE_CONST = 274,
SHORT_CONST = 275,
INT_CONST = 276,
INT64_CONST = 277,
UBYTE_CONST = 278,
USHORT_CONST = 279,
UINT_CONST = 280,
UINT64_CONST = 281,
FLOAT_CONST = 282,
DOUBLE_CONST = 283,
DIMENSIONS = 284,
VARIABLES = 285,
NETCDF = 286,
DATA = 287,
TYPES = 288,
COMPOUND = 289,
ENUM = 290,
OPAQUE_ = 291,
OPAQUESTRING = 292,
GROUP = 293,
PATH = 294,
FILLMARKER = 295,
NIL = 296,
_FILLVALUE = 297,
_FORMAT = 298,
_STORAGE = 299,
_CHUNKSIZES = 300,
_DEFLATELEVEL = 301,
_SHUFFLE = 302,
_ENDIANNESS = 303,
_NOFILL = 304,
_FLETCHER32 = 305,
_NCPROPS = 306,
_ISNETCDF4 = 307,
_SUPERBLOCK = 308,
_FILTER = 309,
DATASETID = 310
};
#endif
/* Tokens. */
#define NC_UNLIMITED_K 258
#define CHAR_K 259
#define BYTE_K 260
#define SHORT_K 261
#define INT_K 262
#define FLOAT_K 263
#define DOUBLE_K 264
#define UBYTE_K 265
#define USHORT_K 266
#define UINT_K 267
#define INT64_K 268
#define UINT64_K 269
#define STRING_K 270
#define IDENT 271
#define TERMSTRING 272
#define CHAR_CONST 273
#define BYTE_CONST 274
#define SHORT_CONST 275
#define INT_CONST 276
#define INT64_CONST 277
#define UBYTE_CONST 278
#define USHORT_CONST 279
#define UINT_CONST 280
#define UINT64_CONST 281
#define FLOAT_CONST 282
#define DOUBLE_CONST 283
#define DIMENSIONS 284
#define VARIABLES 285
#define NETCDF 286
#define DATA 287
#define TYPES 288
#define COMPOUND 289
#define ENUM 290
#define OPAQUE_ 291
#define OPAQUESTRING 292
#define GROUP 293
#define PATH 294
#define FILLMARKER 295
#define NIL 296
#define _FILLVALUE 297
#define _FORMAT 298
#define _STORAGE 299
#define _CHUNKSIZES 300
#define _DEFLATELEVEL 301
#define _SHUFFLE 302
#define _ENDIANNESS 303
#define _NOFILL 304
#define _FLETCHER32 305
#define _NCPROPS 306
#define _ISNETCDF4 307
#define _SUPERBLOCK 308
#define _FILTER 309
#define DATASETID 310
/* Value type. */
#if ! defined YYSTYPE && ! defined YYSTYPE_IS_DECLARED
union YYSTYPE
typedef union YYSTYPE
#line 142 "ncgen.y"
{
#line 138 "ncgen.y" /* yacc.c:1909 */
Symbol* sym;
unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
long mark; /* track indices into the sequence*/
int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant constant;
#line 117 "ncgeny.h" /* yacc.c:1909 */
};
typedef union YYSTYPE YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
NCConstant constant;
}
/* Line 1529 of yacc.c. */
#line 168 "ncgen.tab.h"
YYSTYPE;
# define yystype YYSTYPE /* obsolescent; will be withdrawn */
# define YYSTYPE_IS_DECLARED 1
# define YYSTYPE_IS_TRIVIAL 1
#endif
extern YYSTYPE ncglval;
int ncgparse (void);
#endif /* !YY_NCG_NCGEN_TAB_H_INCLUDED */