mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-27 07:30:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow double NaNs and Infinities in double to double conversions. Add tests for nccopy deflation, shuffling, and dimension-fixing options. Add man-page documentation for new nccopy topions, with examples. Run tst_nccopy3.sh test even for netCDF-4 builds.

Browse Source
This commit is contained in:
Russ Rew 2010-08-31 22:41:00 +00:00
parent 4ce85da310
commit 20b1e1615a
8 changed files with 193 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libsrc4/nc4hdf.c
View File

@ -3708,8 +3708,8 @@ nc4_convert_type(const void *src, void *dest,
case NC_DOUBLE:
for (dp = (double *)src, dp1 = dest; count < len; count++)
{
if (*dp > X_DOUBLE_MAX || *dp < X_DOUBLE_MIN)
(*range_error)++;
/* if (*dp > X_DOUBLE_MAX || *dp < X_DOUBLE_MIN) */
/* (*range_error)++; */
*dp1++ = *dp++;
}
break;

14
ncdump/Makefile.am
View File

@ -11,18 +11,19 @@ LDADD = -lm ${top_builddir}/liblib/libnetcdf.la @EXTERN_LDFLAGS@
# These tests are run for both netCDF-4 and non-netCDF-4 builds.
check_PROGRAMS = rewrite-scalar ctest ctest64 ncdump tst_utf8
TESTS = run_tests.sh tst_64bit.sh ctest ctest64 tst_output.sh \
tst_lengths.sh tst_calendars.sh tst_utf8 run_utf8_tests.sh
tst_lengths.sh tst_calendars.sh tst_utf8 run_utf8_tests.sh \
tst_nccopy3.sh
if USE_NETCDF4
# NetCDF-4 has some extra tests.
check_PROGRAMS += tst_create_files tst_h_rdc0 tst_group_data \
tst_enum_data tst_opaque_data tst_string_data tst_vlen_data tst_comp \
tst_comp2 tst_nans tst_special_atts tst_unicode tst_fillbug
tst_comp2 tst_nans tst_special_atts tst_unicode tst_fillbug tst_compress
TESTS += tst_create_files tst_group_data tst_enum_data tst_opaque_data \
tst_string_data tst_vlen_data tst_comp tst_comp2 tst_nans \
tst_special_atts tst_netcdf4.sh tst_h_rdc0 tst_unicode tst_fillbug \
tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh
tst_fillbug.sh tst_netcdf4_4.sh tst_compress tst_nccopy4.sh
if EXTRA_TESTS
TESTS += run_back_comp_tests.sh
@ -30,10 +31,6 @@ endif # EXTRA_TESTS
tst_h_rdc0_CPPFLAGS = -I${top_srcdir}/nc_test ${AM_CPPFLAGS}
else #!USE_NETCDF4
TESTS += tst_nccopy3.sh
endif #!USE_NETCDF4
CLEANFILES = test0.nc test1.cdl test1.nc test2.cdl ctest1.cdl ctest.c \
@ -49,7 +46,8 @@ tst_enum_data.cdl tst_small.cdl tst_times.cdl tst_solar_2.cdl \
tst_string_data.cdl tst_fillbug.cdl tst_opaque_data.cdl \
tst_compounds4.cdl tst_utf8.cdl tst_compounds3.cdl \
tst_special_atts.cdl tst_nans.cdl tst_format_att_64.cdl \
tst_vlen_data.cdl tst_solar_1.cdl tst_format_att.cdl
tst_vlen_data.cdl tst_solar_1.cdl tst_format_att.cdl tst_inflated.nc \
tst_deflated.nc
# This is the program we're building, and it's sources.

77
ncdump/nccopy.1
View File

@ -1,14 +1,18 @@
.\" $Id: nccopy.1 400 2010-08-27 21:02:52Z russ $
.TH NCCOPY 1 "$Date$" "Printed: \n(yr-\n(mo-\n(dy" "UNIDATA UTILITIES"
.SH NAME
nccopy \- Copy a netCDF file to specified variant of netCDF format
nccopy \- Copy a netCDF file to specified variant of netCDF format,
optionally compressing data in the output copy.
.SH SYNOPSIS
.ft B
.HP
nccopy
.nh
\%[-k n]
\%[-m n]
\%[-k kind]
\%[-d n]
\%[-s]
\%[-u]
\%[-m bufsize]
\%\fIinfile\fP
\%\fIoutfile\fP
.hy
@ -22,10 +26,8 @@ may be copied to a netCDF 64-bit offset file, permitting larger
variables.
If built with the netCDF-4 library, a netCDF classic file may be
copied to a netCDF-4 file or to a netCDF-4 classic
model file as well, permitting later efficient schema changes, larger
variable sizes, adding variables that use compressed or chunked
storage, and use of other netCDF-4 features in case the output uses
the enhanced netCDF model.
model file as well, permitting data compression, later efficient schema changes, larger variable sizes, and use of other netCDF-4
features in case the output uses the enhanced netCDF model.
.LP
\fBnccopy\fP also serves as an example of a generic netCDF-4 program,
with its ability to read any valid netCDF file and handle nested
@ -38,8 +40,9 @@ As of NetCDF version 4.1, and if DAP support was enabled when \fBnccopy\fP
was built, the file name may specify a DAP URL. This allows \fBnccopy\fP
to convert data on DAP servers to local netCDF files.
.SH OPTIONS
.IP "\fB-k \fRfile_format\fP"
Specifies the kind of file to be created and, by inference,
.IP "\fB-k \fIkind\fP"
Specifies the kind of file to be created that is, the format variant)
and, by inference,
the data model (i.e. netcdf-3 (classic) versus
netcdf-4 (enhanced)).
The possible arguments are as follows.
@ -52,12 +55,37 @@ The possible arguments are as follows.
.RE
.RE
If no value for -k is specified, then the output will use the same
format as the input. Note that attempting some kinds of format
format as the input (except if the input is classic or 64-bit offset
and a compression level is specified with
the -d option, in which case the outputt will be netCDF-4 classic
model).
Note that attempting some kinds of format
conversion will result in an error, if the conversion is not
possible. For example, an attempt to copy a netCDF-4 file that uses
features of the enhanced model to any of the other kinds of netCDF
formats that use the classic model will result an error.
.IP "\fB-m \fIcopy_buffer_size\fP"
.IP "\fB-d \fIn\fP"
Specify deflation level (level of compression) for variable data in
output. 0 corresponds to no compression and 9 to maximum compression,
with higher levels of compression requiring marginally more time to
compress or uncompress than lower levels. Compression achieved may
also depend on chunking parameters, which cannot currently be set to
anything except default chunking in the current nccopy
implementation. If this option is specified for a classic format or
64-bit offset format input file, it is not necessary to also specify
that the output should be netCDF-4 classic model, as that will
be the default. If this option is not specified and the input file
has compressed variables, the compression will still be preserved
in the output.
.IP "\fB-s\fP"
Specify shuffling of variable data bytes before compression or after
decompression. This option is ignored unless a non-zero deflation
level is specified. Turning shuffling on sometimes improves
compression.
.IP "\fB-u\fP"
Convert any unlimited size dimensions in the input to fixed size
dimensions in the output.
.IP "\fB-m \fIbufsize\fP"
Specifies the size, in bytes, of the copy buffer used to
to copy large variables, by copying them in smaller pieces, each no
larger than \fIcopy_bufer_size\fP. The default is 5,000,000 bytes,
@ -74,8 +102,9 @@ netCDF file of the same type:
nccopy foo1.nc foo2.nc
.RE
Note that the above copy will not be as fast as use of a
simple copy utility, because the file is copied structurally, using only the netCDF
API. This means, for example, that if the input file has extra bytes
simple copy utility, because the file is copied using
only the netCDF
API. If the input file has extra bytes
after the end of the
netCDF data, those will not be copied, because they are not accessible
through the netCDF interface. If the original file was generated in
@ -99,6 +128,28 @@ nccopy 'http://test.opendap.org/opendap/data/nc/sst.mnmean.nc.gz?time_bnds' tb.n
Note that URLs that name specific variables as command-line arguments
should generally be quoted, to avoid the shell interpreting special
characters such as `?'.
.LP
Compress all the variables in the input file foo.nc, a netCDF file of any
type, to the output file bar.nc:
.RS
.HP
nccopy -d1 foo.nc bar.nc
.RE
If foo.nc was a classic or 64-bit offset netCDF file, bar.nc will be a
netCDF-4 classic model netCDF file, because the classic and 64-bit
ofset format variants don't support compression. If foo.nc was a
netCDF-4 file with some variables compressed using various deflation
levels, the output will also be a netCDF-4 file of the samme type, but
all the variables, including any uncompressed variables in the input,
will now use deflation level 1.
.LP
Uncompress all the variables in the input file nc4.nc, a netCDF-4
classic model file, and copy to a classic netCDF file,
classic.nc:
.RS
.HP
nccopy -d0 -k1 nc4.nc classic.nc
.RE
.SH "SEE ALSO"
.LP
.BR ncdump(1), ncgen (1),

32
ncdump/nccopy.c
View File

@ -683,9 +683,10 @@ copy_var(int igrp, int varid, int ogrp)
* chunking, endianness, deflation, checksumming, fill, etc. */
stat = copy_var_specials(igrp, varid, ogrp, o_varid);
CHECK(stat, copy_var_specials);
} else { /* TODO: eliminate need for this branch and previous test */
stat = set_var_compressed(igrp, varid, ogrp, o_varid);
CHECK(stat, set_var_compressed);
}
else {
stat = set_var_compressed(igrp, varid, ogrp, o_varid);
CHECK(stat, set_var_compressed);
}
}
}
@ -925,7 +926,11 @@ copy_data(int igrp, int ogrp, size_t copybuf_size)
/* copy infile to outfile using netCDF API, kind specifies which
* netCDF format for output: -1 -> same as input, 1 -> classic, 2 ->
* 64-bit offset, 3 -> netCDF-4, 4 -> netCDF-4 classic model */
* 64-bit offset, 3 -> netCDF-4, 4 -> netCDF-4 classic model.
* However, if compression or shuffling was specified and kind was -1,
* kind is changed to format that supports compression for input of
* type 1 or 2.
*/
static int
copy(char* infile, char* outfile, int kind, size_t copybuf_size)
{
@ -939,8 +944,17 @@ copy(char* infile, char* outfile, int kind, size_t copybuf_size)
stat = nc_inq_format(igrp, &inkind);
CHECK(stat,nc_inq_format);
outkind = (kind == SAME_AS_INPUT) ? inkind : kind;
if (kind == SAME_AS_INPUT) { /* default, kind not specified */
outkind = inkind;
/* allow kind to be deduced in this case, instead of returning error */
if ((inkind == NC_FORMAT_CLASSIC || inkind == NC_FORMAT_64BIT) &&
(compress_level > 0 || shuffle_vars == NC_SHUFFLE) ) {
kind = NC_FORMAT_NETCDF4_CLASSIC;
outkind = kind;
}
} else {
outkind = kind;
}
switch(outkind) {
case NC_FORMAT_CLASSIC:
stat = nc_create(outfile,NC_CLOBBER,&ogrp);
@ -1097,8 +1111,12 @@ main(int argc, char**argv)
break;
case 'd': /* non-default compression level specified */
compress_level = strtol(optarg, NULL, 10);
if(compress_level < 0 || compress_level > 9) {
fprintf(stderr, "invalid deflation level: %d\n", compress_level);
return 1;
}
break;
case 's': /* add shuffling, may improve compression */
case 's': /* shuffling, may improve compression */
shuffle_vars = NC_SHUFFLE;
break;
case 'u': /* convert unlimited dimensions to fixed size */

15
ncdump/tst_brecs.cdl Normal file
View File

@ -0,0 +1,15 @@
netcdf tst_brecs {
dimensions:
time = UNLIMITED ; // (20 currently)
variables:
byte b1(time) ;
byte b2(time) ;
byte b3(time) ;
data:
b1 = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ;
b2 = 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ;
b3 = 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3 ;
}

42
ncdump/tst_compress.c Normal file
View File

@ -0,0 +1,42 @@
/* This is part of the netCDF package. Copyright 2010 University
Corporation for Atmospheric Research/Unidata. See COPYRIGHT file for
conditions of use. See www.unidata.ucar.edu for more info.
Create a compressible test file for nccopy to compress.
$Id$
*/
#include <nc_tests.h>
#include <stdlib.h>
#include <netcdf.h>
#define FILENAME "tst_inflated.nc"
#define DIM_NAME "dim1"
#define DIM1_LEN 10000
#define VAR1_RANK 1
#define VAR_NAME "var"
int
main(int argc, char **argv) { /* create a compressible file, for testing */
int ncid;
int dimid, varid;
int var1_dims[VAR1_RANK]; /* variable shapes */
int var1_data[DIM1_LEN]; /* data to write */
int i;
printf("\n*** Creating compressible test file %s...", FILENAME);
if (nc_create(FILENAME, NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, "dim1", DIM1_LEN, &dimid)) ERR;
var1_dims[0] = dimid;
if (nc_def_var(ncid, "var1", NC_INT, VAR1_RANK, var1_dims, &varid)) ERR;
if (nc_enddef (ncid)) ERR;
for(i=0; i < DIM1_LEN; i++) {
var1_data[i] = 3.0;
}
if (nc_put_var(ncid, varid, var1_data)) ERR;
if (nc_close(ncid)) ERR;
SUMMARIZE_ERR;
FINAL_RESULTS;
}

13
ncdump/tst_nccopy3.sh
View File

@ -7,7 +7,7 @@ echo ""
TESTFILES='c0 c0tmp ctest0 ctest0_64 small small2 test0 test1
tst_calendars tst_mslp tst_mslp_64 tst_ncml tst_small tst_utf8 utf8'
echo "*** Testing nccopy on ncdump/*.nc files"
echo "*** Testing netCDF-3 features of nccopy on ncdump/*.nc files"
for i in $TESTFILES ; do
echo "*** copy $i.nc to copy_of_$i.nc ..."
./nccopy $i.nc copy_of_$i.nc
@ -17,6 +17,17 @@ for i in $TESTFILES ; do
diff copy_of_$i.cdl tmp.cdl
rm copy_of_$i.nc copy_of_$i.cdl tmp.cdl
done
echo "*** Test nccopy -u"
echo "*** creating tst_brecs.nc from tst_brecs.cdl..."
../ncgen/ncgen -b tst_brecs.cdl
# convert record dimension to fixed-size dimension
./nccopy -u tst_brecs.nc copy_of_tst_brecs.nc
./ncdump -n copy_of_tst_brecs tst_brecs.nc | sed '/ = UNLIMITED ;/s/\(.*\) = UNLIMITED ; \/\/ (\(.*\) currently)/\1 = \2 ;/' > tmp.cdl
./ncdump copy_of_tst_brecs.nc > copy_of_tst_brecs.cdl
echo "*** compare result of nccopy -u with expected result ..."
diff copy_of_tst_brecs.cdl tmp.cdl
rm copy_of_tst_brecs.cdl tmp.cdl tst_brecs.nc copy_of_tst_brecs.nc
echo
echo "*** All nccopy tests passed!"
exit 0

34
ncdump/tst_nccopy4.sh
View File

@ -7,14 +7,12 @@ echo ""
# These files are actually in $srcdir in distcheck builds, so they
# need to be handled differently.
# ref_tst_compounds2 ref_tst_compounds3 ref_tst_compounds4
TESTFILES='c0 c0tmp ctest0 ctest0_64 small small2 test0 test1
tst_calendars tst_comp tst_comp2 tst_enum_data tst_fillbug
tst_group_data tst_mslp tst_mslp_64 tst_nans tst_ncml
tst_opaque_data tst_small tst_solar_1 tst_solar_2 tst_solar_cmp
tst_special_atts tst_string_data tst_unicode tst_utf8 tst_vlen_data
utf8'
TESTFILES='tst_comp tst_comp2 tst_enum_data tst_fillbug
tst_group_data tst_nans tst_opaque_data tst_solar_1 tst_solar_2
tst_solar_cmp tst_special_atts tst_string_data tst_unicode
tst_vlen_data'
echo "*** Testing nccopy on ncdump/*.nc files"
echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files"
for i in $TESTFILES ; do
echo "*** copy $i.nc to copy_of_$i.nc ..."
./nccopy $i.nc copy_of_$i.nc
@ -24,6 +22,28 @@ for i in $TESTFILES ; do
diff copy_of_$i.cdl tmp.cdl
rm copy_of_$i.nc copy_of_$i.cdl tmp.cdl
done
echo "*** Test nccopy -d1 can compress a file ..."
./nccopy -d1 tst_inflated.nc tst_deflated.nc
if test `wc -c < tst_deflated.nc` -ge `wc -c < tst_inflated.nc`; then
exit 1
fi
echo "*** Test nccopy -d1 -s can compress even more ..."
./nccopy -d1 tst_inflated.nc tmp.nc
if test `wc -c < tmp.nc` -ge `wc -c < tst_inflated.nc`; then
exit 1
fi
rm tst_deflated.nc tst_inflated.nc tmp.nc
echo "*** Testing nccopy -d1 -s on ncdump/*.nc files"
for i in $TESTFILES ; do
echo "*** nccopy -d1 -s $i.nc copy_of_$i.nc ..."
./nccopy -d1 -s $i.nc copy_of_$i.nc
./ncdump -n copy_of_$i $i.nc > tmp.cdl
./ncdump copy_of_$i.nc > copy_of_$i.cdl
echo "*** compare " with copy_of_$i.cdl
diff copy_of_$i.cdl tmp.cdl
rm copy_of_$i.nc copy_of_$i.cdl tmp.cdl
done
echo
echo "*** All nccopy tests passed!"
exit 0