mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-27 07:30:33 +08:00
nccopy changes to support compression
This commit is contained in:
Russ Rew
parent
126a6fdd2d
commit
4ce85da310
@ -695,9 +695,9 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
|
||||
if (var->options_mask)
|
||||
return NC_EINVAL;
|
||||
|
||||
/* Must not be a scalar. */
|
||||
/* For scalars, just ignore attempt to deflate. */
|
||||
if (!var->ndims)
|
||||
return NC_EINVAL;
|
||||
return NC_NOERR;
|
||||
|
||||
/* Well, if we couldn't find any errors, I guess we have to take
|
||||
* the users settings. Darn! */
|
||||
|
||||
@ -1564,7 +1564,7 @@ The extended CDL syntax for the enhanced
|
||||
data model supported by netCDF-4 allows optional type specifications,
|
||||
including user-defined types, for
|
||||
attributes of user-defined types. See ncdump output or the reference
|
||||
documentation for ncgen4 for details of the extended CDL systax.
|
||||
documentation for ncgen for details of the extended CDL systax.
|
||||
|
||||
In the netCDF example (@pxref{Data Model}), units is an attribute for
|
||||
the variable lat that has a 13-character array value
|
||||
@ -3929,9 +3929,9 @@ netCDF software distribution:
|
||||
reads a netCDF dataset and prints a textual representation of
|
||||
the information in the dataset
|
||||
|
||||
@item ncgen/ncgen4
|
||||
@item ncgen
|
||||
reads a textual representation of a netCDF dataset and generates
|
||||
the corresponding binary netCDF file or a C or FORTRAN program to
|
||||
the corresponding binary netCDF file or a program to
|
||||
create the netCDF dataset
|
||||
|
||||
@item nccopy
|
||||
@ -3947,8 +3947,8 @@ software that can access or manipulate netCDF data, see the NetCDF
|
||||
Software list,
|
||||
@uref{@value{netcdf-url}/software.html}.
|
||||
|
||||
This chapter describes the ncgen, ncgen4, and ncdump utilities.
|
||||
These three tools
|
||||
This chapter describes the ncgen and ncdump utilities.
|
||||
These tools
|
||||
convert between binary netCDF datasets and a text representation of
|
||||
netCDF datasets. The output of ncdump and the input to ncgen is a text
|
||||
description of a netCDF dataset in a tiny language known as CDL
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
.\" $Header: /upc/share/CVS/netcdf-3/ncdump/nccopy.1,v 1.7 2009/12/09 13:49:31 russ Exp $
|
||||
.TH NCCOPY 1 "$Date: 2009/12/09 13:49:31 $" "Printed: \n(yr-\n(mo-\n(dy" "UNIDATA UTILITIES"
|
||||
.\" $Id: nccopy.1 400 2010-08-27 21:02:52Z russ $
|
||||
.TH NCCOPY 1 "$Date$" "Printed: \n(yr-\n(mo-\n(dy" "UNIDATA UTILITIES"
|
||||
.SH NAME
|
||||
nccopy \- Copy a netCDF file to specified variant of netCDF format
|
||||
.SH SYNOPSIS
|
||||
@ -8,6 +8,7 @@ nccopy \- Copy a netCDF file to specified variant of netCDF format
|
||||
nccopy
|
||||
.nh
|
||||
\%[-k n]
|
||||
\%[-m n]
|
||||
\%\fIinfile\fP
|
||||
\%\fIoutfile\fP
|
||||
.hy
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
/*********************************************************************
|
||||
* Copyright 2008, University Corporation for Atmospheric Research
|
||||
* Copyright 2010, University Corporation for Atmospheric Research
|
||||
* See netcdf/README file for copying and redistribution conditions.
|
||||
* $ID$
|
||||
* Thanks to Philippe Poilbarbe and Antonio S. Cofiño for
|
||||
* compression additions.
|
||||
* $Id: nccopy.c 400 2010-08-27 21:02:52Z russ $
|
||||
*********************************************************************/
|
||||
|
||||
#include "config.h" /* for USE_NETCDF4 macro */
|
||||
@ -21,7 +23,7 @@
|
||||
#define CHECK(stat,f) if(stat != NC_NOERR) {check(stat,#f,__FILE__,__LINE__);} else {}
|
||||
|
||||
#ifndef USE_NETCDF4
|
||||
#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model when used with NC_NETCDF4. */
|
||||
#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model if netCDF-4 not available. */
|
||||
#endif
|
||||
|
||||
/* These are in unistd.h; for use with getopt() */
|
||||
@ -29,9 +31,11 @@ extern int optind;
|
||||
extern int opterr;
|
||||
extern char *optarg;
|
||||
|
||||
static char *progname; /* for error messages */
|
||||
static int nofill_flag = 1; /* default is not to fill, because fill
|
||||
* values will be copied anyway */
|
||||
/* Global variables for command-line requests */
|
||||
static char *progname; /* for error messages */
|
||||
static int compress_level = -1; /* default, compress output only if input compressed */
|
||||
static int shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */
|
||||
static int convert_ulim_dims = 0; /* default, preserve unlimited dimensions */
|
||||
|
||||
static void
|
||||
check(int err, const char* fcn, const char* file, const int line)
|
||||
@ -51,17 +55,12 @@ emalloc (size_t size)
|
||||
|
||||
p = (void *) malloc (size==0 ? 1 : size); /* don't malloc(0) */
|
||||
if (p == 0) {
|
||||
fprintf(stderr,"Out of memory!\n");
|
||||
fprintf(stderr,"%s: out of memory\n", progname);
|
||||
exit(1);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/* Forward declaration, because copy_type, copy_vlen_type call each other */
|
||||
#ifdef USE_NETCDF4
|
||||
static int copy_type(int igrp, nc_type typeid, int ogrp);
|
||||
#endif
|
||||
|
||||
/* get group id in output corresponding to group igrp in input,
|
||||
* given parent group id (or root group id) parid in output. */
|
||||
static int
|
||||
@ -135,6 +134,8 @@ inq_var_chunksize(int igrp, int varid, size_t* chunksizep) {
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* Forward declaration, because copy_type, copy_vlen_type call each other */
|
||||
static int copy_type(int igrp, nc_type typeid, int ogrp);
|
||||
|
||||
/*
|
||||
* copy a user-defined variable length type in the group igrp to the
|
||||
@ -438,6 +439,10 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
|
||||
stat = nc_inq_var_deflate(igrp, varid,
|
||||
&shuffle, &deflate, &deflate_level);
|
||||
CHECK(stat, nc_inq_var_deflate);
|
||||
if(compress_level >= 0) { /* change output compression, if requested */
|
||||
deflate = (compress_level <= 0 ? 0 : 1);
|
||||
deflate_level = (compress_level <= 0 ? 0 : compress_level);
|
||||
}
|
||||
if(deflate != 0 || shuffle != 0) {
|
||||
stat = nc_def_var_deflate(ogrp, o_varid,
|
||||
shuffle, deflate, deflate_level);
|
||||
@ -465,6 +470,24 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* TODO: delete need for this, make copy_var_specials work for classic files */
|
||||
static int
|
||||
set_var_compressed(int igrp, int varid, int ogrp, int o_varid)
|
||||
{
|
||||
int stat = NC_NOERR;
|
||||
if (compress_level > 0)
|
||||
{ /* handle compression parameters */
|
||||
int shuffle, deflate, deflate_level;
|
||||
deflate = 1;
|
||||
deflate_level = compress_level;
|
||||
shuffle = shuffle_vars;
|
||||
stat = nc_def_var_deflate(ogrp, o_varid,
|
||||
shuffle, deflate, deflate_level);
|
||||
CHECK(stat, nc_def_var_deflate);
|
||||
}
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* Release the variable chunk cache allocated for variable varid in
|
||||
* group igrp with it. This is not necessary, but will save some
|
||||
* memory if processing one variable at a time. */
|
||||
@ -559,7 +582,7 @@ copy_dims(int igrp, int ogrp)
|
||||
name);
|
||||
}
|
||||
CHECK(stat, nc_inq_dim);
|
||||
if(is_unlim) {
|
||||
if(is_unlim && !convert_ulim_dims) {
|
||||
stat = nc_def_dim(ogrp, name, NC_UNLIMITED, NULL);
|
||||
} else {
|
||||
stat = nc_def_dim(ogrp, name, length, NULL);
|
||||
@ -654,12 +677,16 @@ copy_var(int igrp, int varid, int ogrp)
|
||||
CHECK(stat,nc_inq_format);
|
||||
stat = nc_inq_format(ogrp, &outkind);
|
||||
CHECK(stat,nc_inq_format);
|
||||
if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC) &&
|
||||
(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC)) {
|
||||
/* Copy all netCDF-4 specific variable properties such as
|
||||
* chunking, endianness, deflation, checksumming, fill, etc. */
|
||||
stat = copy_var_specials(igrp, varid, ogrp, o_varid);
|
||||
CHECK(stat, copy_var_specials);
|
||||
if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) {
|
||||
if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) {
|
||||
/* Copy all netCDF-4 specific variable properties such as
|
||||
* chunking, endianness, deflation, checksumming, fill, etc. */
|
||||
stat = copy_var_specials(igrp, varid, ogrp, o_varid);
|
||||
CHECK(stat, copy_var_specials);
|
||||
} else { /* TODO: eliminate need for this branch and previous test */
|
||||
stat = set_var_compressed(igrp, varid, ogrp, o_varid);
|
||||
CHECK(stat, set_var_compressed);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif /* USE_NETCDF4 */
|
||||
@ -725,9 +752,7 @@ copy_schema(int igrp, int ogrp)
|
||||
return stat;
|
||||
}
|
||||
|
||||
/* Return number of values for a variable varid in a group igrp, as
|
||||
* well as array of dimension sizes, assumed to be preallocated to
|
||||
* hold one value for each dimension of variable */
|
||||
/* Return number of values for a variable varid in a group igrp */
|
||||
static int
|
||||
inq_nvals(int igrp, int varid, long long *nvalsp) {
|
||||
int stat = NC_NOERR;
|
||||
@ -943,10 +968,8 @@ copy(char* infile, char* outfile, int kind, size_t copybuf_size)
|
||||
exit(1);
|
||||
}
|
||||
CHECK(stat,nc_create);
|
||||
if(nofill_flag) {
|
||||
stat = nc_set_fill(ogrp,NC_NOFILL,NULL);
|
||||
CHECK(stat,nc_set_fill);
|
||||
}
|
||||
stat = nc_set_fill(ogrp, NC_NOFILL, NULL); /* will just copy fill values, if any */
|
||||
CHECK(stat,nc_set_fill);
|
||||
|
||||
#ifdef USE_NETCDF4
|
||||
/* Because types in one group may depend on types in a different
|
||||
@ -979,12 +1002,15 @@ usage(void)
|
||||
#define USAGE "\
|
||||
[-k n] kind of netCDF format for output file, default same as input\n\
|
||||
1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic model\n\
|
||||
[-m n] size in bytes of copy buffer\n\
|
||||
[-d n] deflation compression level, default same as input (0=none 9=max)\n\
|
||||
[-s] adds shuffle option to deflation compression\n\
|
||||
[-u] converts unlimited dimensions to fixed-size dimensions in output copy\n\
|
||||
[-m n] size in bytes of copy buffer, default is 5000000 bytes\n\
|
||||
infile name of netCDF input file\n\
|
||||
outfile name for netCDF output file\n"
|
||||
|
||||
(void) fprintf(stderr,
|
||||
"%s [-k n] [-m n] infile outfile\n%s",
|
||||
"%s [-k n] [-d n] [-s] [-u] [-m n] infile outfile\n%s",
|
||||
progname,
|
||||
USAGE);
|
||||
}
|
||||
@ -1038,7 +1064,7 @@ main(int argc, char**argv)
|
||||
return 1;
|
||||
}
|
||||
|
||||
while ((c = getopt(argc, argv, "k:m:")) != EOF) {
|
||||
while ((c = getopt(argc, argv, "k:d:sum:")) != EOF) {
|
||||
switch(c) {
|
||||
case 'k': /* for specifying variant of netCDF format to be generated
|
||||
Possible values are:
|
||||
@ -1069,6 +1095,15 @@ main(int argc, char**argv)
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'd': /* non-default compression level specified */
|
||||
compress_level = strtol(optarg, NULL, 10);
|
||||
break;
|
||||
case 's': /* add shuffling, may improve compression */
|
||||
shuffle_vars = NC_SHUFFLE;
|
||||
break;
|
||||
case 'u': /* convert unlimited dimensions to fixed size */
|
||||
convert_ulim_dims = 1;
|
||||
break;
|
||||
case 'm': /* non-default size of data copy buffer */
|
||||
copybuf_size = strtoll(optarg, NULL, 10);
|
||||
break;
|
||||
@ -1088,10 +1123,6 @@ main(int argc, char**argv)
|
||||
inputfile = argv[0];
|
||||
outputfile = argv[1];
|
||||
|
||||
/* set nofill mode to speed up creation of output file, because we
|
||||
* will copy fill values from input anyway */
|
||||
nofill_flag = 1;
|
||||
|
||||
if(strcmp(inputfile, outputfile) == 0) {
|
||||
fprintf(stderr,"output would overwrite input\n");
|
||||
exit(1);
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/*********************************************************************
|
||||
* Copyright 2008, University Corporation for Atmospheric Research
|
||||
* See netcdf/README file for copying and redistribution conditions.
|
||||
* $Header: /upc/share/CVS/netcdf-3/ncdump/ncdump.c,v 1.153 2010/05/17 21:36:51 russ Exp $
|
||||
* $Id: ncdump.c 400 2010-08-27 21:02:52Z russ $
|
||||
*********************************************************************/
|
||||
|
||||
#include <config.h>
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/*********************************************************************
|
||||
* Copyright 2009, University Corporation for Atmospheric Research
|
||||
* See netcdf/README file for copying and redistribution conditions.
|
||||
* "$Header: /upc/share/CVS/netcdf-3/ncdump/nciter.c,v 1.8 2010/04/04 19:57:57 dmh Exp $"
|
||||
* "$Id: nciter.c 400 2010-08-27 21:02:52Z russ $"
|
||||
*********************************************************************/
|
||||
|
||||
#include "config.h" /* for USE_NETCDF4 macro */
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/*********************************************************************
|
||||
* Copyright 2009, University Corporation for Atmospheric Research
|
||||
* See netcdf/README file for copying and redistribution conditions.
|
||||
* "$Id: nciter.h,v 1.4 2010/02/01 21:44:04 russ Exp $"
|
||||
* "$Id: nciter.h 400 2010-08-27 21:02:52Z russ $"
|
||||
*********************************************************************/
|
||||
|
||||
#ifndef _NCITER_
|
||||
|
||||
Loading…
Reference in New Issue
Block a user