From 1f39119f8f5c3f96fdab1f78416ad8627480c200 Mon Sep 17 00:00:00 2001 From: Russ Rew Date: Fri, 27 Aug 2010 21:02:52 +0000 Subject: [PATCH] Ignore attempt to compress scalar variables, instead of returning an error. Add compression and shuffling options, option to fix unlimited dimensions. --- libsrc4/nc4var.c | 4 +- man4/netcdf.texi | 10 ++--- ncdump/nccopy.1 | 5 ++- ncdump/nccopy.c | 97 ++++++++++++++++++++++++++++++++---------------- ncdump/ncdump.c | 2 +- ncdump/nciter.c | 2 +- ncdump/nciter.h | 2 +- 7 files changed, 77 insertions(+), 45 deletions(-) diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index e317abecb..07cf81963 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -695,9 +695,9 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, if (var->options_mask) return NC_EINVAL; - /* Must not be a scalar. */ + /* For scalars, just ignore attempt to deflate. */ if (!var->ndims) - return NC_EINVAL; + return NC_NOERR; /* Well, if we couldn't find any errors, I guess we have to take * the users settings. Darn! */ diff --git a/man4/netcdf.texi b/man4/netcdf.texi index 54c10dc10..8c0939afd 100644 --- a/man4/netcdf.texi +++ b/man4/netcdf.texi @@ -1564,7 +1564,7 @@ The extended CDL syntax for the enhanced data model supported by netCDF-4 allows optional type specifications, including user-defined types, for attributes of user-defined types. See ncdump output or the reference -documentation for ncgen4 for details of the extended CDL systax. +documentation for ncgen for details of the extended CDL systax. In the netCDF example (@pxref{Data Model}), units is an attribute for the variable lat that has a 13-character array value @@ -3929,9 +3929,9 @@ netCDF software distribution: reads a netCDF dataset and prints a textual representation of the information in the dataset -@item ncgen/ncgen4 +@item ncgen reads a textual representation of a netCDF dataset and generates -the corresponding binary netCDF file or a C or FORTRAN program to +the corresponding binary netCDF file or a program to create the netCDF dataset @item nccopy @@ -3947,8 +3947,8 @@ software that can access or manipulate netCDF data, see the NetCDF Software list, @uref{@value{netcdf-url}/software.html}. -This chapter describes the ncgen, ncgen4, and ncdump utilities. -These three tools +This chapter describes the ncgen and ncdump utilities. +These tools convert between binary netCDF datasets and a text representation of netCDF datasets. The output of ncdump and the input to ncgen is a text description of a netCDF dataset in a tiny language known as CDL diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 8e9704375..759d60e2d 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -1,5 +1,5 @@ -.\" $Header: /upc/share/CVS/netcdf-3/ncdump/nccopy.1,v 1.7 2009/12/09 13:49:31 russ Exp $ -.TH NCCOPY 1 "$Date: 2009/12/09 13:49:31 $" "Printed: \n(yr-\n(mo-\n(dy" "UNIDATA UTILITIES" +.\" $Id$ +.TH NCCOPY 1 "$Date$" "Printed: \n(yr-\n(mo-\n(dy" "UNIDATA UTILITIES" .SH NAME nccopy \- Copy a netCDF file to specified variant of netCDF format .SH SYNOPSIS @@ -8,6 +8,7 @@ nccopy \- Copy a netCDF file to specified variant of netCDF format nccopy .nh \%[-k n] +\%[-m n] \%\fIinfile\fP \%\fIoutfile\fP .hy diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index dddf7dcd1..14e6a4a75 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -1,7 +1,9 @@ /********************************************************************* - * Copyright 2008, University Corporation for Atmospheric Research + * Copyright 2010, University Corporation for Atmospheric Research * See netcdf/README file for copying and redistribution conditions. - * $ID$ + * Thanks to Philippe Poilbarbe and Antonio S. Cofiño for + * compression additions. + * $Id$ *********************************************************************/ #include "config.h" /* for USE_NETCDF4 macro */ @@ -21,7 +23,7 @@ #define CHECK(stat,f) if(stat != NC_NOERR) {check(stat,#f,__FILE__,__LINE__);} else {} #ifndef USE_NETCDF4 -#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model when used with NC_NETCDF4. */ +#define NC_CLASSIC_MODEL 0x0100 /* Enforce classic model if netCDF-4 not available. */ #endif /* These are in unistd.h; for use with getopt() */ @@ -29,9 +31,11 @@ extern int optind; extern int opterr; extern char *optarg; -static char *progname; /* for error messages */ -static int nofill_flag = 1; /* default is not to fill, because fill - * values will be copied anyway */ +/* Global variables for command-line requests */ +static char *progname; /* for error messages */ +static int compress_level = -1; /* default, compress output only if input compressed */ +static int shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */ +static int convert_ulim_dims = 0; /* default, preserve unlimited dimensions */ static void check(int err, const char* fcn, const char* file, const int line) @@ -51,17 +55,12 @@ emalloc (size_t size) p = (void *) malloc (size==0 ? 1 : size); /* don't malloc(0) */ if (p == 0) { - fprintf(stderr,"Out of memory!\n"); + fprintf(stderr,"%s: out of memory\n", progname); exit(1); } return p; } -/* Forward declaration, because copy_type, copy_vlen_type call each other */ -#ifdef USE_NETCDF4 -static int copy_type(int igrp, nc_type typeid, int ogrp); -#endif - /* get group id in output corresponding to group igrp in input, * given parent group id (or root group id) parid in output. */ static int @@ -135,6 +134,8 @@ inq_var_chunksize(int igrp, int varid, size_t* chunksizep) { return stat; } +/* Forward declaration, because copy_type, copy_vlen_type call each other */ +static int copy_type(int igrp, nc_type typeid, int ogrp); /* * copy a user-defined variable length type in the group igrp to the @@ -438,6 +439,10 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid) stat = nc_inq_var_deflate(igrp, varid, &shuffle, &deflate, &deflate_level); CHECK(stat, nc_inq_var_deflate); + if(compress_level >= 0) { /* change output compression, if requested */ + deflate = (compress_level <= 0 ? 0 : 1); + deflate_level = (compress_level <= 0 ? 0 : compress_level); + } if(deflate != 0 || shuffle != 0) { stat = nc_def_var_deflate(ogrp, o_varid, shuffle, deflate, deflate_level); @@ -465,6 +470,24 @@ copy_var_specials(int igrp, int varid, int ogrp, int o_varid) return stat; } +/* TODO: delete need for this, make copy_var_specials work for classic files */ +static int +set_var_compressed(int igrp, int varid, int ogrp, int o_varid) +{ + int stat = NC_NOERR; + if (compress_level > 0) + { /* handle compression parameters */ + int shuffle, deflate, deflate_level; + deflate = 1; + deflate_level = compress_level; + shuffle = shuffle_vars; + stat = nc_def_var_deflate(ogrp, o_varid, + shuffle, deflate, deflate_level); + CHECK(stat, nc_def_var_deflate); + } + return stat; +} + /* Release the variable chunk cache allocated for variable varid in * group igrp with it. This is not necessary, but will save some * memory if processing one variable at a time. */ @@ -559,7 +582,7 @@ copy_dims(int igrp, int ogrp) name); } CHECK(stat, nc_inq_dim); - if(is_unlim) { + if(is_unlim && !convert_ulim_dims) { stat = nc_def_dim(ogrp, name, NC_UNLIMITED, NULL); } else { stat = nc_def_dim(ogrp, name, length, NULL); @@ -654,12 +677,16 @@ copy_var(int igrp, int varid, int ogrp) CHECK(stat,nc_inq_format); stat = nc_inq_format(ogrp, &outkind); CHECK(stat,nc_inq_format); - if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC) && - (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC)) { - /* Copy all netCDF-4 specific variable properties such as - * chunking, endianness, deflation, checksumming, fill, etc. */ - stat = copy_var_specials(igrp, varid, ogrp, o_varid); - CHECK(stat, copy_var_specials); + if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) { + if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) { + /* Copy all netCDF-4 specific variable properties such as + * chunking, endianness, deflation, checksumming, fill, etc. */ + stat = copy_var_specials(igrp, varid, ogrp, o_varid); + CHECK(stat, copy_var_specials); + } else { /* TODO: eliminate need for this branch and previous test */ + stat = set_var_compressed(igrp, varid, ogrp, o_varid); + CHECK(stat, set_var_compressed); + } } } #endif /* USE_NETCDF4 */ @@ -725,9 +752,7 @@ copy_schema(int igrp, int ogrp) return stat; } -/* Return number of values for a variable varid in a group igrp, as - * well as array of dimension sizes, assumed to be preallocated to - * hold one value for each dimension of variable */ +/* Return number of values for a variable varid in a group igrp */ static int inq_nvals(int igrp, int varid, long long *nvalsp) { int stat = NC_NOERR; @@ -943,10 +968,8 @@ copy(char* infile, char* outfile, int kind, size_t copybuf_size) exit(1); } CHECK(stat,nc_create); - if(nofill_flag) { - stat = nc_set_fill(ogrp,NC_NOFILL,NULL); - CHECK(stat,nc_set_fill); - } + stat = nc_set_fill(ogrp, NC_NOFILL, NULL); /* will just copy fill values, if any */ + CHECK(stat,nc_set_fill); #ifdef USE_NETCDF4 /* Because types in one group may depend on types in a different @@ -979,12 +1002,15 @@ usage(void) #define USAGE "\ [-k n] kind of netCDF format for output file, default same as input\n\ 1 classic, 2 64-bit offset, 3 netCDF-4, 4 netCDF-4 classic model\n\ - [-m n] size in bytes of copy buffer\n\ + [-d n] deflation compression level, default same as input (0=none 9=max)\n\ + [-s] adds shuffle option to deflation compression\n\ + [-u] converts unlimited dimensions to fixed-size dimensions in output copy\n\ + [-m n] size in bytes of copy buffer, default is 5000000 bytes\n\ infile name of netCDF input file\n\ outfile name for netCDF output file\n" (void) fprintf(stderr, - "%s [-k n] [-m n] infile outfile\n%s", + "%s [-k n] [-d n] [-s] [-u] [-m n] infile outfile\n%s", progname, USAGE); } @@ -1038,7 +1064,7 @@ main(int argc, char**argv) return 1; } - while ((c = getopt(argc, argv, "k:m:")) != EOF) { + while ((c = getopt(argc, argv, "k:d:sum:")) != EOF) { switch(c) { case 'k': /* for specifying variant of netCDF format to be generated Possible values are: @@ -1069,6 +1095,15 @@ main(int argc, char**argv) } } break; + case 'd': /* non-default compression level specified */ + compress_level = strtol(optarg, NULL, 10); + break; + case 's': /* add shuffling, may improve compression */ + shuffle_vars = NC_SHUFFLE; + break; + case 'u': /* convert unlimited dimensions to fixed size */ + convert_ulim_dims = 1; + break; case 'm': /* non-default size of data copy buffer */ copybuf_size = strtoll(optarg, NULL, 10); break; @@ -1088,10 +1123,6 @@ main(int argc, char**argv) inputfile = argv[0]; outputfile = argv[1]; - /* set nofill mode to speed up creation of output file, because we - * will copy fill values from input anyway */ - nofill_flag = 1; - if(strcmp(inputfile, outputfile) == 0) { fprintf(stderr,"output would overwrite input\n"); exit(1); diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index 95bc18b6e..db6684870 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -1,7 +1,7 @@ /********************************************************************* * Copyright 2008, University Corporation for Atmospheric Research * See netcdf/README file for copying and redistribution conditions. - * $Header: /upc/share/CVS/netcdf-3/ncdump/ncdump.c,v 1.153 2010/05/17 21:36:51 russ Exp $ + * $Id$ *********************************************************************/ #include diff --git a/ncdump/nciter.c b/ncdump/nciter.c index 6534b335f..a39deb956 100644 --- a/ncdump/nciter.c +++ b/ncdump/nciter.c @@ -1,7 +1,7 @@ /********************************************************************* * Copyright 2009, University Corporation for Atmospheric Research * See netcdf/README file for copying and redistribution conditions. - * "$Header: /upc/share/CVS/netcdf-3/ncdump/nciter.c,v 1.8 2010/04/04 19:57:57 dmh Exp $" + * "$Id$" *********************************************************************/ #include "config.h" /* for USE_NETCDF4 macro */ diff --git a/ncdump/nciter.h b/ncdump/nciter.h index bf2ddf9b0..61bdafd9f 100644 --- a/ncdump/nciter.h +++ b/ncdump/nciter.h @@ -1,7 +1,7 @@ /********************************************************************* * Copyright 2009, University Corporation for Atmospheric Research * See netcdf/README file for copying and redistribution conditions. - * "$Id: nciter.h,v 1.4 2010/02/01 21:44:04 russ Exp $" + * "$Id$" *********************************************************************/ #ifndef _NCITER_