mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-27 07:30:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into ejh_global_att_read

Browse Source
This commit is contained in:
Ed Hartnett 2018-07-31 19:32:36 -06:00 committed by GitHub
commit 7ab6d7bdda
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 1456 additions and 399 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cf
View File

@ -1,6 +1,6 @@
#!/bin/bash
#NB=1
DB=1
#DB=1
#X=-x
#FAST=1
@ -118,7 +118,7 @@ FLAGS="$FLAGS --disable-diskless"
#FLAGS="$FLAGS --enable-jna"
#FLAGS="$FLAGS --disable-properties-attribute"
#FLAGS="$FLAGS --disable-silent-rules"
FLAGS="$FLAGS --disable-filter-testing"
#FLAGS="$FLAGS --disable-filter-testing"
#FLAGS="$FLAGS --enable-metadata-perf"
if test "x$TESTSERVERS" != x ; then

2
cf.cmake
View File

@ -60,7 +60,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true"
FLAGS="$FLAGS -DENABLE_LOGGING=true"
#FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true"
#FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true"
#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true"
# Disables
FLAGS="$FLAGS -DENABLE_EXAMPLES=false"

4
configure.ac
View File

@ -1406,9 +1406,10 @@ AC_SUBST(HAS_PARALLEL4,[$enable_parallel4])
AC_SUBST(HAS_DISKLESS,[yes])
AC_SUBST(HAS_MMAP,[$enable_mmap])
AC_SUBST(HAS_JNA,[$enable_jna])
AC_SUBST(RELAX_COORD_BOUND,[$enable_relax_coord_bound])
AC_SUBST(RELAX_COORD_BOUND,[$enable_zero_length_coord_bound])
AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill])
# Include some specifics for netcdf on windows.
#AH_VERBATIM([_WIN32_STRICMP],
AH_BOTTOM(
@ -1475,6 +1476,7 @@ AX_SET_META([NC_HAS_PARALLEL],[$enable_parallel],[yes])
AX_SET_META([NC_HAS_PARALLEL4],[$enable_parallel4],[yes])
AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes])
AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
AX_SET_META([NC_RELAX_COORD_BOUND], [$enable_zero_length_coord_bound],[yes])
# Automake says that this is always run in top_builddir
# and that srcdir is defined (== top_srcdir)

1
docs/all-error-codes.md
View File

@ -71,6 +71,7 @@ may occur.
#define NC_ENOTFOUND (-90) // No such file
#define NC_ECANTREMOVE (-91) // Cannot remove file
#define NC_EINTERNAL (-92) // NetCDF Library Internal Error
#define NC_EPNETCDF (-93) // Error at PnetCDF layer
~~~~
# NetCDF-4 Error Codes {#nc4-error-codes}

7
docs/tutorial.dox
View File

@ -815,8 +815,11 @@ independent (any processor may access the data without waiting for others). All
netCDF metadata writing operations are collective. That is, all creation of
groups, types, variables, dimensions, or attributes. Data reads and writes
(e.g. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent, the
default) or collective. To make writes to a variable collective, call
nc_var_par_access().
default) or collective. To change from collective to independent mode or vis
versa, call nc_var_par_access() with argument 'access' set to either
NC_INDEPENDENT or NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid'
is ignored, as PnetCDF does not support per-variable collective/independent
mode change.
\page tutorial_ncids Numbering of NetCDF IDs

4
examples/C/parallel_vara.c
View File

@ -150,7 +150,7 @@ int main(int argc, char** argv)
err = nc_enddef(ncid); ERR
/* set to use MPI/PnetCDF collective I/O */
err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR
err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR
/* now we are in data mode */
start[0] = 0;
@ -176,7 +176,7 @@ int main(int argc, char** argv)
err = nc_inq_varid(ncid, "var", &varid); ERR
/* set to use MPI/PnetCDF collective I/O */
err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR
err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR
/* each process reads its subarray from the file */
err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR

1
include/netcdf.h
View File

@ -426,6 +426,7 @@ by the desired type. */
#define NC_ENOTFOUND (-90) /**< No such file */
#define NC_ECANTREMOVE (-91) /**< Can't remove file */
#define NC_EINTERNAL (-92) /**< NetCDF Library Internal Error */
#define NC_EPNETCDF (-93) /**< Error at PnetCDF layer */
/* The following was added in support of netcdf-4. Make all netcdf-4
error codes < -100 so that errors can be added to netcdf-3 if

5
include/netcdf_meta.h.in
View File

@ -48,10 +48,11 @@
#define NC_HAS_MMAP @NC_HAS_MMAP@ /*!< mmap support. */
#define NC_HAS_JNA @NC_HAS_JNA@ /*!< jna support. */
#define NC_HAS_PNETCDF @NC_HAS_PNETCDF@ /*!< pnetcdf support. */
#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and
/or pnetcdf. */
#define NC_HAS_PARALLEL4 @NC_HAS_PARALLEL4@ /*!< parallel IO support via hdf5 */
#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and/or pnetcdf. */
#define NC_HAS_CDF5 @NC_HAS_CDF5@ /*!< CDF5 support. */
#define NC_HAS_ERANGE_FILL @NC_HAS_ERANGE_FILL@ /*!< ERANGE_FILL Support */
#define NC_RELAX_COORD_BOUND @NC_RELAX_COORD_BOUND@ /*!< RELAX_COORD_BOUND */
#endif

3
libdap4/d4data.c
View File

@ -99,8 +99,7 @@ NCD4_processdata(NCD4meta* meta)
for(i=0;i<nclistlength(toplevel);i++) {
NCD4node* var = (NCD4node*)nclistget(toplevel,i);
if(var->data.localchecksum != var->data.remotechecksum) {
fprintf(stderr,"Checksum mismatch: %s\n",var->name);
fflush(stderr);
nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name);
ret = NC_EDAP;
goto done;
}

2
libdispatch/derror.c
View File

@ -196,6 +196,8 @@ const char *nc_strerror(int ncerr1)
return "NetCDF: cannot delete file";
case NC_EINTERNAL:
return "NetCDF: internal library error; Please contact Unidata support";
case NC_EPNETCDF:
return "NetCDF: PnetCDF error";
case NC_EHDFERR:
return "NetCDF: HDF error";
case NC_ECANTREAD:

4
libhdf5/nc4mem.c
View File

@ -75,9 +75,9 @@ NC4_create_image_file(NC_FILE_INFO_T* h5, size_t initialsz)
hdfid = NC4_image_init(h5);
if(hdfid < 0)
{stat = NC_EHDFERR; goto done;}
/* Remember HDF5 file identifier. */
((NC_HDF5_FILE_INFO_T *)h5->format_file_info)->hdfid = hdfid;
done:
return stat;
}
@ -90,7 +90,7 @@ NC4_extract_file_image(NC_FILE_INFO_T* h5)
herr_t herr;
NC_memio mem;
assert(h5 && h5->format_file_info && !h5->no_write);
assert(h5 && h5->format_file_info && !h5->no_write);
/* Get the file access property list */
fapl = h5->mem.fapl;

167
libsrcp/ncpdispatch.c
View File

@ -9,20 +9,19 @@
#include <mpi.h>
#include "nc.h"
#include "ncdispatch.h"
#include "fbits.h"
/* Must follow netcdf.h */
#include <pnetcdf.h>
#define NCP_MODE_DATA 0x0001
#define NCP_MODE_INDEP 0x0002
typedef struct NCP_INFO
{
/* pnetcdf_file will be true if the file is created/opened with the
* parallel-netcdf library. pnetcdf_access_mode keeps track of
* whether independpent or collective mode is
* desired. pnetcdf_ndims keeps track of how many dims each var
* has, which I need to know to convert start, count, and stride
* arrays from size_t to MPI_Offset. (I can't use an inq function
* to find out the number of dims, because these are collective in
* pnetcdf.) */
/* pnetcdf_access_mode keeps track of whether independpent or collective
* mode is set currently.
*/
int pnetcdf_access_mode;
} NCP_INFO;
@ -46,8 +45,8 @@ NCP_create(const char *path, int cmode,
{
int res, default_format;
NCP_INFO* nc5;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
MPI_Comm comm;
MPI_Info info;
/* Check the cmode for only valid flags*/
if(cmode & ~LEGAL_CREATE_FLAGS)
@ -88,23 +87,14 @@ NCP_create(const char *path, int cmode,
/* Link nc5 and nc */
NCP_DATA_SET(nc,nc5);
/* Fix up the cmode by keeping only essential flags;
these are the flags that are the same in netcf.h and pnetcdf.h
*/
/* It turns out that pnetcdf.h defines a flag called
NC_64BIT_DATA (not to be confused with NC_64BIT_OFFSET).
This flag is essential to getting ncmpi_create to create
a proper pnetcdf format file.
We have set the value of NC_64BIT_DATA to be the same as in pnetcdf.h
(as of pnetcdf version 1.6.0) to avoid conflicts.
In any case, this flag must be set.
*/
/* PnetCDF recognizes the flags below for create and ignores NC_LOCK and NC_SHARE */
cmode &= (NC_WRITE | NC_NOCLOBBER | NC_SHARE | NC_64BIT_OFFSET | NC_64BIT_DATA);
res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid));
/* Default to independent access, like netCDF-4/HDF5 files. */
fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */
done:
return res;
}
@ -117,50 +107,36 @@ NCP_open(const char *path, int cmode,
{
int res;
NCP_INFO* nc5;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
MPI_Comm comm;
MPI_Info info;
/* Check the cmode for only valid flags*/
if(cmode & ~LEGAL_OPEN_FLAGS)
{res = NC_EINVAL; goto done;}
/* Cannot have both MPIO flags */
if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
{res = NC_EINVAL; goto done;}
/* No MPI environment initialized */
if (mpidata == NULL)
{res = NC_ENOPAR; goto done;}
/* Appears that this comment is wrong; allow 64 bit offset*/
/* Cannot have 64 bit offset flag */
/* if(cmode & (NC_64BIT_OFFSET)) {res = NC_EINVAL; goto done;} */
if(mpidata != NULL) {
comm = ((NC_MPI_INFO *)mpidata)->comm;
info = ((NC_MPI_INFO *)mpidata)->info;
} else {
comm = MPI_COMM_WORLD;
info = MPI_INFO_NULL;
}
/* PnetCDF recognizes the flags NC_WRITE and NC_NOCLOBBER for file open
* and ignores NC_LOCK, NC_SHARE, NC_64BIT_OFFSET, and NC_64BIT_DATA.
* Ignoring the NC_64BIT_OFFSET and NC_64BIT_DATA flags is because the
* file is already in one of the CDF-formats, and setting these 2 flags
* will not change the format of that file.
*/
cmode &= (NC_WRITE | NC_NOCLOBBER);
comm = ((NC_MPI_INFO *)mpidata)->comm;
info = ((NC_MPI_INFO *)mpidata)->info;
/* Create our specific NCP_INFO instance */
nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO));
if(nc5 == NULL) {res = NC_ENOMEM; goto done;}
/* file open automatically enters data mode */
fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
/* Link nc5 and nc */
NCP_DATA_SET(nc,nc5);
res = ncmpi_open(comm, path, cmode, info, &(nc->int_ncid));
/* Default to independent access, like netCDF-4/HDF5 files. */
if(!res) {
if (res == NC_NOERR) {
res = ncmpi_begin_indep_data(nc->int_ncid);
nc5->pnetcdf_access_mode = NC_INDEPENDENT;
fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
}
done:
return res;
@ -170,8 +146,14 @@ static int
NCP_redef(int ncid)
{
NC* nc;
NCP_INFO* nc5;
int status = NC_check_id(ncid, &nc);
if(status != NC_NOERR) return status;
nc5 = NCP_DATA(nc);
fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
return ncmpi_redef(nc->int_ncid);
}
@ -193,9 +175,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t
nc5 = NCP_DATA(nc);
assert(nc5);
/* causes implicitly defined warning; may be because of old installed pnetcdf? */
#if 1
/* In PnetCDF ncmpi__enddef() is only implemented in v1.5.0 and later */
#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10500)
/* ncmpi__enddef() was first implemented in PnetCDF v1.5.0 */
status = ncmpi__enddef(nc->int_ncid, mpi_h_minfree, mpi_v_align,
mpi_v_minfree, mpi_r_align);
#else
@ -203,7 +184,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t
#endif
if(!status) {
if (nc5->pnetcdf_access_mode == NC_INDEPENDENT)
fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA);
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP))
status = ncmpi_begin_indep_data(nc->int_ncid);
}
return status;
@ -257,7 +239,12 @@ NCP_set_fill(int ncid, int fillmode, int *old_mode_ptr)
NC* nc;
int status = NC_check_id(ncid, &nc);
if(status != NC_NOERR) return status;
#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601)
/* ncmpi_set_fill was first implemented in PnetCDF 1.6.1 */
return ncmpi_set_fill(nc->int_ncid,fillmode,old_mode_ptr);
#else
return NC_EPNETCDF;
#endif
}
static int
@ -314,7 +301,7 @@ NCP_inq_type(int ncid, nc_type typeid, char* name, size_t* size)
{
/* Assert mode & NC_FORMAT_CDF5 */
if (typeid < NC_BYTE || typeid >= NC_STRING)
return NC_EBADTYPE;
return NC_EBADTYPE;
if(name)
strcpy(name, NC_atomictypename(typeid));
if(size)
@ -440,15 +427,14 @@ NCP_get_att(
{
NC* nc;
int status;
nc_type xtype;
status = NC_check_id(ncid, &nc);
if(status != NC_NOERR) return status;
status = NCP_inq_att(ncid,varid,name,&xtype,NULL);
if(status != NC_NOERR) return status;
if(memtype == NC_NAT) memtype = xtype;
if (memtype == NC_NAT) {
status = NCP_inq_att(ncid,varid,name,&memtype,NULL);
if (status != NC_NOERR) return status;
}
switch (memtype) {
case NC_CHAR:
@ -611,7 +597,7 @@ NCP_get_vara(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break;
@ -705,7 +691,7 @@ NCP_put_vara(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break;
@ -801,7 +787,7 @@ NCP_get_vars(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break;
@ -897,7 +883,7 @@ NCP_put_vars(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break;
@ -995,7 +981,7 @@ NCP_get_varm(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break;
@ -1093,7 +1079,7 @@ NCP_put_varm(int ncid,
if (status) return status;
}
if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
switch(memtype) {
case NC_BYTE:
status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break;
@ -1171,7 +1157,13 @@ NCP_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep,
if(deflatep) *deflatep = 0;
if(fletcher32p) *fletcher32p = 0;
if(contiguousp) *contiguousp = NC_CONTIGUOUS;
#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601)
/* ncmpi_inq_var_fill was first implemented in PnetCDF 1.6.1 */
if(no_fill) ncmpi_inq_var_fill(nc->int_ncid, varid, no_fill, fill_valuep);
#else
/* PnetCDF 1.6.0 and priors support NC_NOFILL only */
if(no_fill) *no_fill = 1;
#endif
if(endiannessp) return NC_ENOTNC4;
if(idp) return NC_ENOTNC4;
if(nparamsp) return NC_ENOTNC4;
@ -1185,7 +1177,12 @@ NCP_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
NC* nc;
int status = NC_check_id(ncid, &nc);
if(status != NC_NOERR) return status;
#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601)
/* ncmpi_def_var_fill was first implemented in PnetCDF 1.6.1 */
return ncmpi_def_var_fill(nc->int_ncid, varid, no_fill, fill_value);
#else
return NC_EPNETCDF;
#endif
}
static int
@ -1198,19 +1195,43 @@ NCP_var_par_access(int ncid, int varid, int par_access)
if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE)
return NC_EINVAL;
#ifdef _DO_NOT_IGNORE_VARID_
if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */
return NC_EINVAL;
#endif
status = NC_check_id(ncid, &nc);
if(status != NC_NOERR) return status;
nc5 = NCP_DATA(nc);
assert(nc5);
if(par_access == nc5->pnetcdf_access_mode)
return NC_NOERR;
nc5->pnetcdf_access_mode = par_access;
if (par_access == NC_INDEPENDENT)
return ncmpi_begin_indep_data(nc->int_ncid);
else
return ncmpi_end_indep_data(nc->int_ncid);
/* if currently in data mode */
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) {
if (par_access == NC_INDEPENDENT) {
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP))
return NC_NOERR;
else { /* currently in collective data mode */
fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
return ncmpi_begin_indep_data(nc->int_ncid);
}
}
else { /* want to enter collective data mode */
if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) {
fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
return ncmpi_end_indep_data(nc->int_ncid);
}
else
return NC_NOERR;
}
}
else { /* currently in define mode */
if (par_access == NC_INDEPENDENT)
fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
else
fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP);
}
return NC_NOERR;
}
#ifdef USE_NETCDF4

5
nc_test/tst_parallel2.c
View File

@ -135,8 +135,13 @@ main(int argc, char **argv)
sleep(mpi_rank);
#endif /* USE_MPE */
#ifdef USE_PNETCDF
/* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/
if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR;
#else
/* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/
if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR;
#endif
if (!mpi_rank)
start_time = MPI_Wtime();

6
nc_test/tst_pnetcdf.c
View File

@ -68,10 +68,8 @@ int main(int argc, char* argv[])
}
if (nc_enddef(ncid)) ERR;
for (i=0; i<NVARS; i++) {
/* Note NC_INDEPENDENT is the default */
if (nc_var_par_access(ncid, varid[i], NC_INDEPENDENT)) ERR;
}
/* Note NC_INDEPENDENT is the default */
if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR;
/* write all variables */
buf = (int*) malloc(NX * sizeof(int));

6
nc_test/tst_small.c
View File

@ -34,7 +34,7 @@
int format; \
nc_inq_format_extended(ncid,&format,NULL); \
if (format == NC_FORMATX_PNETCDF) { \
if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;\
if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;\
}\
}
#else
@ -425,9 +425,7 @@ test_two_growing_with_att(const char *testfile)
{int format;
nc_inq_format_extended(ncid,&format,NULL);
if (format == NC_FORMATX_PNETCDF) {
for (v = 0; v < NUM_VARS; v++) {
if (nc_var_par_access(ncid, varid[v], NC_COLLECTIVE)) ERR;
}
if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;
}}
#endif
count[0] = 1;

17
nc_test/util.c
View File

@ -927,14 +927,8 @@ write_file(char *filename)
error("nc_enddef: %s", nc_strerror(err));
#ifdef USE_PNETCDF
{ int i,format;
nc_inq_format_extended(ncid, &format, NULL);
if (format == NC_FORMATX_PNETCDF) {
for (i = 0; i < numVars; i++) {
err = nc_var_par_access(ncid, i, NC_COLLECTIVE);
IF (err) error("nc_var_par_access: %s", nc_strerror(err));
}
}}
err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE);
IF (err) error("nc_var_par_access: %s", nc_strerror(err));
#endif
put_vars(ncid);
@ -1284,6 +1278,8 @@ char* nc_err_code_name(int err)
case (NC_EAUTH): return "NC_EAUTH";
case (NC_ENOTFOUND): return "NC_ENOTFOUND";
case (NC_ECANTREMOVE): return "NC_ECANTREMOVE";
case (NC_EINTERNAL): return "NC_EINTERNAL";
case (NC_EPNETCDF): return "NC_EPNETCDF";
case (NC_EHDFERR): return "NC_EHDFERR";
case (NC_ECANTREAD): return "NC_ECANTREAD";
case (NC_ECANTWRITE): return "NC_ECANTWRITE";
@ -1315,8 +1311,9 @@ char* nc_err_code_name(int err)
case (NC_EDISKLESS): return "NC_EDISKLESS";
case (NC_ECANTEXTEND): return "NC_ECANTEXTEND";
case (NC_EMPI): return "NC_EMPI";
case (NC_ENULLPAD): return "NC_NULLPAD";
// case (NC_EURL): return "NC_EURL";
case (NC_ENULLPAD): return "NC_NULLPAD";
case (NC_EINMEMORY): return "NC_EINMEMORY";
// case (NC_EURL): return "NC_EURL";
// case (NC_ECONSTRAINT): return "NC_ECONSTRAINT";
#ifdef USE_PNETCDF
case (NC_ESMALL): return "NC_ESMALL";

8
nc_test4/Make0
View File

@ -1,5 +1,5 @@
# Test c output
T=tst_varsperf
T=bigmeta
#SRC=hdf5plugins/H5Zmisc.c
@ -12,7 +12,8 @@ T=tst_varsperf
#PAR=1
#SZIP=1
CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include
CFLAGS = -Wall -g -O0 -I.. -I../include
LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf
@ -31,7 +32,8 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
all:: cmp
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CMD} ./t ${ARGS}
# ${CMD} ./t ${ARGS}
cmp::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \

5
nc_test4/bigmeta.c
View File

@ -134,7 +134,6 @@ static void
buildatts(int grpid, int varid)
{
char name[NC_MAX_NAME+1];
int attid;
int i, count;
count = (varid == NC_GLOBAL? ngroupattrs : nvarattrs);
@ -149,7 +148,7 @@ static void
buildgroup(int parent, int grpindex, int depth)
{
char name[NC_MAX_NAME+1];
int i, grpid, dimid, typid, varid, attid;
int i, grpid, varid;
int dimids[NDIMS];
if(depth == 0) return;
@ -193,9 +192,7 @@ buildgroup(int parent, int grpindex, int depth)
int
main(int argc, char **argv)
{
int stat = NC_NOERR;
int i, ncid;
int grpid, dimid, varid, typid, attid;
time_t starttime, endtime;
long long delta;
int tag;

6
nc_test4/findplugin.in
View File

@ -99,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH"
return 0
}
# debug
if test "x$1" != x ; then
findplugin
echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|"
fi

3
nc_test4/tst_filter.sh
View File

@ -110,7 +110,8 @@ echo " *** Pass: nccopy simple filter"
echo " *** Testing pass-thru of filters"
rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc
${NCCOPY} ./filtered.nc ./tst_filter2.nc
# Prevent failure by allowing any chunk size
${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc
${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt
sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt
test -s tst_filter2.txt

23
ncdump/CMakeLists.txt
View File

@ -4,18 +4,23 @@ IF(BUILD_SHARED_LIBS AND WIN32)
ENDIF()
SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c)
SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c)
SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c)
SET(ocprint_FILES ocprint.c)
IF(USE_X_GETOPT)
SET(ncdump_FILES ${ncdump_FILES} XGetopt.c)
SET(nccopy_FILES ${nccopy_FILES} XGetopt.c)
SET(ocprint_FILES ${ocprint_FILES} XGetopt.c)
ENDIF()
ADD_EXECUTABLE(ncdump ${ncdump_FILES})
ADD_EXECUTABLE(nccopy ${nccopy_FILES})
ADD_EXECUTABLE(ocprint ${ocprint_FILES})
TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS})
TARGET_LINK_LIBRARIES(ocprint netcdf ${ALL_TLL_LIBS})
####
# We have to do a little tweaking
@ -37,6 +42,13 @@ IF(MSVC)
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
ENDIF()
@ -211,6 +223,11 @@ ENDIF(MSVC)
SET_TESTS_PROPERTIES(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE)
ENDIF(HAVE_BASH)
add_sh_test(ncdump tst_nccopy5)
IF(HAVE_BASH)
SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE)
ENDIF(HAVE_BASH)
IF(USE_NETCDF4)
add_sh_test(ncdump tst_ncgen4)
ENDIF(USE_NETCDF4)
@ -231,12 +248,16 @@ IF(MSVC)
SET_TARGET_PROPERTIES(nccopy
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
SET_TARGET_PROPERTIES(ocprint
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
ENDIF()
INSTALL(TARGETS ncdump RUNTIME DESTINATION bin COMPONENT utilities)
INSTALL(TARGETS nccopy RUNTIME DESTINATION bin COMPONENT utilities)
INSTALL(TARGETS ocprint RUNTIME DESTINATION bin COMPONENT utilities)
SET(MAN_FILES nccopy.1 ncdump.1)
# Note, the L512.bin file is file containing exactly 512 bytes each of value 0.

8
ncdump/Make0
View File

@ -1,12 +1,12 @@
# Test c output
T=tst_unicode
T=tst_chunking
#ARGS=./x.nc
#TF=test_atomic_array.cdl
#CMD=valgrind --leak-check=full
CMD=gdb --args
#CMD=gdb --args
#HDF4=1
#PAR=1
@ -36,11 +36,11 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH}
all:: comp
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CMD} ./t ${ARGS}
${CMD} ./$T ${ARGS}
comp::
export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \
${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
${CC} -o $T ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \
x.nc: x.cdl
ncgen -4 x.cdl

10
ncdump/Makefile.am
View File

@ -26,7 +26,7 @@ utils.c nciter.h nciter.c nccomps.h
# netCDF API
bin_PROGRAMS += nccopy
nccopy_SOURCES = nccopy.c nciter.c nciter.h chunkspec.h chunkspec.c \
utils.h utils.c dimmap.h dimmap.c
utils.h utils.c dimmap.h dimmap.c list.c list.h
# A simple netcdf-4 metadata -> xml printer. Do not install.
if USE_NETCDF4
@ -80,14 +80,14 @@ tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar
# Tests for netCDF-4 behavior.
TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \
tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \
tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \
tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \
run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \
tst_ncgen4.sh
# The tst_nccopy4.sh test script depends on the output of a bunch of
# other tests. Record dependencies so parallel builds work.
# Record interscript dependencies so parallel builds work.
tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log \
tst_fillbug.log tst_netcdf4_4.log tst_h_scalar.log
tst_nccopy5.log: tst_nccopy4.log
endif #!USE_NETCDF4
TESTS += tst_inmemory_nc3.sh
@ -111,7 +111,7 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \
ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \
ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \
tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \
tst_nccopy3.sh tst_nccopy4.sh ref_nc_test_netcdf4_4_0.nc \
tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \
run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \
ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \
ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \

249
ncdump/chunkspec.c
View File

@ -7,38 +7,80 @@
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <netcdf.h>
#include "netcdf.h"
#include "list.h"
#include "utils.h"
#include "chunkspec.h"
/* Structure mapping dimension IDs to corresponding chunksizes. */
static struct {
static struct DimChunkSpecs {
size_t ndims; /* number of dimensions in chunkspec string */
int *dimids; /* ids for dimensions in chunkspec string */
int *idimids; /* (input) ids for dimensions in chunkspec string */
size_t *chunksizes; /* corresponding chunk sizes */
bool_t omit; /* true if chunking to be turned off */
} chunkspecs;
} dimchunkspecs;
struct VarChunkSpec {
size_t rank; /* number of dimensions in chunkspec string */
size_t chunksizes[NC_MAX_VAR_DIMS]; /* corresponding chunk sizes */
bool_t omit; /* true if chunking to be turned off */
int igrpid; /* container of the (input) variable */
int ivarid; /* (input) Variable whose chunks are specified */
};
static List* varchunkspecs = NULL; /* List<VarChunkSpec> */
/* Forward */
static int dimchunkspec_parse(int ncid, const char *spec);
static int varchunkspec_parse(int ncid, const char *spec);
void
chunkspecinit(void)
{
/* initialization */
if(varchunkspecs == NULL)
varchunkspecs = listnew();
memset(&dimchunkspecs,0,sizeof(dimchunkspecs));
}
/*
* Parse chunkspec string and convert into chunkspec_t structure.
* Parse chunkspec string of either kind.
* Returns NC_NOERR if no error, NC_EINVAL if spec was malformed.
*/
int
chunkspec_parse(int igrp, const char *spec)
{
/* Decide if this is a per-variable or per-dimension chunkspec */
if (!spec || *spec == '\0')
return NC_NOERR; /* Use defaults */
if(strchr(spec,':') == NULL)
return dimchunkspec_parse(igrp,spec);
else
return varchunkspec_parse(igrp,spec);
}
/*
* Parse chunkspec string and convert into dimchunkspec structure.
* ncid: location ID of open netCDF file or group in an open file
* spec: string of form
* dim1/n1,dim2/n2,...,dimk/nk
*
* specifying chunk size (ni) to be used for dimension named
* dimi. Dimension names may be absolute,
* e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be
* omitted, in which case it is assumed to be the entire
* dimension size. That is also the default for dimensions
* not mentioned in the string.
* If the chunkspec string is "/", specifying no dimensions or
* chunk sizes, it indicates chunking to be turned off on output.
* specifying chunk size (ni) to be used for dimension named
* dimi. Dimension names may be absolute,
* e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be
* omitted, in which case it is assumed to be the entire
* dimension size. That is also the default for dimensions
* not mentioned in the string. However, for unlimited dimensions,
* the default is a default size: 4 megabytes or the
* existing unlimited size if smaller.
* If the chunkspec string is "/", specifying no dimensions or
* chunk sizes, it indicates chunking to be turned off on output.
*
* Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive
* unescaped commas or no chunksize specified for dimension.
*/
int
chunkspec_parse(int ncid, const char *spec) {
static int
dimchunkspec_parse(int igrp, const char *spec)
{
const char *cp; /* character cursor */
const char *pp = spec; /* previous char cursor for detecting escapes */
const char *np; /* beginning of current dimension name */
@ -47,12 +89,12 @@ chunkspec_parse(int ncid, const char *spec) {
int ret;
int comma_seen = 0;
chunkspecs.ndims = 0;
chunkspecs.omit = false;
dimchunkspecs.ndims = 0;
dimchunkspecs.omit = false;
if (!spec || *spec == '\0') /* default chunking */
return NC_NOERR;
if (spec[0] == '/' && spec[1] == '\0') { /* no chunking */
chunkspecs.omit = true;
dimchunkspecs.omit = true;
return NC_NOERR;
}
/* Count unescaped commas, handle consecutive unescaped commas as error */
@ -69,9 +111,9 @@ chunkspec_parse(int ncid, const char *spec) {
pp = cp;
}
ndims++;
chunkspecs.ndims = ndims;
chunkspecs.dimids = (int *) emalloc(ndims * sizeof(int));
chunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t));
dimchunkspecs.ndims = ndims;
dimchunkspecs.idimids = (int *) emalloc(ndims * sizeof(int));
dimchunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t));
/* Look up dimension ids and assign chunksizes */
pp = spec;
np = spec;
@ -97,15 +139,15 @@ chunkspec_parse(int ncid, const char *spec) {
}
*dp = '\0';
/* look up dimension id from dimension pathname */
ret = nc_inq_dimid2(ncid, dimname, &dimid);
ret = nc_inq_dimid2(igrp, dimname, &dimid);
if(ret != NC_NOERR)
break;
chunkspecs.dimids[idim] = dimid;
dimchunkspecs.idimids[idim] = dimid;
/* parse and assign corresponding chunksize */
pp++; /* now points to first digit of chunksize, ',', or '\0' */
if(*pp == ',' || *pp == '\0') { /* no size specified, use dim len */
size_t dimlen;
ret = nc_inq_dimlen(ncid, dimid, &dimlen);
ret = nc_inq_dimlen(igrp, dimid, &dimlen);
if(ret != NC_NOERR)
return(ret);
chunksize = dimlen;
@ -120,7 +162,7 @@ chunkspec_parse(int ncid, const char *spec) {
return (NC_EINVAL);
chunksize = (size_t)val;
}
chunkspecs.chunksizes[idim] = chunksize;
dimchunkspecs.chunksizes[idim] = chunksize;
idim++;
free(dimname);
if(*cp == '\0')
@ -135,11 +177,11 @@ chunkspec_parse(int ncid, const char *spec) {
/* Return size in chunkspec string specified for dimension corresponding to dimid, 0 if not found */
size_t
chunkspec_size(int dimid) {
dimchunkspec_size(int indimid) {
int idim;
for(idim = 0; idim < chunkspecs.ndims; idim++) {
if(dimid == chunkspecs.dimids[idim]) {
return chunkspecs.chunksizes[idim];
for(idim = 0; idim < dimchunkspecs.ndims; idim++) {
if(indimid == dimchunkspecs.idimids[idim]) {
return dimchunkspecs.chunksizes[idim];
}
}
return 0;
@ -149,15 +191,154 @@ chunkspec_size(int dimid) {
* chunkspec string on command line, 0 if no chunkspec string was
* specified. */
int
chunkspec_ndims(void) {
return chunkspecs.ndims;
dimchunkspec_ndims(void) {
return dimchunkspecs.ndims;
}
/* Return whether chunking should be omitted, due to explicit
* command-line specification. */
bool_t
chunkspec_omit(void) {
return chunkspecs.omit;
dimchunkspec_omit(void) {
return dimchunkspecs.omit;
}
/*
* Parse per-variable chunkspec string and convert into varchunkspec structure.
* ncid: location ID of open netCDF file or group in an open file
* spec: string of form
* var:n1,n2,...nk
*
* specifying chunk size (ni) to be used for ith dimension of
* variable named var. Variable names may be absolute.
* e.g. "/grp_a/grp_a1/var".
* If no chunk sizes are specified, then the variable is not chunked at all.
*
* Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive
* unescaped commas or no chunksize specified for dimension.
*/
static int
varchunkspec_parse(int igrp, const char *spec0)
{
int ret = NC_NOERR;
int rank;
int i;
int dimids[NC_MAX_VAR_DIMS];
struct VarChunkSpec* chunkspec = NULL;
char* spec = NULL;
char* p, *q; /* for walking strings */
/* Copy spec so we can modify in place */
spec = strdup(spec0);
if(spec == NULL) {ret = NC_ENOMEM; goto done;}
chunkspec = calloc(1,sizeof(struct VarChunkSpec));
if(chunkspec == NULL) {ret = NC_ENOMEM; goto done;}
chunkspec->igrpid = igrp;
/* First, find the end of the variable part */
p = strchr(spec,':');
if(p == NULL)
{ret = NC_EINVAL; goto done;}
*p++ = '\0';
/* Lookup the variable by name */
ret = nc_inq_varid2(igrp, spec, &chunkspec->ivarid, &chunkspec->igrpid);
if(ret != NC_NOERR) goto done;
/* Iterate over dimension sizes */
while(*p) {
unsigned long dimsize;
q = strchr(p,',');
if(q == NULL)
q = p + strlen(p); /* Fake the endpoint */
else
*q++ = '\0';
/* Scan as unsigned long */
if(sscanf(p,"%lu",&dimsize) != 1)
{ret = NC_EINVAL; goto done;} /* Apparently not a valid dimension size */
if(chunkspec->rank >= NC_MAX_VAR_DIMS) {ret = NC_EINVAL; goto done;} /* to many chunks */
chunkspec->chunksizes[chunkspec->rank] = (size_t)dimsize;
chunkspec->rank++;
p = q;
}
/* Now do some validity checking */
/* Get some info about the var (from input) */
ret = nc_inq_var(chunkspec->igrpid,chunkspec->ivarid,NULL,NULL,&rank,dimids,NULL);
if(ret != NC_NOERR) goto done;
/* 1. check # chunksizes == rank of variable */
if(rank != chunkspec->rank) {ret = NC_EINVAL; goto done;}
/* 2. check that chunksizes are legal for the given dimension sizes */
for(i=0;i<rank;i++) {
size_t len;
ret = nc_inq_dimlen(igrp,dimids[i],&len);
if(ret != NC_NOERR) goto done;
if(chunkspec->chunksizes[i] > len) {ret = NC_EBADCHUNK; goto done;}
}
/* add the chunkspec to our list */
listpush(varchunkspecs,chunkspec);
chunkspec = NULL;
done:
if(chunkspec != NULL)
free(chunkspec);
if(spec != NULL)
free(spec);
return ret;
}
/* Accessors */
bool_t
varchunkspec_exists(int igrpid, int ivarid)
{
int i;
for(i=0;i<listlength(varchunkspecs);i++) {
struct VarChunkSpec* spec = listget(varchunkspecs,i);
if(spec->igrpid == igrpid && spec->ivarid == ivarid)
return true;
}
return false;
}
bool_t
varchunkspec_omit(int igrpid, int ivarid)
{
int i;
for(i=0;i<listlength(varchunkspecs);i++) {
struct VarChunkSpec* spec = listget(varchunkspecs,i);
if(spec->igrpid == igrpid && spec->ivarid == ivarid)
return spec->omit;
}
return dimchunkspecs.omit;
}
size_t*
varchunkspec_chunksizes(int igrpid, int ivarid)
{
int i;
for(i=0;i<listlength(varchunkspecs);i++) {
struct VarChunkSpec* spec = listget(varchunkspecs,i);
if(spec->igrpid == igrpid && spec->ivarid == ivarid)
return spec->chunksizes;
}
return NULL;
}
size_t
varchunkspec_rank(int igrpid, int ivarid)
{
int i;
for(i=0;i<listlength(varchunkspecs);i++) {
struct VarChunkSpec* spec = listget(varchunkspecs,i);
if(spec->igrpid == igrpid && spec->ivarid == ivarid)
return spec->rank;
}
return 0;
}

17
ncdump/chunkspec.h
View File

@ -15,17 +15,28 @@ chunkspec_parse(int ncid, const char *spec);
/* Return chunk size in chunkspec string specified for dimension
* corresponding to dimid, 0 if not found */
extern size_t
chunkspec_size(int dimid);
dimchunkspec_size(int dimid);
/* Return number of dimensions for which chunking was specified in
* chunkspec string on command line, 0 if no chunkspec string was
* specified. */
extern int
chunkspec_ndims(void);
dimchunkspec_ndims(void);
/* Return whether chunking should be omitted, due to explicit
* command-line specification. */
extern bool_t
chunkspec_omit(void);
dimchunkspec_omit(void);
extern bool_t varchunkspec_omit(int grpid, int varid);
extern size_t* varchunkspec_chunksizes(int grpid, int varid);
extern size_t varchunkspec_ndims(int grpid, int varid);
extern bool_t varchunkspec_exists(int grpid, int varid);
extern void chunkspecinit(void);
#endif /* _CHUNKSPEC_H_ */

221
ncdump/list.c Normal file
View File

@ -0,0 +1,221 @@
/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
See the COPYRIGHT file for more information. */
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "list.h"
int listnull(void* e) {return e == NULL;}
#ifndef TRUE
#define TRUE 1
#endif
#ifndef FALSE
#define FALSE 0
#endif
#define DEFAULTALLOC 16
#define ALLOCINCR 16
List* listnew(void)
{
List* l;
/*
if(!initialized) {
memset((void*)&DATANULL,0,sizeof(void*));
initialized = 1;
}
*/
l = (List*)malloc(sizeof(List));
if(l) {
l->alloc=0;
l->length=0;
l->content=NULL;
}
return l;
}
int
listfree(List* l)
{
if(l) {
l->alloc = 0;
if(l->content != NULL) {free(l->content); l->content = NULL;}
free(l);
}
return TRUE;
}
int
listsetalloc(List* l, unsigned long sz)
{
void** newcontent = NULL;
if(l == NULL) return FALSE;
if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);}
if(l->alloc >= sz) {return TRUE;}
newcontent=(void**)calloc(sz,sizeof(void*));
if(newcontent != NULL && l->alloc > 0 && l->length > 0 && l->content != NULL) {
memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length);
}
if(l->content != NULL) free(l->content);
l->content=newcontent;
l->alloc=sz;
return TRUE;
}
int
listsetlength(List* l, unsigned long sz)
{
if(l == NULL) return FALSE;
if(sz > l->alloc && !listsetalloc(l,sz)) return FALSE;
l->length = sz;
return TRUE;
}
void*
listget(List* l, unsigned long index)
{
if(l == NULL || l->length == 0) return NULL;
if(index >= l->length) return NULL;
return l->content[index];
}
int
listset(List* l, unsigned long index, void* elem)
{
if(l == NULL) return FALSE;
if(index >= l->length) return FALSE;
l->content[index] = elem;
return TRUE;
}
/* Insert at position i of l; will push up elements i..|seq|. */
int
listinsert(List* l, unsigned long index, void* elem)
{
int i; /* do not make unsigned */
if(l == NULL) return FALSE;
if(index > l->length) return FALSE;
listsetalloc(l,0);
for(i=(int)l->length;i>index;i--) l->content[i] = l->content[i-1];
l->content[index] = elem;
l->length++;
return TRUE;
}
int
listpush(List* l, void* elem)
{
if(l == NULL) return FALSE;
if(l->length >= l->alloc) listsetalloc(l,0);
l->content[l->length] = elem;
l->length++;
return TRUE;
}
void*
listpop(List* l)
{
if(l == NULL || l->length == 0) return NULL;
l->length--;
return l->content[l->length];
}
void*
listtop(List* l)
{
if(l == NULL || l->length == 0) return NULL;
return l->content[l->length - 1];
}
void*
listremove(List* l, unsigned long i)
{
unsigned long len;
void* elem;
if(l == NULL || (len=l->length) == 0) return NULL;
if(i >= len) return NULL;
elem = l->content[i];
for(i+=1;i<len;i++) l->content[i-1] = l->content[i];
l->length--;
return elem;
}
/* Duplicate and return the content (null terminate) */
void**
listdup(List* l)
{
void** result = (void**)malloc(sizeof(void*)*(l->length+1));
memcpy((void*)result,(void*)l->content,sizeof(void*)*l->length);
result[l->length] = (void*)0;
return result;
}
int
listcontains(List* l, void* elem)
{
unsigned long i;
for(i=0;i<listlength(l);i++) {
if(elem == listget(l,i)) return 1;
}
return 0;
}
/* Remove element by value; only removes first encountered */
int
listelemremove(List* l, void* elem)
{
unsigned long len;
unsigned long i;
int found = 0;
if(l == NULL || (len=l->length) == 0) return 0;
for(i=0;i<listlength(l);i++) {
void* candidate = l->content[i];
if(elem == candidate) {
for(i+=1;i<len;i++) l->content[i-1] = l->content[i];
l->length--;
found = 1;
break;
}
}
return found;
}
/* Extends list to include a unique operator
which remove duplicate values; NULL values removed
return value is always 1.
*/
int
listunique(List* l)
{
unsigned long i,j,k,len;
void** content;
if(l == NULL || l->length == 0) return 1;
len = l->length;
content = l->content;
for(i=0;i<len;i++) {
for(j=i+1;j<len;j++) {
if(content[i] == content[j]) {
/* compress out jth element */
for(k=j+1;k<len;k++) content[k-1] = content[k];
len--;
}
}
}
l->length = len;
return 1;
}
List*
listclone(List* l)
{
List* clone = listnew();
*clone = *l;
clone->content = listdup(l);
return clone;
}

62
ncdump/list.h Normal file
View File

@ -0,0 +1,62 @@
/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc.
See the COPYRIGHT file for more information. */
#ifndef LIST_H
#define LIST_H 1
/* Define the type of the elements in the list*/
#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__)
#define EXTERNC extern "C"
#else
#define EXTERNC extern
#endif
EXTERNC int listnull(void*);
typedef struct List {
unsigned long alloc;
unsigned long length;
void** content;
} List;
EXTERNC List* listnew(void);
EXTERNC int listfree(List*);
EXTERNC int listsetalloc(List*,unsigned long);
EXTERNC int listsetlength(List*,unsigned long);
/* Set the ith element */
EXTERNC int listset(List*,unsigned long,void*);
/* Get value at position i */
EXTERNC void* listget(List*,unsigned long);/* Return the ith element of l */
/* Insert at position i; will push up elements i..|seq|. */
EXTERNC int listinsert(List*,unsigned long,void*);
/* Remove element at position i; will move higher elements down */
EXTERNC void* listremove(List* l, unsigned long i);
/* Tail operations */
EXTERNC int listpush(List*,void*); /* Add at Tail */
EXTERNC void* listpop(List*);
EXTERNC void* listtop(List*);
/* Duplicate and return the content (null terminate) */
EXTERNC void** listdup(List*);
/* Look for value match */
EXTERNC int listcontains(List*, void*);
/* Remove element by value; only removes first encountered */
EXTERNC int listelemremove(List* l, void* elem);
/* remove duplicates */
EXTERNC int listunique(List*);
/* Create a clone of a list */
EXTERNC List* listclone(List*);
/* Following are always "in-lined"*/
#define listclear(l) listsetlength((l),0)
#define listextend(l,len) listsetalloc((l),(len)+(l->alloc))
#define listcontents(l) ((l)==NULL?NULL:(l)->content)
#define listlength(l) ((l)==NULL?0:(int)(l)->length)
#endif /*LIST_H*/

35
ncdump/nccopy.1
View File

@ -352,6 +352,41 @@ memory before writing it to disk on close:
.HP
nccopy \-w \-c time/1000,lat/40,lon/40 slow.nc fast.nc
.RE
.SH "Chunking Rules"
.LP
The complete set of chunking rules is captured here. As a rough
summary, these rules preserve all chunking properties from the
input file. These rules apply only when the selected output
format supports chunking, i.e. for the netcdf-4 variants.
.LP
The following rules are applied in the given order independently
for each variable to be copied from input to output. The rules are
written assuming we are trying to determine the chunking for a given
output variable Vout that comes from an input variable Vin.
.NP
For each dimension of Vout explicitly specified on the command line
using the '-c' option, apply the chunking value for that
dimension. regardless of input format or input properties.
.NP
For dimensions of V not named on the command line, preserve chunk
sizes from the corresponding input variable.
.NP
If V is netcdf-4 and contiguous, and none of its dimensions are
named on the command line, and chunking is not mandated by other
options, then make V be contiguous.
.NP
If the input variable is contiguous (or is some netcdf-3
variant) and there are no options requiring chunking, or the '/'
special case for the '-c' option is specified, then the output
variable V is marked as contiguous.
.NP
Handle all remaining cases when some or all chunk sizes are not determined by the command line or the input variable. This includes the non-chunked input cases such as netcdf-3, cdf5, and DAP. In these cases:
Retain all chunk sizes determined by (1) and (2); and
Compute the remaining chunk sizes automatically, with some reasonable
.SH "SEE ALSO"
.LP
.BR ncdump(1), ncgen(1), netcdf(3)

533
ncdump/nccopy.c
View File

@ -23,6 +23,7 @@
#include "dimmap.h"
#include "nccomps.h"
#include "ncfilter.h"
#include "list.h"
#undef DEBUGFILTER
@ -48,6 +49,8 @@ int optind;
#define ESCAPESD "0123456789"
#define ESCAPES " !\"#$%&'()*,:;<=>?[]\\^`{|}~"
#define DFALTUNLIMSIZE (4* 1<<20) /*4 megabytes */
#ifdef USE_NETCDF4
/* The unique id for a variable requires also the enclosing group id */
@ -73,20 +76,65 @@ static int suppressfilters = 0; /* 1 => do not apply any output filters unless s
#endif
/* table of formats for legal -k values */
static struct Kvalues {
char* name;
int kind;
} legalkinds[] = {
/* NetCDF-3 classic format (32-bit offsets) */
{"classic", NC_FORMAT_CLASSIC}, /* canonical format name */
{"nc3", NC_FORMAT_CLASSIC}, /* short format name */
{"1", NC_FORMAT_CLASSIC}, /* deprecated, use "-3" or "-k nc3" instead */
/* NetCDF-3 64-bit offset format */
{"64-bit offset", NC_FORMAT_64BIT_OFFSET}, /* canonical format name */
{"nc6", NC_FORMAT_64BIT_OFFSET}, /* short format name */
{"2", NC_FORMAT_64BIT_OFFSET}, /* deprecated, use "-6" or "-k nc6" instead */
{"64-bit-offset", NC_FORMAT_64BIT_OFFSET}, /* deprecated alias */
/* NetCDF-4 HDF5-based format */
{"netCDF-4", NC_FORMAT_NETCDF4}, /* canonical format name */
{"nc4", NC_FORMAT_NETCDF4}, /* short format name */
{"3", NC_FORMAT_NETCDF4}, /* deprecated, use "-4" or "-k nc4" instead */
{"netCDF4", NC_FORMAT_NETCDF4}, /* deprecated aliases */
{"hdf5", NC_FORMAT_NETCDF4},
{"enhanced", NC_FORMAT_NETCDF4},
/* NetCDF-4 HDF5-based format, restricted to classic data model */
{"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC}, /* canonical format name */
{"nc7", NC_FORMAT_NETCDF4_CLASSIC}, /* short format name */
{"4", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated, use "-7" or -k nc7" */
{"netCDF-4-classic", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated aliases */
{"netCDF-4_classic", NC_FORMAT_NETCDF4_CLASSIC},
{"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC},
{"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC},
{"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC},
/* The 64-bit data (CDF5) kind (5) */
{"5", NC_FORMAT_CDF5},
{"64-bit-data", NC_FORMAT_CDF5},
{"64-bit data", NC_FORMAT_CDF5},
{"nc5", NC_FORMAT_CDF5},
{"cdf5", NC_FORMAT_CDF5},
/* null terminate*/
{NULL,0}
};
/* Global variables for command-line requests */
char *progname; /* for error messages */
static int option_kind = SAME_AS_INPUT;
static int option_deflate_level = -1; /* default, compress output only if input compressed */
static int option_shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */
static int option_fix_unlimdims = 0; /* default, preserve unlimited dimensions */
static char* option_chunkspec = 0; /* default, no chunk specification */
static List* option_chunkspecs = NULL; /* default, no chunk specification */
static size_t option_copy_buffer_size = COPY_BUFFER_SIZE;
static size_t option_chunk_cache_size = CHUNK_CACHE_SIZE; /* default from config.h */
static size_t option_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /* default from config.h */
static int option_read_diskless = 0; /* default, don't read input into memory on open */
static int option_write_diskless = 0; /* default, don't write output to diskless file */
#ifdef USE_NETCDF4
static int option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of
static size_t option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of
* chunksizes of its dimensions is smaller
* than this */
#endif
@ -97,9 +145,9 @@ static char** option_lgrps = 0; /* list of group names specified with -g
static idnode_t* option_grpids = 0; /* list of grpids matching list specified with -g option */
static bool_t option_grpstruct = false; /* if -g set, copy structure for non-selected groups */
static int option_nlvars = 0; /* Number of variables specified with -v * option on command line */
static char** option_lvars = 0; /* list of variable names specified with -v
* option on command line */
static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */
static char** option_lvars = 0; /* list of variable names specified with -v
* option on command line */
static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */
static int option_compute_chunkcaches = 0; /* default, don't try still flaky estimate of
* chunk cache for each variable */
@ -186,7 +234,7 @@ computeFQN(VarID vid, char** fqnp)
q = escname;
for(first=1;*p;first=0) {
if((first && strchr(ESCAPESD,*p) != NULL)
|| strchr(ESCAPES,*p) != NULL) *q++ = '\\';
|| strchr(ESCAPES,*p) != NULL) *q++ = '\\';
*q++ = *p++;
}
*q++ = '\0'; /* guarantee */
@ -196,44 +244,6 @@ done:
return stat;
}
#if 0
static int
parseFQN(int ncid, const char* fqn0, VarID* idp)
{
int stat = NC_NOERR;
char* fqn;
VarID vid;
char* p;
char* q;
char* segment;
vid.grpid = ncid;
if(fqn0 == NULL || fqn0[1] != '/')
{stat = NC_EBADNAME; goto done;}
fqn = strdup(fqn0+1); /* skip leading '/'*/
p = fqn;
for(;;) {
int newgrp;
segment = p;
q = p;
while(*p != '\0' && *p != '/') {
if(*p == '\\') p++;
*q++ = *p++;
}
if(*p == '\0') break;
*p++ = '\0';
if((stat=nc_inq_grp_ncid(vid.grpid,segment,&newgrp))) goto done;
vid.grpid = newgrp;
}
/* Segment should point to the varname */
if((stat=nc_inq_varid(vid.grpid,segment,&vid.varid))) goto done;
done:
if(fqn) free(fqn);
if(stat == NC_NOERR && idp != NULL) *idp = vid;
return stat;
}
#endif
static int
parsefilterspec(const char* optarg0, struct FilterSpec* spec)
{
@ -266,9 +276,9 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec)
if(optarg[0]=='/')
spec->fqn = strdup(optarg);
else {
spec->fqn = (char*)malloc(1+strlen(optarg)+1);
spec->fqn = (char*)malloc(1+strlen(optarg)+1);
strcpy(spec->fqn,"/");
strcat(spec->fqn,optarg);
strcat(spec->fqn,optarg);
}
/* Check for special cases */
@ -280,10 +290,10 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec)
/* Collect the id+parameters */
if((stat = NC_parsefilterspec(remainder,&id,&nparams,&params)) == NC_NOERR) {
if(spec != NULL) {
spec->filterid = id;
spec->nparams = nparams;
spec->params = params;
if(spec != NULL) {
spec->filterid = id;
spec->nparams = nparams;
spec->params = params;
}
}
@ -336,7 +346,7 @@ static int
inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid,
size_t* chunkcache_sizep,
size_t *chunkcache_nelemsp,
float * chunkcache_preemptionp)
float * chunkcache_preemptionp)
{
int stat = NC_NOERR;
int ndims;
@ -583,7 +593,7 @@ copy_groups(int iroot, int oroot)
/* get full group name of input group */
NC_CHECK(nc_inq_grpname(grpids[i], grpname));
if (option_grpstruct || group_wanted(grpids[i], option_nlgrps, option_grpids)) {
NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL));
NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL));
grpname_full = emalloc(len_name + 1);
NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, grpname_full));
/* Make sure, the parent group is also wanted (root group is always wanted) */
@ -653,90 +663,10 @@ copy_types(int igrp, int ogrp)
return stat;
}
/* Copy all netCDF-4 specific variable properties such as chunking,
* endianness, deflation, checksumming, fill, etc. */
static int
copy_var_specials(int igrp, int varid, int ogrp, int o_varid)
{
int stat = NC_NOERR;
{ /* handle chunking parameters */
int ndims;
NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
if (ndims > 0) { /* no chunking for scalar variables */
int contig = 0;
size_t *chunkp = (size_t *) emalloc(ndims * sizeof(size_t));
int *dimids = (int *) emalloc(ndims * sizeof(int));
int idim;
/* size of a chunk: product of dimension chunksizes and size of value */
size_t csprod = val_size(ogrp, o_varid);
int is_unlimited = 0;
NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, chunkp));
NC_CHECK(nc_inq_vardimid(igrp, varid, dimids));
for(idim = 0; idim < ndims; idim++) {
int idimid = dimids[idim];
int odimid = dimmap_odimid(idimid);
size_t chunksize = chunkspec_size(idimid);
if(chunksize > 0) { /* found in chunkspec */
chunkp[idim] = chunksize;
}
csprod *= chunkp[idim];
if(dimmap_ounlim(odimid))
is_unlimited = 1;
}
/* Explicitly set chunking, even if default */
/* If product of chunksizes is too small and no unlimited
* dimensions used, don't chunk. Also if chunking
* explicitly turned off with chunk spec, don't chunk. */
if ((csprod < option_min_chunk_bytes && !is_unlimited) || contig == 1
|| chunkspec_omit() == true) {
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL));
} else {
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp));
}
free(dimids);
free(chunkp);
}
}
{ /* handle compression parameters, copying from input, overriding
* with command-line options */
int shuffle_in=0, deflate_in=0, deflate_level_in=0;
int shuffle_out=0, deflate_out=0, deflate_level_out=0;
if(option_deflate_level != 0) {
NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in));
if(option_deflate_level == -1) { /* not specified, copy input compression and shuffling */
shuffle_out = shuffle_in;
deflate_out = deflate_in;
deflate_level_out = deflate_level_in;
} else if(option_deflate_level > 0) { /* change to specified compression, shuffling */
shuffle_out = option_shuffle_vars;
deflate_out=1;
deflate_level_out = option_deflate_level;
}
NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out));
}
}
{ /* handle checksum parameters */
int fletcher32 = 0;
NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32));
if(fletcher32 != 0) {
NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32));
}
}
{ /* handle endianness */
int endianness = 0;
NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness));
if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */
NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness));
}
}
return stat;
}
/* Copy netCDF-4 specific variable filter properties */
/* Watch out if input is netcdf-3 */
static int
copy_var_filter(int igrp, int varid, int ogrp, int o_varid)
copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind)
{
int stat = NC_NOERR;
VarID vid = {igrp,varid};
@ -747,14 +677,11 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid)
actualspec = {NULL,0,0,0,NULL};
int i;
char* ofqn = NULL;
int format, oformat;
int inputdefined, outputdefined, unfiltered;
int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC);
int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC);
/* Get file format of the input and output */
if((stat=nc_inq_format(vid.grpid,&format))) goto done;
if((stat=nc_inq_format(ovid.grpid,&oformat))) goto done;
if(oformat != NC_FORMAT_NETCDF4 && oformat != NC_FORMAT_NETCDF4_CLASSIC)
if(!outnc4)
goto done; /* Can only use filter when output is some netcdf4 variant */
/* Compute the output vid's FQN */
@ -768,7 +695,7 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid)
/* Is there a filter on the output variable */
outputdefined = 0; /* default is no filter defined */
/* Only bother to look if output is netcdf-4 variant */
if(oformat == NC_FORMAT_NETCDF4 || oformat == NC_FORMAT_NETCDF4_CLASSIC) {
if(outnc4) {
/* See if any output filter spec is defined for this output variable */
for(i=0;i<nfilterspecs;i++) {
if(strcmp(filterspecs[i].fqn,ofqn)==0) {
@ -782,7 +709,7 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid)
/* Is there a filter on the input variable */
inputdefined = 0; /* default is no filter defined */
/* Only bother to look if input is netcdf-4 variant */
if(format == NC_FORMAT_NETCDF4 || format == NC_FORMAT_NETCDF4_CLASSIC) {
if(innc4) {
stat=nc_inq_var_filter(vid.grpid,vid.varid,&inspec.filterid,&inspec.nparams,NULL);
if(stat && stat != NC_EFILTER)
goto done; /* true error */
@ -839,6 +766,218 @@ done:
return stat;
}
/* Propagate chunking from input to output taking -c flags into account. */
/* Subsumes old set_var_chunked */
static int
copy_chunking(int igrp, int i_varid, int ogrp, int o_varid, int ndims, int inkind, int outkind)
{
int stat = NC_NOERR;
int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC);
int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC);
/* First, check the file kinds */
if(!outnc4)
return stat; /* no chunking */
/* See if a scalar */
if(ndims == 0)
return stat; /* scalars cannot be chunked */
/* If var specific chunking was specified for this output variable
then it overrides all else.
*/
/* Note, using goto done instead of nested if-then-else */
if(varchunkspec_exists(igrp,i_varid)) {
if(varchunkspec_omit(igrp,i_varid)) {
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL));
} else {
size_t* ochunkp = varchunkspec_chunksizes(igrp,i_varid);
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp));
}
goto done;
}
{ /* Try dim-specific chunking */
int idim;
/* size of a chunk: product of dimension chunksizes and size of value */
size_t csprod;
int is_unlimited = 0;
size_t typesize;
size_t ichunkp[NC_MAX_VAR_DIMS];
size_t ochunkp[NC_MAX_VAR_DIMS];
int dimids[NC_MAX_VAR_DIMS];
int icontig = 1;
int ocontig = 1; /* until proven otherwise */
/* See if chunking was suppressed */
if(dimchunkspec_omit())
goto done; /* do nothing */
/* Setup for chunking */
typesize = val_size(ogrp, o_varid);
csprod = typesize;
memset(&dimids,0,sizeof(dimids));
memset(&ichunkp,0,sizeof(ichunkp));
memset(&ochunkp,0,sizeof(ochunkp));
/* Get the chunking, if any, on the current input variable */
NC_CHECK(nc_inq_var_chunking(igrp, i_varid, &icontig, ichunkp));
if(!icontig)
ocontig = 0; /* If input is chunked, then so is output */
/* Prepare to iterate over the dimids of this input variable */
NC_CHECK(nc_inq_vardimid(igrp, i_varid, dimids));
/* Assign chunk sizes for all dimensions of variable;
even if we decide to not chunk */
for(idim = 0; idim < ndims; idim++) {
int idimid = dimids[idim];
int odimid = dimmap_odimid(idimid);
size_t chunksize;
size_t dimlen;
/* Get input dimension length */
NC_CHECK(nc_inq_dimlen(igrp, idimid, &dimlen));
/* Check for unlimited */
if(dimmap_ounlim(odimid)) {
is_unlimited = 1;
ocontig = 0; /* force chunking */
}
/* If the -c set a chunk size for this dimension, use it */
chunksize = dimchunkspec_size(idimid);
if(chunksize > 0) { /* found in chunkspec */
ochunkp[idim] = chunksize;
ocontig = 0; /* cannot use contiguous */
goto next;
}
/* Not specified in -c; Apply defaulting rules as defined in nccopy.1 */
/* If input is chunked, then use that chunk size */
if(!icontig) {
ochunkp[idim] = ichunkp[idim];
ocontig = 0;
goto next;
}
/* If input is netcdf-4 then use the input size as the chunk size;
but do not force chunking.
*/
if(!innc4) {
ochunkp[idim] = dimlen;
goto next;
}
/* Default for unlimited is max(4 megabytes, current dim size) */
if(is_unlimited) {
size_t mb4dimsize = DFALTUNLIMSIZE / typesize;
ochunkp[idim] = (dimlen > mb4dimsize ? mb4dimsize : dimlen);
} else {
/* final default is the current dimension size */
ochunkp[idim] = dimlen;
}
next:
/* compute on-going dimension product */
csprod *= ochunkp[idim];
}
/* Finally, if total chunksize is too small (and dim is not unlimited) => do not chunk */
if(csprod < option_min_chunk_bytes && !is_unlimited) {
ocontig = 1; /* Force contiguous */
}
/* Apply the chunking, if any */
if(ocontig) { /* We can use contiguous output */
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL));
} else {
NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp));
}
}
done:
return stat;
}
/* Copy all netCDF-4 specific variable properties such as chunking,
* endianness, deflation, checksumming, fill, etc. */
static int
copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind)
{
int stat = NC_NOERR;
int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC);
int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC);
if(!outnc4)
return stat; /* Ignore non-netcdf4 files */
{ /* handle chunking parameters */
int ndims;
NC_CHECK(nc_inq_varndims(igrp, varid, &ndims));
if (ndims > 0) { /* no chunking for scalar variables */
NC_CHECK(copy_chunking(igrp, varid, ogrp, o_varid, ndims, inkind, outkind));
}
}
{ /* handle compression parameters, copying from input, overriding
* with command-line options */
int shuffle_in=0, deflate_in=0, deflate_level_in=0;
int shuffle_out=0, deflate_out=0, deflate_level_out=0;
if(innc4) { /* See if the input variable has deflation applied */
NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in));
}
if(option_deflate_level == -1) {
/* not specified by -d flag, copy input compression and shuffling */
shuffle_out = shuffle_in;
deflate_out = deflate_in;
deflate_level_out = deflate_level_in;
} else if(option_deflate_level > 0) { /* change to specified compression, shuffling */
shuffle_out = option_shuffle_vars;
deflate_out=1;
deflate_level_out = option_deflate_level;
} else if(option_deflate_level == 0) { /* special case; force off */
shuffle_out = 0;
deflate_out = 0;
deflate_level_out = 0;
}
/* Apply output deflation */
if(outnc4) {
/* Note that if we invoke this function and even if shuffle and deflate flags are 0,
then default chunking will be turned on; so do a special check for that. */
if(shuffle_out != 0 || deflate_out != 0)
NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out));
}
}
if(innc4 && outnc4)
{ /* handle checksum parameters */
int fletcher32 = 0;
NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32));
if(fletcher32 != 0) {
NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32));
}
}
if(innc4 && outnc4)
{ /* handle endianness */
int endianness = 0;
NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness));
if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */
NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness));
}
}
/* handle other general filters */
NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid, inkind, outkind));
return stat;
}
#if 0
Subsumed into copy_chunking.
/* Set output variable o_varid (in group ogrp) to use chunking
* specified on command line, only called for classic format input and
* netCDF-4 format output, so no existing chunk lengths to override. */
@ -850,7 +989,7 @@ set_var_chunked(int ogrp, int o_varid)
int odim;
size_t chunk_threshold = CHUNK_THRESHOLD;
if(chunkspec_ndims() == 0) /* no chunking specified on command line */
if(dimchunkspec_ndims() == 0) /* no chunking specified on command line */
return stat;
NC_CHECK(nc_inq_varndims(ogrp, o_varid, &ndims));
@ -878,7 +1017,7 @@ set_var_chunked(int ogrp, int o_varid)
if(dimmap_ounlim(odimid))
is_unlimited = 1; /* whether vriable is unlimited */
if(idimid != -1) {
size_t chunksize = chunkspec_size(idimid); /* from chunkspec */
size_t chunksize = dimchunkspec_size(idimid); /* from chunkspec */
size_t dimlen;
NC_CHECK(nc_inq_dimlen(ogrp, odimid, &dimlen));
if( (chunksize > 0) || dimmap_ounlim(odimid)) {
@ -901,7 +1040,7 @@ set_var_chunked(int ogrp, int o_varid)
for(odim = 0; odim < ndims; odim++) {
int odimid = dimids[odim];
int idimid = dimmap_idimid(odimid);
size_t chunksize = chunkspec_size(idimid);
size_t chunksize = dimchunkspec_size(idimid);
if(chunksize > 0) {
chunkp[odim] = chunksize;
} else {
@ -919,7 +1058,9 @@ set_var_chunked(int ogrp, int o_varid)
}
return stat;
}
#endif
#if 0
/* Set variable to compression specified on command line */
static int
set_var_compressed(int ogrp, int o_varid)
@ -931,6 +1072,7 @@ set_var_compressed(int ogrp, int o_varid)
}
return stat;
}
#endif
/* Release the variable chunk cache allocated for variable varid in
* group grp. This is not necessary, but will save some memory when
@ -1110,19 +1252,11 @@ copy_var(int igrp, int varid, int ogrp)
int outkind;
NC_CHECK(nc_inq_format(igrp, &inkind));
NC_CHECK(nc_inq_format(ogrp, &outkind));
if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) {
if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) {
/* Copy all netCDF-4 specific variable properties such as
* chunking, endianness, deflation, checksumming, fill, etc. */
NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid));
} else {
/* Set chunking if specified in command line option */
NC_CHECK(set_var_chunked(ogrp, o_varid));
/* Set compression if specified in command line option */
NC_CHECK(set_var_compressed(ogrp, o_varid));
}
NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid));
}
/* Copy all variable properties such as
* chunking, endianness, deflation, checksumming, fill, etc.
* Ok to call if outkind is netcdf-3
*/
NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid, inkind, outkind));
}
#endif /* USE_NETCDF4 */
free(idimids);
@ -1667,7 +1801,7 @@ copy(char* infile, char* outfile)
|| inkind == NC_FORMAT_CDF5) {
if (option_deflate_level > 0 ||
option_shuffle_vars == NC_SHUFFLE ||
option_chunkspec)
listlength(option_chunkspecs) > 0)
{
outkind = NC_FORMAT_NETCDF4_CLASSIC;
}
@ -1675,10 +1809,14 @@ copy(char* infile, char* outfile)
}
#ifdef USE_NETCDF4
if(option_chunkspec) {
/* Now that input is open, can parse option_chunkspec into binary
if(listlength(option_chunkspecs) > 0) {
int i;
/* Now that input is open, can parse option_chunkspecs into binary
* structure. */
NC_CHECK(chunkspec_parse(igrp, option_chunkspec));
for(i=0;i<listlength(option_chunkspecs);i++) {
char* spec = (char*)listget(option_chunkspecs,i);
NC_CHECK(chunkspec_parse(igrp, spec));
}
}
#endif /* USE_NETCDF4 */
@ -1862,50 +2000,8 @@ main(int argc, char**argv)
struct FilterSpec filterspec;
#endif
/* table of formats for legal -k values */
struct Kvalues {
char* name;
int kind;
} legalkinds[] = {
/* NetCDF-3 classic format (32-bit offsets) */
{"classic", NC_FORMAT_CLASSIC}, /* canonical format name */
{"nc3", NC_FORMAT_CLASSIC}, /* short format name */
{"1", NC_FORMAT_CLASSIC}, /* deprecated, use "-3" or "-k nc3" instead */
/* NetCDF-3 64-bit offset format */
{"64-bit offset", NC_FORMAT_64BIT_OFFSET}, /* canonical format name */
{"nc6", NC_FORMAT_64BIT_OFFSET}, /* short format name */
{"2", NC_FORMAT_64BIT_OFFSET}, /* deprecated, use "-6" or "-k nc6" instead */
{"64-bit-offset", NC_FORMAT_64BIT_OFFSET}, /* deprecated alias */
/* NetCDF-4 HDF5-based format */
{"netCDF-4", NC_FORMAT_NETCDF4}, /* canonical format name */
{"nc4", NC_FORMAT_NETCDF4}, /* short format name */
{"3", NC_FORMAT_NETCDF4}, /* deprecated, use "-4" or "-k nc4" instead */
{"netCDF4", NC_FORMAT_NETCDF4}, /* deprecated aliases */
{"hdf5", NC_FORMAT_NETCDF4},
{"enhanced", NC_FORMAT_NETCDF4},
/* NetCDF-4 HDF5-based format, restricted to classic data model */
{"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC}, /* canonical format name */
{"nc7", NC_FORMAT_NETCDF4_CLASSIC}, /* short format name */
{"4", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated, use "-7" or -k nc7" */
{"netCDF-4-classic", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated aliases */
{"netCDF-4_classic", NC_FORMAT_NETCDF4_CLASSIC},
{"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC},
{"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC},
{"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC},
/* The 64-bit data (CDF5) kind (5) */
{"5", NC_FORMAT_CDF5},
{"64-bit-data", NC_FORMAT_CDF5},
{"64-bit data", NC_FORMAT_CDF5},
{"nc5", NC_FORMAT_CDF5},
{"cdf5", NC_FORMAT_CDF5},
/* null terminate*/
{NULL,0}
};
chunkspecinit();
option_chunkspecs = listnew();
opterr = 1;
progname = argv[0];
@ -1915,7 +2011,7 @@ main(int argc, char**argv)
usage();
}
while ((c = getopt(argc, argv, "k:3467d:sum:c:h:e:rwxg:G:v:V:F:L:")) != -1) {
while ((c = getopt(argc, argv, "k:3467d:sum:c:h:e:rwxg:G:v:V:F:L:M:")) != -1) {
switch(c) {
case 'k': /* for specifying variant of netCDF format to be generated
Format names:
@ -2008,7 +2104,7 @@ main(int argc, char**argv)
break;
case 'c': /* optional chunking spec for each dimension in list */
/* save chunkspec string for parsing later, once we know input ncid */
option_chunkspec = strdup(optarg);
listpush(option_chunkspecs,strdup(optarg));
break;
case 'g': /* group names */
/* make list of names of groups specified */
@ -2030,15 +2126,15 @@ main(int argc, char**argv)
make_lvars (optarg, &option_nlvars, &option_lvars);
option_varstruct = false;
break;
case 'L': /* Set logging, if logging support was compiled in. */
case 'L': /* Set logging, if logging support was compiled in. */
#ifdef LOGGING
{
int level = atoi(optarg);
if(level >= 0)
nc_set_log_level(level);
}
{
int level = atoi(optarg);
if(level >= 0)
nc_set_log_level(level);
}
#endif
break;
break;
case 'F': /* optional filter spec for a specified variable */
#ifdef USE_NETCDF4
/* If the arg is "none" then suppress all filters
@ -2058,7 +2154,16 @@ main(int argc, char**argv)
#else
error("-F requires netcdf-4");
#endif
case 'M': /* set min chunk size */
#ifdef USE_NETCDF4
option_min_chunk_bytes = atol(optarg);
if(option_min_chunk_bytes < 0)
error("-M value must be non-negative integer");
break;
#else
error("-M requires netcdf-4");
#endif
default:
usage();
}

130
ncdump/tst_chunking.c
View File

@ -3,49 +3,135 @@
conditions of use. See www.unidata.ucar.edu for more info.
Create a chunkable test file for nccopy to test chunking.
$Id$
*/
#include <netcdf.h>
#include <nc_tests.h>
#include "err_macros.h"
#define DEBUG
static int ret = NC_NOERR;
/* Make trackable ERR macro replacement */
static int lerr(int stat, const char* file, int lineno) {
fflush(stdout); /* Make sure our stdout is synced with stderr. */
err++;
fprintf(stderr, "Sorry! Unexpected result(%d), %s, line: %d\n",ret,file,lineno);
fflush(stderr); \
return 2; \
}
#define LERR lerr(ret,__FILE__,__LINE__)
#define FILE_NAME "tst_chunking.nc"
#define VAR_RANK 7
#define IVAR_NAME "ivar"
#define FVAR_NAME "fvar"
#define GRP_NAME "g"
#define UNLIM_NAME "unlimited"
#define UNLIM_SIZE 10
#define DEFLATE_LEVEL 1
#define NVALS 45360 /* 7 * 4 * 2 * 3 * 5 * 6 * 9 */
static const char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"};
static const size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9};
int
main(int argc, char **argv) {
int ncid;
main(int argc, char **argv)
{
/* mutually exclusive command line options */
int option_group = 0;
int option_deflate = 0;
int option_unlimited = 0;
/* file metadata */
int mode = NC_CLOBBER;
int ncid, grpid;
int ivarid, fvarid;
char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"};
size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9};
int ivar_dims[VAR_RANK];
int fvar_dims[VAR_RANK];
int ivar_data[NVALS];
float fvar_data[NVALS];
int r, i;
char* file_name = FILE_NAME;
int unlimid;
printf("*** Creating chunkable test file %s...", FILE_NAME);
if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR;
for(r = 0; r < VAR_RANK; r++) {
if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) ERR;
fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r];
}
if (nc_def_var(ncid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) ERR;
if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) ERR;
if (nc_enddef (ncid)) ERR;
for(i=0; i < NVALS; i++) {
ivar_data[i] = i;
fvar_data[i] = NVALS - i;
/* Parse command line */
if(argc >= 2) {
file_name = argv[1];
}
if (nc_put_var(ncid, ivarid, ivar_data)) ERR;
if (nc_put_var(ncid, fvarid, fvar_data)) ERR;
if (nc_close(ncid)) ERR;
if(argc >= 3) {
if(strcmp(argv[2],"group")==0) {
option_group = 1;
mode |= NC_NETCDF4;
} else if(strcmp(argv[2],"deflate")==0) {
option_deflate = 1;
mode |= NC_NETCDF4;
} else if(strcmp(argv[2],"unlimited")==0) {
option_unlimited = 1;
} else {
fprintf(stderr,"usage: tst_chunking [<filename> [group|deflate|unlimited]]\n");
exit(1);
}
}
printf("*** Creating chunkable test file %s...\n", file_name);
if(option_deflate)
printf("\toption: deflate\n");
else if(option_unlimited)
printf("\toption: unlimited\n");
else if(option_group)
printf("\toption: group\n");
if (nc_create(file_name, mode, &ncid)) LERR;
for(r = 0; r < VAR_RANK; r++) {
if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) LERR;
fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r];
}
if(option_unlimited) {
int udims[2];
if (nc_def_dim(ncid, UNLIM_NAME, 0, &unlimid)) LERR;
udims[0] = unlimid;
udims[1] = ivar_dims[0];
if (nc_def_var(ncid, IVAR_NAME, NC_INT, 2, udims, &ivarid)) LERR;
} else {
if (option_group) {
if (nc_def_grp(ncid, GRP_NAME, &grpid)) LERR;
} else
grpid = ncid;
if (nc_def_var(grpid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) LERR;
if(option_deflate) {
if(nc_def_var_deflate(grpid,ivarid,NC_NOSHUFFLE, option_deflate, DEFLATE_LEVEL)) LERR;
}
}
/* fvar is unchanged */
if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) LERR;
if (nc_enddef (ncid)) LERR;
/* Fill in the data */
if(option_unlimited) {
int nvals = UNLIM_SIZE * dim_lens[0];
size_t start[2] = {0,0};
size_t count[2];
for(i=0;i<nvals;i++) {
ivar_data[i] = i;
}
count[0] = UNLIM_SIZE;
count[1] = dim_lens[0];
if (nc_put_vara(ncid, ivarid, start, count, ivar_data)) LERR;
} else {
for(i=0; i < NVALS; i++) {
ivar_data[i] = i;
}
if (nc_put_var(ncid, ivarid, ivar_data)) LERR;
}
/* fvar is unchanged */
for(i=0; i < NVALS; i++) {
fvar_data[i] = NVALS - i;
}
if (nc_put_var(ncid, fvarid, fvar_data)) LERR;
if (nc_close(ncid)) LERR;
SUMMARIZE_ERR;
FINAL_RESULTS;
}

2
ncdump/tst_nccopy4.sh
View File

@ -83,7 +83,7 @@ diff tmp.cdl tmp-chunked.cdl
$NCCOPY -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc
${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl
diff tmp.cdl tmp-unchunked.cdl
$NCCOPY -c / tmp-chunked.nc tmp-unchunked.nc
$NCCOPY -c // tmp-chunked.nc tmp-unchunked.nc
${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl
diff tmp.cdl tmp-unchunked.cdl
echo "*** Test that nccopy -c works as intended for record dimension default (1)"

167
ncdump/tst_nccopy5.sh Executable file
View File

@ -0,0 +1,167 @@
#!/bin/bash
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# If we want to run valgrind
#NCCOPY="valgrind --leak-check=full ${NCCOPY}"
# Choose tests to run
T1=1
T2=1
T3=1
T4=1
# For a netCDF-4 build, test nccopy chunking rules
set -e
echo ""
# Trim off leading and trailing whitespace
# Also remove any <cr>
# usage: trim <line>
# Leaves result in variable TRIMMED
trim() {
# trim leading whitespace and remove <cr>
TMP=`echo "$1" |tr -d '\r' | sed -e 's/^[ ]*//'`
# trim trailing whitespace
TRIMMED=`echo "$TMP" | sed -e 's/[ ]*$//'`
}
# usage: checkfvar <file>
checkfvar() {
# Make sure that fvar was not chunked
C5FVAR=`sed -e '/fvar:_ChunkSizes/p' -e d <$1`
if test "x$C5FVAR" != x ; then
echo "***Fail: fvar was chunked"
exit 1
fi
}
# usage: verifychunkline line1 line2
verifychunkline() {
# trim leading whitespace
trim "$1"; L1="$TRIMMED"
trim "$2"; L2="$TRIMMED"
if test "x$L1" != "x$L2" ; then
echo "chunk line mismatch |$L1| |$L2|"
exit 1;
fi
}
# Remove any temporary files
cleanup() {
rm -f tmp_nc5.nc tmp_nc5a.nc
rm -f tmp_nc5.cdl tmp_nc5a.cdl tmp_nc5b.cdl
}
# remove all created files
reset() {
cleanup
rm -fr tst_nc5.nc tst_nc5.cdl
}
reset
if test "x$T1" = x1 ; then
# Create a simple classic input file
./tst_chunking tst_nc5.nc
# Save a .cdl version
${NCDUMP} tst_nc5.nc > tst_nc5.cdl
echo "*** Test nccopy -c with per-variable chunking; classic->enhanced"
# This should produce same as -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/
${NCCOPY} -c ivar:7,1,2,1,5,1,9 tst_nc5.nc tmp_nc5.nc
${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
# Verify that the core cdl is the same
diff tst_nc5.cdl tmp_nc5.cdl
# Look at the output chunking of ivar
rm -f tmp_nc5a.cdl # reuse
${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
# extract the chunking line
TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d <tmp_nc5.cdl`
# track line to match
BASELINE='ivar:_ChunkSizes = 7, 1, 2, 1, 5, 1, 9 ;'
verifychunkline "$TESTLINE" "$BASELINE"
# Make sure that fvar was not chunked
checkfvar tmp_nc5.cdl
fi # T1
if test "x$T2" = x1 ; then
echo "*** Test nccopy -c with per-variable chunking; enhanced->enhanced"
reset
./tst_chunking tst_nc5.nc deflate
${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl
${NCCOPY} -c ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc
${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
diff tst_nc5.cdl tmp_nc5.cdl
# Look at the output chunking
rm -f tmp_nc5.cdl # reuse
${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
# extract the chunking line
TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d <tmp_nc5.cdl`
# track line to match
BASELINE='ivar:_ChunkSizes = 4, 1, 2, 1, 5, 2, 3 ;'
verifychunkline "$TESTLINE" "$BASELINE"
# Make sure that fvar was not chunked
checkfvar tmp_nc5.cdl
fi # T2
if test "x$T3" = x1 ; then
echo "*** Test nccopy -c with FQN var name; enhanced ->enhanced"
reset
./tst_chunking tst_nc5.nc group
${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl
${NCCOPY} -c /g/ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc
${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
diff tst_nc5.cdl tmp_nc5.cdl
# Verify chunking
${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
# extract the chunking line
TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d <tmp_nc5.cdl`
# track line to match
BASELINE='ivar:_ChunkSizes = 4, 1, 2, 1, 5, 2, 3 ;'
verifychunkline "$TESTLINE" "$BASELINE"
fi #T3
if test "x$T4" = x1 ; then
echo "*** Test nccopy -c with unlimited dimension; classic ->enhanced"
reset
./tst_chunking tst_nc5.nc unlimited
${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl
${NCCOPY} -c ivar:5,3 tst_nc5.nc tmp_nc5.nc
${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
diff tst_nc5.cdl tmp_nc5.cdl
# Verify chunking
${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl
# extract the chunking line
TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d <tmp_nc5.cdl`
# track line to match
BASELINE=' ivar:_ChunkSizes = 5, 3 ; '
verifychunkline "$TESTLINE" "$BASELINE"
# Make sure that fvar was not chunked
checkfvar tmp_nc5.cdl
fi # T4
# Cleanup all created files
reset
echo "*** All nccopy tests passed!"
exit 0

132
ncdump/utils.c
View File

@ -45,7 +45,6 @@ emalloc ( /* check return from malloc */
return p;
}
void
check(int err, const char* file, const int line)
{
@ -142,6 +141,99 @@ print_name(const char* name) {
free(ename);
}
/* Convert a full path name to a group to the specific groupid. */
int
nc_inq_grpid2(int ncid, const char *grpname0, int *grpidp)
{
int ret = NC_NOERR;
char* grpname = NULL;
#ifdef USE_NETCDF4
char *sp = NULL;
#endif
grpname = strdup(grpname0);
if(grpname == NULL) {ret = NC_ENOMEM; goto done;}
#ifdef USE_NETCDF4
/* If '/' doesn't occur in name, just return id found by nc_inq_grpid() */
sp = strrchr(grpname, '/');
if(!sp) { /* No '/' in grpname, so return nc_inq_grpid() result */
ret = nc_inq_grp_ncid(ncid, grpname, grpidp);
goto done;
}
{ /* Parse group name out and get grpid using that */
char* p, *q;
int next;
p = grpname;
if(grpname[0] == '/') {
/* get ncid of the root group */
ncid = getrootid(ncid);
p++; /* skip leading '/' */
}
/* Walk down looking for each group in path in turn */
while(*p) {
q = strchr(p,'/');
if(q == NULL) q = p+strlen(p); /* point to trailing nul */
else *q++ = '\0';
/* Lookup this path segment wrt to current group */
if((ret=nc_inq_ncid(ncid,p,&next))) goto done;
/* move to next segment */
p = q;
ncid = next;
}
if(grpidp) *grpidp = ncid;
}
#else /* !USE_NETCDF4 */
/* Just return root */
if(grpidp) *grpidp = ncid;
#endif /* USE_NETCDF4 */
done:
if(grpname) free(grpname);
return ret;
}
/* Convert a full path name to a varid to the specific varid + grpid */
int
nc_inq_varid2(int ncid, const char *path0, int* varidp, int* grpidp)
{
int ret = NC_NOERR;
int grpid, varid;
char *v, *g, *prefix;
/* If '/' doesn't occur in name, just return id found by
* nc_inq_grpid()
*/
char* path = NULL;
path = strdup(path0);
if(path == NULL) {ret = NC_ENOMEM; goto done;}
/* Find the rightmost '/' and tag the start of the path */
g = strrchr(path,'/');
if(g == NULL) {
v = path;
prefix = "/"; /* make sure not free'd */
} else {
*g++ = '\0'; /* separate out the prefix */
prefix = path;
v = g;
}
/* convert the group prefix to a group id */
if((ret=nc_inq_grpid2(ncid,prefix,&grpid)))
goto done;
/* Lookup the var in the terminal group */
if((ret=nc_inq_varid(grpid,v,&varid)))
goto done;
if(grpidp)
*grpidp = grpid;
if(varidp)
*varidp = varid;
done:
if(path) free(path);
return ret;
}
/* Missing functionality that should be in nc_inq_dimid(), to get
* dimid from a full dimension path name that may include group
* names */
@ -172,7 +264,6 @@ nc_inq_dimid2(int ncid, const char *dimname, int *dimidp) {
return ret;
}
/*
* return 1 if varid identifies a record variable
* else return 0
@ -762,3 +853,40 @@ getrootid(int grpid)
return current;
}
#if 0
static int
parseFQN(int ncid, const char* fqn0, VarID* idp)
{
int stat = NC_NOERR;
char* fqn;
VarID vid;
char* p;
char* q;
char* segment;
vid.grpid = ncid;
if(fqn0 == NULL || fqn0[1] != '/')
{stat = NC_EBADNAME; goto done;}
fqn = strdup(fqn0+1); /* skip leading '/'*/
p = fqn;
for(;;) {
int newgrp;
segment = p;
q = p;
while(*p != '\0' && *p != '/') {
if(*p == '\\') p++;
*q++ = *p++;
}
if(*p == '\0') break;
*p++ = '\0';
if((stat=nc_inq_grp_ncid(vid.grpid,segment,&newgrp))) goto done;
vid.grpid = newgrp;
}
/* Segment should point to the varname */
if((stat=nc_inq_varid(vid.grpid,segment,&vid.varid))) goto done;
done:
if(fqn) free(fqn);
if(stat == NC_NOERR && idp != NULL) *idp = vid;
return stat;
}
#endif

6
ncdump/utils.h
View File

@ -85,6 +85,12 @@ void print_name(const char *name);
* names */
extern int nc_inq_dimid2(int ncid, const char *dimname, int *dimidp);
/* Convert a full path name to a group to the specific groupid. */
extern int nc_inq_grpid2(int ncid, const char *grpname0, int *grpidp);
/* Convert a full path name to a varid to the specific varid + grpid */
extern int nc_inq_varid2(int ncid, const char *path0, int* varidp, int* grpidp);
/* Test if variable is a record variable */
extern int isrecvar ( int ncid, int varid );