From c8f3a647b6538f287a3147bf1ba089e9125b5e1e Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 13 Jul 2018 16:32:43 -0500 Subject: [PATCH 1/7] add NC_RELAX_COORD_BOUND to netcdf_meta.h.in --- configure.ac | 4 +++- include/netcdf_meta.h.in | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2d2eff41a..f748d002d 100644 --- a/configure.ac +++ b/configure.ac @@ -1397,9 +1397,10 @@ AC_SUBST(HAS_PARALLEL4,[$enable_parallel4]) AC_SUBST(HAS_DISKLESS,[yes]) AC_SUBST(HAS_MMAP,[$enable_mmap]) AC_SUBST(HAS_JNA,[$enable_jna]) -AC_SUBST(RELAX_COORD_BOUND,[$enable_relax_coord_bound]) +AC_SUBST(RELAX_COORD_BOUND,[$enable_zero_length_coord_bound]) AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill]) + # Include some specifics for netcdf on windows. #AH_VERBATIM([_WIN32_STRICMP], AH_BOTTOM( @@ -1466,6 +1467,7 @@ AX_SET_META([NC_HAS_PARALLEL],[$enable_parallel],[yes]) AX_SET_META([NC_HAS_PARALLEL4],[$enable_parallel4],[yes]) AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes]) AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes]) +AX_SET_META([NC_RELAX_COORD_BOUND], [$enable_zero_length_coord_bound],[yes]) # Automake says that this is always run in top_builddir # and that srcdir is defined (== top_srcdir) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 54a2ff0e3..6ad98f1d8 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -53,5 +53,6 @@ #define NC_HAS_CDF5 @NC_HAS_CDF5@ /*!< CDF5 support. */ #define NC_HAS_ERANGE_FILL @NC_HAS_ERANGE_FILL@ /*!< ERANGE_FILL Support */ +#define NC_RELAX_COORD_BOUND @NC_RELAX_COORD_BOUND@ /*!< RELAX_COORD_BOUND */ #endif From d0cb241cfe0457a2b937ea6d7babe314bfcb2c60 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sat, 14 Jul 2018 18:03:42 -0500 Subject: [PATCH 2/7] add NC_HAS_PARALLEL4 to netcdf_meta.h.in --- include/netcdf_meta.h.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 54a2ff0e3..3f93b452a 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -48,8 +48,8 @@ #define NC_HAS_MMAP @NC_HAS_MMAP@ /*!< mmap support. */ #define NC_HAS_JNA @NC_HAS_JNA@ /*!< jna support. */ #define NC_HAS_PNETCDF @NC_HAS_PNETCDF@ /*!< pnetcdf support. */ -#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and -/or pnetcdf. */ +#define NC_HAS_PARALLEL4 @NC_HAS_PARALLEL4@ /*!< parallel IO support via hdf5 */ +#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and/or pnetcdf. */ #define NC_HAS_CDF5 @NC_HAS_CDF5@ /*!< CDF5 support. */ #define NC_HAS_ERANGE_FILL @NC_HAS_ERANGE_FILL@ /*!< ERANGE_FILL Support */ From 739dfa2aa2127cea842a90a75a69f68aa41c34a1 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sun, 15 Jul 2018 16:19:21 -0500 Subject: [PATCH 3/7] PnetCDF does not support per-variable collective/independent data mode change; modify nc_var_par_access to ignore varid --- docs/tutorial.dox | 6 +++- examples/C/parallel_vara.c | 4 +-- libsrcp/ncpdispatch.c | 71 ++++++++++++++++++++++++++++++-------- nc_test/tst_parallel2.c | 5 +++ nc_test/tst_pnetcdf.c | 6 ++-- nc_test/tst_small.c | 6 ++-- nc_test/util.c | 10 ++---- 7 files changed, 74 insertions(+), 34 deletions(-) diff --git a/docs/tutorial.dox b/docs/tutorial.dox index 640204b4f..3d3287353 100644 --- a/docs/tutorial.dox +++ b/docs/tutorial.dox @@ -809,7 +809,11 @@ create/open a netCDF file with parallel access. Parallel file access is either collective (all processors must participate) or independent (any processor may access the data without -waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access(). +waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. +To change from collective to independent mode or vis versa, call +nc_var_par_access() with argument 'access' set to either NC_INDEPENDENT or +NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid' is ignored, as +PnetCDF does not support per-variable collective/independent mode change. \page tutorial_ncids Numbering of NetCDF IDs diff --git a/examples/C/parallel_vara.c b/examples/C/parallel_vara.c index 4aa789c81..757d91769 100644 --- a/examples/C/parallel_vara.c +++ b/examples/C/parallel_vara.c @@ -150,7 +150,7 @@ int main(int argc, char** argv) err = nc_enddef(ncid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* now we are in data mode */ start[0] = 0; @@ -176,7 +176,7 @@ int main(int argc, char** argv) err = nc_inq_varid(ncid, "var", &varid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* each process reads its subarray from the file */ err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index d651abc4f..9e561057c 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -9,10 +9,14 @@ #include #include "nc.h" #include "ncdispatch.h" +#include "fbits.h" /* Must follow netcdf.h */ #include +#define NCP_MODE_DATA 0x0001 +#define NCP_MODE_INDEP 0x0002 + typedef struct NCP_INFO { /* pnetcdf_file will be true if the file is created/opened with the @@ -104,6 +108,9 @@ NCP_create(const char *path, int cmode, res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); + /* Default to independent access, like netCDF-4/HDF5 files. */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ done: return res; @@ -152,6 +159,9 @@ NCP_open(const char *path, int cmode, nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); if(nc5 == NULL) {res = NC_ENOMEM; goto done;} + /* file open automatically enters data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); @@ -160,7 +170,7 @@ NCP_open(const char *path, int cmode, /* Default to independent access, like netCDF-4/HDF5 files. */ if(!res) { res = ncmpi_begin_indep_data(nc->int_ncid); - nc5->pnetcdf_access_mode = NC_INDEPENDENT; + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); } done: return res; @@ -170,8 +180,14 @@ static int NCP_redef(int ncid) { NC* nc; + NCP_INFO* nc5; + int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; + + nc5 = NCP_DATA(nc); + fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + return ncmpi_redef(nc->int_ncid); } @@ -203,7 +219,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t #endif if(!status) { - if (nc5->pnetcdf_access_mode == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) status = ncmpi_begin_indep_data(nc->int_ncid); } return status; @@ -611,7 +628,7 @@ NCP_get_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -705,7 +722,7 @@ NCP_put_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -801,7 +818,7 @@ NCP_get_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -897,7 +914,7 @@ NCP_put_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -995,7 +1012,7 @@ NCP_get_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1093,7 +1110,7 @@ NCP_put_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1198,19 +1215,43 @@ NCP_var_par_access(int ncid, int varid, int par_access) if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE) return NC_EINVAL; +#ifdef _DO_NOT_IGNORE_VARID_ + if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */ + return NC_EINVAL; +#endif + status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; nc5 = NCP_DATA(nc); assert(nc5); - if(par_access == nc5->pnetcdf_access_mode) - return NC_NOERR; - nc5->pnetcdf_access_mode = par_access; - if (par_access == NC_INDEPENDENT) - return ncmpi_begin_indep_data(nc->int_ncid); - else - return ncmpi_end_indep_data(nc->int_ncid); + /* if currently in data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) { + if (par_access == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) + return NC_NOERR; + else { /* currently in collective data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_begin_indep_data(nc->int_ncid); + } + } + else { /* want to enter collective data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_end_indep_data(nc->int_ncid); + } + else + return NC_NOERR; + } + } + else { /* currently in define mode */ + if (par_access == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + else + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + } + return NC_NOERR; } #ifdef USE_NETCDF4 diff --git a/nc_test/tst_parallel2.c b/nc_test/tst_parallel2.c index 92c271d26..45f61a4bc 100644 --- a/nc_test/tst_parallel2.c +++ b/nc_test/tst_parallel2.c @@ -135,8 +135,13 @@ main(int argc, char **argv) sleep(mpi_rank); #endif /* USE_MPE */ +#ifdef USE_PNETCDF +/* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/ + if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR; +#else /* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/ if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR; +#endif if (!mpi_rank) start_time = MPI_Wtime(); diff --git a/nc_test/tst_pnetcdf.c b/nc_test/tst_pnetcdf.c index 812ab5dfd..bcf8d21d5 100644 --- a/nc_test/tst_pnetcdf.c +++ b/nc_test/tst_pnetcdf.c @@ -68,10 +68,8 @@ int main(int argc, char* argv[]) } if (nc_enddef(ncid)) ERR; - for (i=0; i Date: Thu, 26 Jul 2018 20:16:02 -0600 Subject: [PATCH 4/7] This PR primarily addresses Issue https://github.com/Unidata/netcdf-c/issues/725. After a long discussion, I implemented the rules at the end of that issue. They are documented in nccopy.1. Additionally, I added a new, per-variable, -c flag that allows for the direct setting of the chunking parameters for a variable. The form is -c var:c1,c2,...ck where var is the name of the variable (possibly a fully qualified name) and the ci are the chunksizes for that variable. It must be the case that the rank of the variable is k. If the new form is used as well as the old form, then the new form overrides the old form for the specified variable. Note that multiple occurrences of the new form -c flag may be specified. Misc. Other fixes 1. Added -M option to nccopy to specify the minimum allowable chunksize. 2. Removed the unused variables from bigmeta.c (Issue https://github.com/Unidata/netcdf-c/issues/1079) 3. Fixed failure of nc_test4/tst_filter.sh by using the new -M flag (#1) to allow filter test on a small chunk size. --- cf | 4 +- cf.cmake | 2 +- libdap4/d4data.c | 3 +- nc_test4/Make0 | 8 +- nc_test4/bigmeta.c | 5 +- nc_test4/findplugin.in | 6 + nc_test4/tst_filter.sh | 3 +- ncdump/CMakeLists.txt | 7 +- ncdump/Make0 | 8 +- ncdump/Makefile.am | 10 +- ncdump/chunkspec.c | 249 ++++++++++++++++--- ncdump/chunkspec.h | 17 +- ncdump/list.c | 221 +++++++++++++++++ ncdump/list.h | 62 +++++ ncdump/nccopy.1 | 35 +++ ncdump/nccopy.c | 533 ++++++++++++++++++++++++----------------- ncdump/tst_chunking.c | 130 ++++++++-- ncdump/tst_nccopy5.sh | 167 +++++++++++++ ncdump/utils.c | 132 +++++++++- ncdump/utils.h | 6 + 20 files changed, 1310 insertions(+), 298 deletions(-) create mode 100644 ncdump/list.c create mode 100644 ncdump/list.h create mode 100755 ncdump/tst_nccopy5.sh diff --git a/cf b/cf index a2a3d7c9d..1d123b52d 100644 --- a/cf +++ b/cf @@ -1,6 +1,6 @@ #!/bin/bash #NB=1 -DB=1 +#DB=1 #X=-x #FAST=1 @@ -118,7 +118,7 @@ FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-jna" #FLAGS="$FLAGS --disable-properties-attribute" #FLAGS="$FLAGS --disable-silent-rules" -FLAGS="$FLAGS --disable-filter-testing" +#FLAGS="$FLAGS --disable-filter-testing" #FLAGS="$FLAGS --enable-metadata-perf" if test "x$TESTSERVERS" != x ; then diff --git a/cf.cmake b/cf.cmake index 9c257a248..1dd4ad581 100644 --- a/cf.cmake +++ b/cf.cmake @@ -60,7 +60,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true" FLAGS="$FLAGS -DENABLE_LOGGING=true" #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true" #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true" -#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" +FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" # Disables FLAGS="$FLAGS -DENABLE_EXAMPLES=false" diff --git a/libdap4/d4data.c b/libdap4/d4data.c index a60210be0..c448e865c 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -99,8 +99,7 @@ NCD4_processdata(NCD4meta* meta) for(i=0;idata.localchecksum != var->data.remotechecksum) { - fprintf(stderr,"Checksum mismatch: %s\n",var->name); - fflush(stderr); + nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } diff --git a/nc_test4/Make0 b/nc_test4/Make0 index df06ce208..22f7a0a75 100644 --- a/nc_test4/Make0 +++ b/nc_test4/Make0 @@ -1,5 +1,5 @@ # Test c output -T=tst_varsperf +T=bigmeta #SRC=hdf5plugins/H5Zmisc.c @@ -12,7 +12,8 @@ T=tst_varsperf #PAR=1 #SZIP=1 -CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include +#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include +CFLAGS = -Wall -g -O0 -I.. -I../include LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf @@ -31,7 +32,8 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH} all:: cmp export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CMD} ./t ${ARGS} + +# ${CMD} ./t ${ARGS} cmp:: export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ diff --git a/nc_test4/bigmeta.c b/nc_test4/bigmeta.c index 20e8fcfcb..fb2054eb2 100644 --- a/nc_test4/bigmeta.c +++ b/nc_test4/bigmeta.c @@ -134,7 +134,6 @@ static void buildatts(int grpid, int varid) { char name[NC_MAX_NAME+1]; - int attid; int i, count; count = (varid == NC_GLOBAL? ngroupattrs : nvarattrs); @@ -149,7 +148,7 @@ static void buildgroup(int parent, int grpindex, int depth) { char name[NC_MAX_NAME+1]; - int i, grpid, dimid, typid, varid, attid; + int i, grpid, varid; int dimids[NDIMS]; if(depth == 0) return; @@ -193,9 +192,7 @@ buildgroup(int parent, int grpindex, int depth) int main(int argc, char **argv) { - int stat = NC_NOERR; int i, ncid; - int grpid, dimid, varid, typid, attid; time_t starttime, endtime; long long delta; int tag; diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index a7af4d7de..06568234e 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -99,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH" return 0 } + +# debug +if test "x$1" != x ; then +findplugin +echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|" +fi diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index f695a8427..6e2201dc9 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -110,7 +110,8 @@ echo " *** Pass: nccopy simple filter" echo " *** Testing pass-thru of filters" rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc -${NCCOPY} ./filtered.nc ./tst_filter2.nc +# Prevent failure by allowing any chunk size +${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc ${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt test -s tst_filter2.txt diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 190d4261e..67b6e67d0 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -4,7 +4,7 @@ IF(BUILD_SHARED_LIBS AND WIN32) ENDIF() SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c) -SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c) +SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) @@ -211,6 +211,11 @@ ENDIF(MSVC) SET_TESTS_PROPERTIES(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE) ENDIF(HAVE_BASH) + add_sh_test(ncdump tst_nccopy5) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE) + ENDIF(HAVE_BASH) + IF(USE_NETCDF4) add_sh_test(ncdump tst_ncgen4) ENDIF(USE_NETCDF4) diff --git a/ncdump/Make0 b/ncdump/Make0 index c3a594b69..1d02f46ce 100644 --- a/ncdump/Make0 +++ b/ncdump/Make0 @@ -1,12 +1,12 @@ # Test c output -T=tst_unicode +T=tst_chunking #ARGS=./x.nc #TF=test_atomic_array.cdl #CMD=valgrind --leak-check=full -CMD=gdb --args +#CMD=gdb --args #HDF4=1 #PAR=1 @@ -36,11 +36,11 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH} all:: comp export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CMD} ./t ${ARGS} + ${CMD} ./$T ${ARGS} comp:: export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \ + ${CC} -o $T ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \ x.nc: x.cdl ncgen -4 x.cdl diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 0629c2087..cf3b4abdf 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -26,7 +26,7 @@ utils.c nciter.h nciter.c nccomps.h # netCDF API bin_PROGRAMS += nccopy nccopy_SOURCES = nccopy.c nciter.c nciter.h chunkspec.h chunkspec.c \ -utils.h utils.c dimmap.h dimmap.c +utils.h utils.c dimmap.h dimmap.c list.c list.h # A simple netcdf-4 metadata -> xml printer. Do not install. if USE_NETCDF4 @@ -80,14 +80,14 @@ tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar # Tests for netCDF-4 behavior. TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \ tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \ -tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \ +tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \ run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \ tst_ncgen4.sh -# The tst_nccopy4.sh test script depends on the output of a bunch of -# other tests. Record dependencies so parallel builds work. +# Record interscript dependencies so parallel builds work. tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log \ tst_fillbug.log tst_netcdf4_4.log tst_h_scalar.log +tst_nccopy5.log: tst_nccopy4.log endif #!USE_NETCDF4 TESTS += tst_inmemory_nc3.sh @@ -111,7 +111,7 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \ ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \ ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \ tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \ -tst_nccopy3.sh tst_nccopy4.sh ref_nc_test_netcdf4_4_0.nc \ +tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \ run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \ ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \ ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \ diff --git a/ncdump/chunkspec.c b/ncdump/chunkspec.c index c92dc48fb..62dd0506d 100644 --- a/ncdump/chunkspec.c +++ b/ncdump/chunkspec.c @@ -7,38 +7,80 @@ #include #include #include -#include +#include "netcdf.h" +#include "list.h" #include "utils.h" #include "chunkspec.h" /* Structure mapping dimension IDs to corresponding chunksizes. */ -static struct { +static struct DimChunkSpecs { size_t ndims; /* number of dimensions in chunkspec string */ - int *dimids; /* ids for dimensions in chunkspec string */ + int *idimids; /* (input) ids for dimensions in chunkspec string */ size_t *chunksizes; /* corresponding chunk sizes */ bool_t omit; /* true if chunking to be turned off */ -} chunkspecs; +} dimchunkspecs; + +struct VarChunkSpec { + size_t rank; /* number of dimensions in chunkspec string */ + size_t chunksizes[NC_MAX_VAR_DIMS]; /* corresponding chunk sizes */ + bool_t omit; /* true if chunking to be turned off */ + int igrpid; /* container of the (input) variable */ + int ivarid; /* (input) Variable whose chunks are specified */ +}; + +static List* varchunkspecs = NULL; /* List */ + +/* Forward */ +static int dimchunkspec_parse(int ncid, const char *spec); +static int varchunkspec_parse(int ncid, const char *spec); + +void +chunkspecinit(void) +{ + /* initialization */ + if(varchunkspecs == NULL) + varchunkspecs = listnew(); + memset(&dimchunkspecs,0,sizeof(dimchunkspecs)); +} /* - * Parse chunkspec string and convert into chunkspec_t structure. + * Parse chunkspec string of either kind. + * Returns NC_NOERR if no error, NC_EINVAL if spec was malformed. + */ +int +chunkspec_parse(int igrp, const char *spec) +{ + /* Decide if this is a per-variable or per-dimension chunkspec */ + if (!spec || *spec == '\0') + return NC_NOERR; /* Use defaults */ + if(strchr(spec,':') == NULL) + return dimchunkspec_parse(igrp,spec); + else + return varchunkspec_parse(igrp,spec); +} + +/* + * Parse chunkspec string and convert into dimchunkspec structure. * ncid: location ID of open netCDF file or group in an open file * spec: string of form * dim1/n1,dim2/n2,...,dimk/nk - * - * specifying chunk size (ni) to be used for dimension named - * dimi. Dimension names may be absolute, - * e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be - * omitted, in which case it is assumed to be the entire - * dimension size. That is also the default for dimensions - * not mentioned in the string. - * If the chunkspec string is "/", specifying no dimensions or - * chunk sizes, it indicates chunking to be turned off on output. + * specifying chunk size (ni) to be used for dimension named + * dimi. Dimension names may be absolute, + * e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be + * omitted, in which case it is assumed to be the entire + * dimension size. That is also the default for dimensions + * not mentioned in the string. However, for unlimited dimensions, + * the default is a default size: 4 megabytes or the + * existing unlimited size if smaller. + * If the chunkspec string is "/", specifying no dimensions or + * chunk sizes, it indicates chunking to be turned off on output. * * Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive * unescaped commas or no chunksize specified for dimension. */ -int -chunkspec_parse(int ncid, const char *spec) { +static int +dimchunkspec_parse(int igrp, const char *spec) +{ const char *cp; /* character cursor */ const char *pp = spec; /* previous char cursor for detecting escapes */ const char *np; /* beginning of current dimension name */ @@ -47,12 +89,12 @@ chunkspec_parse(int ncid, const char *spec) { int ret; int comma_seen = 0; - chunkspecs.ndims = 0; - chunkspecs.omit = false; + dimchunkspecs.ndims = 0; + dimchunkspecs.omit = false; if (!spec || *spec == '\0') /* default chunking */ return NC_NOERR; if (spec[0] == '/' && spec[1] == '\0') { /* no chunking */ - chunkspecs.omit = true; + dimchunkspecs.omit = true; return NC_NOERR; } /* Count unescaped commas, handle consecutive unescaped commas as error */ @@ -69,9 +111,9 @@ chunkspec_parse(int ncid, const char *spec) { pp = cp; } ndims++; - chunkspecs.ndims = ndims; - chunkspecs.dimids = (int *) emalloc(ndims * sizeof(int)); - chunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t)); + dimchunkspecs.ndims = ndims; + dimchunkspecs.idimids = (int *) emalloc(ndims * sizeof(int)); + dimchunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t)); /* Look up dimension ids and assign chunksizes */ pp = spec; np = spec; @@ -97,15 +139,15 @@ chunkspec_parse(int ncid, const char *spec) { } *dp = '\0'; /* look up dimension id from dimension pathname */ - ret = nc_inq_dimid2(ncid, dimname, &dimid); + ret = nc_inq_dimid2(igrp, dimname, &dimid); if(ret != NC_NOERR) break; - chunkspecs.dimids[idim] = dimid; + dimchunkspecs.idimids[idim] = dimid; /* parse and assign corresponding chunksize */ pp++; /* now points to first digit of chunksize, ',', or '\0' */ if(*pp == ',' || *pp == '\0') { /* no size specified, use dim len */ size_t dimlen; - ret = nc_inq_dimlen(ncid, dimid, &dimlen); + ret = nc_inq_dimlen(igrp, dimid, &dimlen); if(ret != NC_NOERR) return(ret); chunksize = dimlen; @@ -120,7 +162,7 @@ chunkspec_parse(int ncid, const char *spec) { return (NC_EINVAL); chunksize = (size_t)val; } - chunkspecs.chunksizes[idim] = chunksize; + dimchunkspecs.chunksizes[idim] = chunksize; idim++; free(dimname); if(*cp == '\0') @@ -135,11 +177,11 @@ chunkspec_parse(int ncid, const char *spec) { /* Return size in chunkspec string specified for dimension corresponding to dimid, 0 if not found */ size_t -chunkspec_size(int dimid) { +dimchunkspec_size(int indimid) { int idim; - for(idim = 0; idim < chunkspecs.ndims; idim++) { - if(dimid == chunkspecs.dimids[idim]) { - return chunkspecs.chunksizes[idim]; + for(idim = 0; idim < dimchunkspecs.ndims; idim++) { + if(indimid == dimchunkspecs.idimids[idim]) { + return dimchunkspecs.chunksizes[idim]; } } return 0; @@ -149,15 +191,154 @@ chunkspec_size(int dimid) { * chunkspec string on command line, 0 if no chunkspec string was * specified. */ int -chunkspec_ndims(void) { - return chunkspecs.ndims; +dimchunkspec_ndims(void) { + return dimchunkspecs.ndims; } /* Return whether chunking should be omitted, due to explicit * command-line specification. */ bool_t -chunkspec_omit(void) { - return chunkspecs.omit; +dimchunkspec_omit(void) { + return dimchunkspecs.omit; +} + + +/* + * Parse per-variable chunkspec string and convert into varchunkspec structure. + * ncid: location ID of open netCDF file or group in an open file + * spec: string of form + * var:n1,n2,...nk + * + * specifying chunk size (ni) to be used for ith dimension of + * variable named var. Variable names may be absolute. + * e.g. "/grp_a/grp_a1/var". + * If no chunk sizes are specified, then the variable is not chunked at all. + * + * Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive + * unescaped commas or no chunksize specified for dimension. + */ +static int +varchunkspec_parse(int igrp, const char *spec0) +{ + int ret = NC_NOERR; + int rank; + int i; + int dimids[NC_MAX_VAR_DIMS]; + struct VarChunkSpec* chunkspec = NULL; + char* spec = NULL; + char* p, *q; /* for walking strings */ + + /* Copy spec so we can modify in place */ + spec = strdup(spec0); + if(spec == NULL) {ret = NC_ENOMEM; goto done;} + + chunkspec = calloc(1,sizeof(struct VarChunkSpec)); + if(chunkspec == NULL) {ret = NC_ENOMEM; goto done;} + + chunkspec->igrpid = igrp; + + /* First, find the end of the variable part */ + p = strchr(spec,':'); + if(p == NULL) + {ret = NC_EINVAL; goto done;} + *p++ = '\0'; + /* Lookup the variable by name */ + ret = nc_inq_varid2(igrp, spec, &chunkspec->ivarid, &chunkspec->igrpid); + if(ret != NC_NOERR) goto done; + + /* Iterate over dimension sizes */ + while(*p) { + unsigned long dimsize; + q = strchr(p,','); + if(q == NULL) + q = p + strlen(p); /* Fake the endpoint */ + else + *q++ = '\0'; + + /* Scan as unsigned long */ + if(sscanf(p,"%lu",&dimsize) != 1) + {ret = NC_EINVAL; goto done;} /* Apparently not a valid dimension size */ + if(chunkspec->rank >= NC_MAX_VAR_DIMS) {ret = NC_EINVAL; goto done;} /* to many chunks */ + chunkspec->chunksizes[chunkspec->rank] = (size_t)dimsize; + chunkspec->rank++; + p = q; + } + /* Now do some validity checking */ + /* Get some info about the var (from input) */ + ret = nc_inq_var(chunkspec->igrpid,chunkspec->ivarid,NULL,NULL,&rank,dimids,NULL); + if(ret != NC_NOERR) goto done; + + /* 1. check # chunksizes == rank of variable */ + if(rank != chunkspec->rank) {ret = NC_EINVAL; goto done;} + + /* 2. check that chunksizes are legal for the given dimension sizes */ + for(i=0;ichunksizes[i] > len) {ret = NC_EBADCHUNK; goto done;} + } + + /* add the chunkspec to our list */ + listpush(varchunkspecs,chunkspec); + chunkspec = NULL; + +done: + if(chunkspec != NULL) + free(chunkspec); + if(spec != NULL) + free(spec); + return ret; +} + +/* Accessors */ + +bool_t +varchunkspec_exists(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return true; + } + return false; +} + +bool_t +varchunkspec_omit(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->omit; + } + return dimchunkspecs.omit; +} + +size_t* +varchunkspec_chunksizes(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->chunksizes; + } + return NULL; +} + +size_t +varchunkspec_rank(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->rank; + } + return 0; } diff --git a/ncdump/chunkspec.h b/ncdump/chunkspec.h index e0885dc33..5c7d68d18 100644 --- a/ncdump/chunkspec.h +++ b/ncdump/chunkspec.h @@ -15,17 +15,28 @@ chunkspec_parse(int ncid, const char *spec); /* Return chunk size in chunkspec string specified for dimension * corresponding to dimid, 0 if not found */ extern size_t -chunkspec_size(int dimid); +dimchunkspec_size(int dimid); /* Return number of dimensions for which chunking was specified in * chunkspec string on command line, 0 if no chunkspec string was * specified. */ extern int -chunkspec_ndims(void); +dimchunkspec_ndims(void); /* Return whether chunking should be omitted, due to explicit * command-line specification. */ extern bool_t -chunkspec_omit(void); +dimchunkspec_omit(void); + +extern bool_t varchunkspec_omit(int grpid, int varid); + +extern size_t* varchunkspec_chunksizes(int grpid, int varid); + +extern size_t varchunkspec_ndims(int grpid, int varid); + +extern bool_t varchunkspec_exists(int grpid, int varid); + +extern void chunkspecinit(void); + #endif /* _CHUNKSPEC_H_ */ diff --git a/ncdump/list.c b/ncdump/list.c new file mode 100644 index 000000000..30570a43c --- /dev/null +++ b/ncdump/list.c @@ -0,0 +1,221 @@ +/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ +#include +#include +#include + +#include "list.h" + +int listnull(void* e) {return e == NULL;} + +#ifndef TRUE +#define TRUE 1 +#endif +#ifndef FALSE +#define FALSE 0 +#endif + +#define DEFAULTALLOC 16 +#define ALLOCINCR 16 + +List* listnew(void) +{ + List* l; +/* + if(!initialized) { + memset((void*)&DATANULL,0,sizeof(void*)); + initialized = 1; + } +*/ + l = (List*)malloc(sizeof(List)); + if(l) { + l->alloc=0; + l->length=0; + l->content=NULL; + } + return l; +} + +int +listfree(List* l) +{ + if(l) { + l->alloc = 0; + if(l->content != NULL) {free(l->content); l->content = NULL;} + free(l); + } + return TRUE; +} + +int +listsetalloc(List* l, unsigned long sz) +{ + void** newcontent = NULL; + if(l == NULL) return FALSE; + if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);} + if(l->alloc >= sz) {return TRUE;} + newcontent=(void**)calloc(sz,sizeof(void*)); + if(newcontent != NULL && l->alloc > 0 && l->length > 0 && l->content != NULL) { + memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length); + } + if(l->content != NULL) free(l->content); + l->content=newcontent; + l->alloc=sz; + return TRUE; +} + +int +listsetlength(List* l, unsigned long sz) +{ + if(l == NULL) return FALSE; + if(sz > l->alloc && !listsetalloc(l,sz)) return FALSE; + l->length = sz; + return TRUE; +} + +void* +listget(List* l, unsigned long index) +{ + if(l == NULL || l->length == 0) return NULL; + if(index >= l->length) return NULL; + return l->content[index]; +} + +int +listset(List* l, unsigned long index, void* elem) +{ + if(l == NULL) return FALSE; + if(index >= l->length) return FALSE; + l->content[index] = elem; + return TRUE; +} + +/* Insert at position i of l; will push up elements i..|seq|. */ +int +listinsert(List* l, unsigned long index, void* elem) +{ + int i; /* do not make unsigned */ + if(l == NULL) return FALSE; + if(index > l->length) return FALSE; + listsetalloc(l,0); + for(i=(int)l->length;i>index;i--) l->content[i] = l->content[i-1]; + l->content[index] = elem; + l->length++; + return TRUE; +} + +int +listpush(List* l, void* elem) +{ + if(l == NULL) return FALSE; + if(l->length >= l->alloc) listsetalloc(l,0); + l->content[l->length] = elem; + l->length++; + return TRUE; +} + +void* +listpop(List* l) +{ + if(l == NULL || l->length == 0) return NULL; + l->length--; + return l->content[l->length]; +} + +void* +listtop(List* l) +{ + if(l == NULL || l->length == 0) return NULL; + return l->content[l->length - 1]; +} + +void* +listremove(List* l, unsigned long i) +{ + unsigned long len; + void* elem; + if(l == NULL || (len=l->length) == 0) return NULL; + if(i >= len) return NULL; + elem = l->content[i]; + for(i+=1;icontent[i-1] = l->content[i]; + l->length--; + return elem; +} + +/* Duplicate and return the content (null terminate) */ +void** +listdup(List* l) +{ + void** result = (void**)malloc(sizeof(void*)*(l->length+1)); + memcpy((void*)result,(void*)l->content,sizeof(void*)*l->length); + result[l->length] = (void*)0; + return result; +} + +int +listcontains(List* l, void* elem) +{ + unsigned long i; + for(i=0;ilength) == 0) return 0; + for(i=0;icontent[i]; + if(elem == candidate) { + for(i+=1;icontent[i-1] = l->content[i]; + l->length--; + found = 1; + break; + } + } + return found; +} + + + + +/* Extends list to include a unique operator + which remove duplicate values; NULL values removed + return value is always 1. +*/ + +int +listunique(List* l) +{ + unsigned long i,j,k,len; + void** content; + if(l == NULL || l->length == 0) return 1; + len = l->length; + content = l->content; + for(i=0;ilength = len; + return 1; +} + +List* +listclone(List* l) +{ + List* clone = listnew(); + *clone = *l; + clone->content = listdup(l); + return clone; +} diff --git a/ncdump/list.h b/ncdump/list.h new file mode 100644 index 000000000..2206af37d --- /dev/null +++ b/ncdump/list.h @@ -0,0 +1,62 @@ +/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ +#ifndef LIST_H +#define LIST_H 1 + +/* Define the type of the elements in the list*/ + +#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) +#define EXTERNC extern "C" +#else +#define EXTERNC extern +#endif + +EXTERNC int listnull(void*); + +typedef struct List { + unsigned long alloc; + unsigned long length; + void** content; +} List; + +EXTERNC List* listnew(void); +EXTERNC int listfree(List*); +EXTERNC int listsetalloc(List*,unsigned long); +EXTERNC int listsetlength(List*,unsigned long); + +/* Set the ith element */ +EXTERNC int listset(List*,unsigned long,void*); +/* Get value at position i */ +EXTERNC void* listget(List*,unsigned long);/* Return the ith element of l */ +/* Insert at position i; will push up elements i..|seq|. */ +EXTERNC int listinsert(List*,unsigned long,void*); +/* Remove element at position i; will move higher elements down */ +EXTERNC void* listremove(List* l, unsigned long i); + +/* Tail operations */ +EXTERNC int listpush(List*,void*); /* Add at Tail */ +EXTERNC void* listpop(List*); +EXTERNC void* listtop(List*); + +/* Duplicate and return the content (null terminate) */ +EXTERNC void** listdup(List*); + +/* Look for value match */ +EXTERNC int listcontains(List*, void*); + +/* Remove element by value; only removes first encountered */ +EXTERNC int listelemremove(List* l, void* elem); + +/* remove duplicates */ +EXTERNC int listunique(List*); + +/* Create a clone of a list */ +EXTERNC List* listclone(List*); + +/* Following are always "in-lined"*/ +#define listclear(l) listsetlength((l),0) +#define listextend(l,len) listsetalloc((l),(len)+(l->alloc)) +#define listcontents(l) ((l)==NULL?NULL:(l)->content) +#define listlength(l) ((l)==NULL?0:(int)(l)->length) + +#endif /*LIST_H*/ diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 783809319..1eda79971 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -352,6 +352,41 @@ memory before writing it to disk on close: .HP nccopy \-w \-c time/1000,lat/40,lon/40 slow.nc fast.nc .RE + +.SH "Chunking Rules" +.LP +The complete set of chunking rules is captured here. As a rough +summary, these rules preserve all chunking properties from the +input file. These rules apply only when the selected output +format supports chunking, i.e. for the netcdf-4 variants. +.LP +The following rules are applied in the given order independently +for each variable to be copied from input to output. The rules are +written assuming we are trying to determine the chunking for a given +output variable Vout that comes from an input variable Vin. +.NP +For each dimension of Vout explicitly specified on the command line +using the '-c' option, apply the chunking value for that +dimension. regardless of input format or input properties. + +.NP +For dimensions of V not named on the command line, preserve chunk +sizes from the corresponding input variable. +.NP +If V is netcdf-4 and contiguous, and none of its dimensions are +named on the command line, and chunking is not mandated by other +options, then make V be contiguous. +.NP +If the input variable is contiguous (or is some netcdf-3 +variant) and there are no options requiring chunking, or the '/' +special case for the '-c' option is specified, then the output +variable V is marked as contiguous. +.NP +Handle all remaining cases when some or all chunk sizes are not determined by the command line or the input variable. This includes the non-chunked input cases such as netcdf-3, cdf5, and DAP. In these cases: + Retain all chunk sizes determined by (1) and (2); and + Compute the remaining chunk sizes automatically, with some reasonable + + .SH "SEE ALSO" .LP .BR ncdump(1), ncgen(1), netcdf(3) diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 5a8901ec8..914da437b 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -23,6 +23,7 @@ #include "dimmap.h" #include "nccomps.h" #include "ncfilter.h" +#include "list.h" #undef DEBUGFILTER @@ -48,6 +49,8 @@ int optind; #define ESCAPESD "0123456789" #define ESCAPES " !\"#$%&'()*,:;<=>?[]\\^`{|}~" +#define DFALTUNLIMSIZE (4* 1<<20) /*4 megabytes */ + #ifdef USE_NETCDF4 /* The unique id for a variable requires also the enclosing group id */ @@ -73,20 +76,65 @@ static int suppressfilters = 0; /* 1 => do not apply any output filters unless s #endif +/* table of formats for legal -k values */ +static struct Kvalues { + char* name; + int kind; +} legalkinds[] = { + /* NetCDF-3 classic format (32-bit offsets) */ + {"classic", NC_FORMAT_CLASSIC}, /* canonical format name */ + {"nc3", NC_FORMAT_CLASSIC}, /* short format name */ + {"1", NC_FORMAT_CLASSIC}, /* deprecated, use "-3" or "-k nc3" instead */ + + /* NetCDF-3 64-bit offset format */ + {"64-bit offset", NC_FORMAT_64BIT_OFFSET}, /* canonical format name */ + {"nc6", NC_FORMAT_64BIT_OFFSET}, /* short format name */ + {"2", NC_FORMAT_64BIT_OFFSET}, /* deprecated, use "-6" or "-k nc6" instead */ + {"64-bit-offset", NC_FORMAT_64BIT_OFFSET}, /* deprecated alias */ + + /* NetCDF-4 HDF5-based format */ + {"netCDF-4", NC_FORMAT_NETCDF4}, /* canonical format name */ + {"nc4", NC_FORMAT_NETCDF4}, /* short format name */ + {"3", NC_FORMAT_NETCDF4}, /* deprecated, use "-4" or "-k nc4" instead */ + {"netCDF4", NC_FORMAT_NETCDF4}, /* deprecated aliases */ + {"hdf5", NC_FORMAT_NETCDF4}, + {"enhanced", NC_FORMAT_NETCDF4}, + + /* NetCDF-4 HDF5-based format, restricted to classic data model */ + {"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC}, /* canonical format name */ + {"nc7", NC_FORMAT_NETCDF4_CLASSIC}, /* short format name */ + {"4", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated, use "-7" or -k nc7" */ + {"netCDF-4-classic", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated aliases */ + {"netCDF-4_classic", NC_FORMAT_NETCDF4_CLASSIC}, + {"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC}, + {"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC}, + {"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC}, + + /* The 64-bit data (CDF5) kind (5) */ + {"5", NC_FORMAT_CDF5}, + {"64-bit-data", NC_FORMAT_CDF5}, + {"64-bit data", NC_FORMAT_CDF5}, + {"nc5", NC_FORMAT_CDF5}, + {"cdf5", NC_FORMAT_CDF5}, + + /* null terminate*/ + {NULL,0} +}; + /* Global variables for command-line requests */ char *progname; /* for error messages */ static int option_kind = SAME_AS_INPUT; static int option_deflate_level = -1; /* default, compress output only if input compressed */ static int option_shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */ static int option_fix_unlimdims = 0; /* default, preserve unlimited dimensions */ -static char* option_chunkspec = 0; /* default, no chunk specification */ +static List* option_chunkspecs = NULL; /* default, no chunk specification */ static size_t option_copy_buffer_size = COPY_BUFFER_SIZE; static size_t option_chunk_cache_size = CHUNK_CACHE_SIZE; /* default from config.h */ static size_t option_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /* default from config.h */ static int option_read_diskless = 0; /* default, don't read input into memory on open */ static int option_write_diskless = 0; /* default, don't write output to diskless file */ #ifdef USE_NETCDF4 -static int option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of +static size_t option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of * chunksizes of its dimensions is smaller * than this */ #endif @@ -97,9 +145,9 @@ static char** option_lgrps = 0; /* list of group names specified with -g static idnode_t* option_grpids = 0; /* list of grpids matching list specified with -g option */ static bool_t option_grpstruct = false; /* if -g set, copy structure for non-selected groups */ static int option_nlvars = 0; /* Number of variables specified with -v * option on command line */ -static char** option_lvars = 0; /* list of variable names specified with -v - * option on command line */ -static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */ +static char** option_lvars = 0; /* list of variable names specified with -v + * option on command line */ +static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */ static int option_compute_chunkcaches = 0; /* default, don't try still flaky estimate of * chunk cache for each variable */ @@ -186,7 +234,7 @@ computeFQN(VarID vid, char** fqnp) q = escname; for(first=1;*p;first=0) { if((first && strchr(ESCAPESD,*p) != NULL) - || strchr(ESCAPES,*p) != NULL) *q++ = '\\'; + || strchr(ESCAPES,*p) != NULL) *q++ = '\\'; *q++ = *p++; } *q++ = '\0'; /* guarantee */ @@ -196,44 +244,6 @@ done: return stat; } -#if 0 -static int -parseFQN(int ncid, const char* fqn0, VarID* idp) -{ - int stat = NC_NOERR; - char* fqn; - VarID vid; - char* p; - char* q; - char* segment; - - vid.grpid = ncid; - if(fqn0 == NULL || fqn0[1] != '/') - {stat = NC_EBADNAME; goto done;} - fqn = strdup(fqn0+1); /* skip leading '/'*/ - p = fqn; - for(;;) { - int newgrp; - segment = p; - q = p; - while(*p != '\0' && *p != '/') { - if(*p == '\\') p++; - *q++ = *p++; - } - if(*p == '\0') break; - *p++ = '\0'; - if((stat=nc_inq_grp_ncid(vid.grpid,segment,&newgrp))) goto done; - vid.grpid = newgrp; - } - /* Segment should point to the varname */ - if((stat=nc_inq_varid(vid.grpid,segment,&vid.varid))) goto done; -done: - if(fqn) free(fqn); - if(stat == NC_NOERR && idp != NULL) *idp = vid; - return stat; -} -#endif - static int parsefilterspec(const char* optarg0, struct FilterSpec* spec) { @@ -266,9 +276,9 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec) if(optarg[0]=='/') spec->fqn = strdup(optarg); else { - spec->fqn = (char*)malloc(1+strlen(optarg)+1); + spec->fqn = (char*)malloc(1+strlen(optarg)+1); strcpy(spec->fqn,"/"); - strcat(spec->fqn,optarg); + strcat(spec->fqn,optarg); } /* Check for special cases */ @@ -280,10 +290,10 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec) /* Collect the id+parameters */ if((stat = NC_parsefilterspec(remainder,&id,&nparams,¶ms)) == NC_NOERR) { - if(spec != NULL) { - spec->filterid = id; - spec->nparams = nparams; - spec->params = params; + if(spec != NULL) { + spec->filterid = id; + spec->nparams = nparams; + spec->params = params; } } @@ -336,7 +346,7 @@ static int inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, size_t* chunkcache_sizep, size_t *chunkcache_nelemsp, - float * chunkcache_preemptionp) + float * chunkcache_preemptionp) { int stat = NC_NOERR; int ndims; @@ -583,7 +593,7 @@ copy_groups(int iroot, int oroot) /* get full group name of input group */ NC_CHECK(nc_inq_grpname(grpids[i], grpname)); if (option_grpstruct || group_wanted(grpids[i], option_nlgrps, option_grpids)) { - NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL)); + NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL)); grpname_full = emalloc(len_name + 1); NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, grpname_full)); /* Make sure, the parent group is also wanted (root group is always wanted) */ @@ -653,90 +663,10 @@ copy_types(int igrp, int ogrp) return stat; } -/* Copy all netCDF-4 specific variable properties such as chunking, - * endianness, deflation, checksumming, fill, etc. */ -static int -copy_var_specials(int igrp, int varid, int ogrp, int o_varid) -{ - int stat = NC_NOERR; - { /* handle chunking parameters */ - int ndims; - NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); - if (ndims > 0) { /* no chunking for scalar variables */ - int contig = 0; - size_t *chunkp = (size_t *) emalloc(ndims * sizeof(size_t)); - int *dimids = (int *) emalloc(ndims * sizeof(int)); - int idim; - /* size of a chunk: product of dimension chunksizes and size of value */ - size_t csprod = val_size(ogrp, o_varid); - int is_unlimited = 0; - NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, chunkp)); - NC_CHECK(nc_inq_vardimid(igrp, varid, dimids)); - - for(idim = 0; idim < ndims; idim++) { - int idimid = dimids[idim]; - int odimid = dimmap_odimid(idimid); - size_t chunksize = chunkspec_size(idimid); - if(chunksize > 0) { /* found in chunkspec */ - chunkp[idim] = chunksize; - } - csprod *= chunkp[idim]; - if(dimmap_ounlim(odimid)) - is_unlimited = 1; - } - /* Explicitly set chunking, even if default */ - /* If product of chunksizes is too small and no unlimited - * dimensions used, don't chunk. Also if chunking - * explicitly turned off with chunk spec, don't chunk. */ - if ((csprod < option_min_chunk_bytes && !is_unlimited) || contig == 1 - || chunkspec_omit() == true) { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); - } else { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp)); - } - free(dimids); - free(chunkp); - } - } - { /* handle compression parameters, copying from input, overriding - * with command-line options */ - int shuffle_in=0, deflate_in=0, deflate_level_in=0; - int shuffle_out=0, deflate_out=0, deflate_level_out=0; - if(option_deflate_level != 0) { - NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in)); - if(option_deflate_level == -1) { /* not specified, copy input compression and shuffling */ - shuffle_out = shuffle_in; - deflate_out = deflate_in; - deflate_level_out = deflate_level_in; - } else if(option_deflate_level > 0) { /* change to specified compression, shuffling */ - shuffle_out = option_shuffle_vars; - deflate_out=1; - deflate_level_out = option_deflate_level; - } - NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out)); - } - } - { /* handle checksum parameters */ - int fletcher32 = 0; - NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32)); - if(fletcher32 != 0) { - NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32)); - } - } - { /* handle endianness */ - int endianness = 0; - NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness)); - if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */ - NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness)); - } - } - return stat; -} - /* Copy netCDF-4 specific variable filter properties */ /* Watch out if input is netcdf-3 */ static int -copy_var_filter(int igrp, int varid, int ogrp, int o_varid) +copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind) { int stat = NC_NOERR; VarID vid = {igrp,varid}; @@ -747,14 +677,11 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid) actualspec = {NULL,0,0,0,NULL}; int i; char* ofqn = NULL; - int format, oformat; int inputdefined, outputdefined, unfiltered; + int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); + int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); - /* Get file format of the input and output */ - if((stat=nc_inq_format(vid.grpid,&format))) goto done; - if((stat=nc_inq_format(ovid.grpid,&oformat))) goto done; - - if(oformat != NC_FORMAT_NETCDF4 && oformat != NC_FORMAT_NETCDF4_CLASSIC) + if(!outnc4) goto done; /* Can only use filter when output is some netcdf4 variant */ /* Compute the output vid's FQN */ @@ -768,7 +695,7 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid) /* Is there a filter on the output variable */ outputdefined = 0; /* default is no filter defined */ /* Only bother to look if output is netcdf-4 variant */ - if(oformat == NC_FORMAT_NETCDF4 || oformat == NC_FORMAT_NETCDF4_CLASSIC) { + if(outnc4) { /* See if any output filter spec is defined for this output variable */ for(i=0;i 0) { /* found in chunkspec */ + ochunkp[idim] = chunksize; + ocontig = 0; /* cannot use contiguous */ + goto next; + } + + /* Not specified in -c; Apply defaulting rules as defined in nccopy.1 */ + + /* If input is chunked, then use that chunk size */ + if(!icontig) { + ochunkp[idim] = ichunkp[idim]; + ocontig = 0; + goto next; + } + + /* If input is netcdf-4 then use the input size as the chunk size; + but do not force chunking. + */ + if(!innc4) { + ochunkp[idim] = dimlen; + goto next; + } + + /* Default for unlimited is max(4 megabytes, current dim size) */ + if(is_unlimited) { + size_t mb4dimsize = DFALTUNLIMSIZE / typesize; + ochunkp[idim] = (dimlen > mb4dimsize ? mb4dimsize : dimlen); + } else { + /* final default is the current dimension size */ + ochunkp[idim] = dimlen; + } + +next: + /* compute on-going dimension product */ + csprod *= ochunkp[idim]; + } + + /* Finally, if total chunksize is too small (and dim is not unlimited) => do not chunk */ + if(csprod < option_min_chunk_bytes && !is_unlimited) { + ocontig = 1; /* Force contiguous */ + } + + /* Apply the chunking, if any */ + + if(ocontig) { /* We can use contiguous output */ + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); + } else { + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp)); + } + } +done: + return stat; +} + +/* Copy all netCDF-4 specific variable properties such as chunking, + * endianness, deflation, checksumming, fill, etc. */ +static int +copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind) +{ + int stat = NC_NOERR; + int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); + int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); + + if(!outnc4) + return stat; /* Ignore non-netcdf4 files */ + + { /* handle chunking parameters */ + int ndims; + NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); + if (ndims > 0) { /* no chunking for scalar variables */ + NC_CHECK(copy_chunking(igrp, varid, ogrp, o_varid, ndims, inkind, outkind)); + } + } + + { /* handle compression parameters, copying from input, overriding + * with command-line options */ + int shuffle_in=0, deflate_in=0, deflate_level_in=0; + int shuffle_out=0, deflate_out=0, deflate_level_out=0; + if(innc4) { /* See if the input variable has deflation applied */ + NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in)); + } + if(option_deflate_level == -1) { + /* not specified by -d flag, copy input compression and shuffling */ + shuffle_out = shuffle_in; + deflate_out = deflate_in; + deflate_level_out = deflate_level_in; + } else if(option_deflate_level > 0) { /* change to specified compression, shuffling */ + shuffle_out = option_shuffle_vars; + deflate_out=1; + deflate_level_out = option_deflate_level; + } else if(option_deflate_level == 0) { /* special case; force off */ + shuffle_out = 0; + deflate_out = 0; + deflate_level_out = 0; + } + /* Apply output deflation */ + if(outnc4) { + /* Note that if we invoke this function and even if shuffle and deflate flags are 0, + then default chunking will be turned on; so do a special check for that. */ + if(shuffle_out != 0 || deflate_out != 0) + NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out)); + } + } + + if(innc4 && outnc4) + { /* handle checksum parameters */ + int fletcher32 = 0; + NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32)); + if(fletcher32 != 0) { + NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32)); + } + } + + if(innc4 && outnc4) + { /* handle endianness */ + int endianness = 0; + NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness)); + if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */ + NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness)); + } + } + + /* handle other general filters */ + NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid, inkind, outkind)); + + return stat; +} + +#if 0 +Subsumed into copy_chunking. /* Set output variable o_varid (in group ogrp) to use chunking * specified on command line, only called for classic format input and * netCDF-4 format output, so no existing chunk lengths to override. */ @@ -850,7 +989,7 @@ set_var_chunked(int ogrp, int o_varid) int odim; size_t chunk_threshold = CHUNK_THRESHOLD; - if(chunkspec_ndims() == 0) /* no chunking specified on command line */ + if(dimchunkspec_ndims() == 0) /* no chunking specified on command line */ return stat; NC_CHECK(nc_inq_varndims(ogrp, o_varid, &ndims)); @@ -878,7 +1017,7 @@ set_var_chunked(int ogrp, int o_varid) if(dimmap_ounlim(odimid)) is_unlimited = 1; /* whether vriable is unlimited */ if(idimid != -1) { - size_t chunksize = chunkspec_size(idimid); /* from chunkspec */ + size_t chunksize = dimchunkspec_size(idimid); /* from chunkspec */ size_t dimlen; NC_CHECK(nc_inq_dimlen(ogrp, odimid, &dimlen)); if( (chunksize > 0) || dimmap_ounlim(odimid)) { @@ -901,7 +1040,7 @@ set_var_chunked(int ogrp, int o_varid) for(odim = 0; odim < ndims; odim++) { int odimid = dimids[odim]; int idimid = dimmap_idimid(odimid); - size_t chunksize = chunkspec_size(idimid); + size_t chunksize = dimchunkspec_size(idimid); if(chunksize > 0) { chunkp[odim] = chunksize; } else { @@ -919,7 +1058,9 @@ set_var_chunked(int ogrp, int o_varid) } return stat; } +#endif +#if 0 /* Set variable to compression specified on command line */ static int set_var_compressed(int ogrp, int o_varid) @@ -931,6 +1072,7 @@ set_var_compressed(int ogrp, int o_varid) } return stat; } +#endif /* Release the variable chunk cache allocated for variable varid in * group grp. This is not necessary, but will save some memory when @@ -1110,19 +1252,11 @@ copy_var(int igrp, int varid, int ogrp) int outkind; NC_CHECK(nc_inq_format(igrp, &inkind)); NC_CHECK(nc_inq_format(ogrp, &outkind)); - if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) { - if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) { - /* Copy all netCDF-4 specific variable properties such as - * chunking, endianness, deflation, checksumming, fill, etc. */ - NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid)); - } else { - /* Set chunking if specified in command line option */ - NC_CHECK(set_var_chunked(ogrp, o_varid)); - /* Set compression if specified in command line option */ - NC_CHECK(set_var_compressed(ogrp, o_varid)); - } - NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid)); - } + /* Copy all variable properties such as + * chunking, endianness, deflation, checksumming, fill, etc. + * Ok to call if outkind is netcdf-3 + */ + NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid, inkind, outkind)); } #endif /* USE_NETCDF4 */ free(idimids); @@ -1667,7 +1801,7 @@ copy(char* infile, char* outfile) || inkind == NC_FORMAT_CDF5) { if (option_deflate_level > 0 || option_shuffle_vars == NC_SHUFFLE || - option_chunkspec) + listlength(option_chunkspecs) > 0) { outkind = NC_FORMAT_NETCDF4_CLASSIC; } @@ -1675,10 +1809,14 @@ copy(char* infile, char* outfile) } #ifdef USE_NETCDF4 - if(option_chunkspec) { - /* Now that input is open, can parse option_chunkspec into binary + if(listlength(option_chunkspecs) > 0) { + int i; + /* Now that input is open, can parse option_chunkspecs into binary * structure. */ - NC_CHECK(chunkspec_parse(igrp, option_chunkspec)); + for(i=0;i= 0) - nc_set_log_level(level); - } + { + int level = atoi(optarg); + if(level >= 0) + nc_set_log_level(level); + } #endif - break; + break; case 'F': /* optional filter spec for a specified variable */ #ifdef USE_NETCDF4 /* If the arg is "none" then suppress all filters @@ -2058,7 +2154,16 @@ main(int argc, char**argv) #else error("-F requires netcdf-4"); #endif + case 'M': /* set min chunk size */ +#ifdef USE_NETCDF4 + option_min_chunk_bytes = atol(optarg); + if(option_min_chunk_bytes < 0) + error("-M value must be non-negative integer"); break; +#else + error("-M requires netcdf-4"); +#endif + default: usage(); } diff --git a/ncdump/tst_chunking.c b/ncdump/tst_chunking.c index 17e59100c..589f1c683 100644 --- a/ncdump/tst_chunking.c +++ b/ncdump/tst_chunking.c @@ -3,49 +3,135 @@ conditions of use. See www.unidata.ucar.edu for more info. Create a chunkable test file for nccopy to test chunking. - - $Id$ */ #include #include #include "err_macros.h" +#define DEBUG + +static int ret = NC_NOERR; + +/* Make trackable ERR macro replacement */ +static int lerr(int stat, const char* file, int lineno) { + fflush(stdout); /* Make sure our stdout is synced with stderr. */ + err++; + fprintf(stderr, "Sorry! Unexpected result(%d), %s, line: %d\n",ret,file,lineno); + fflush(stderr); \ + return 2; \ +} +#define LERR lerr(ret,__FILE__,__LINE__) + #define FILE_NAME "tst_chunking.nc" #define VAR_RANK 7 #define IVAR_NAME "ivar" #define FVAR_NAME "fvar" +#define GRP_NAME "g" +#define UNLIM_NAME "unlimited" +#define UNLIM_SIZE 10 +#define DEFLATE_LEVEL 1 #define NVALS 45360 /* 7 * 4 * 2 * 3 * 5 * 6 * 9 */ +static const char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; +static const size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; + int -main(int argc, char **argv) { - int ncid; +main(int argc, char **argv) +{ + /* mutually exclusive command line options */ + int option_group = 0; + int option_deflate = 0; + int option_unlimited = 0; + /* file metadata */ + int mode = NC_CLOBBER; + int ncid, grpid; int ivarid, fvarid; - char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; - size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; int ivar_dims[VAR_RANK]; int fvar_dims[VAR_RANK]; int ivar_data[NVALS]; float fvar_data[NVALS]; int r, i; + char* file_name = FILE_NAME; + int unlimid; - printf("*** Creating chunkable test file %s...", FILE_NAME); - - if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR; - for(r = 0; r < VAR_RANK; r++) { - if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) ERR; - fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; - } - if (nc_def_var(ncid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) ERR; - if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) ERR; - if (nc_enddef (ncid)) ERR; - for(i=0; i < NVALS; i++) { - ivar_data[i] = i; - fvar_data[i] = NVALS - i; + /* Parse command line */ + if(argc >= 2) { + file_name = argv[1]; } - if (nc_put_var(ncid, ivarid, ivar_data)) ERR; - if (nc_put_var(ncid, fvarid, fvar_data)) ERR; - if (nc_close(ncid)) ERR; + if(argc >= 3) { + if(strcmp(argv[2],"group")==0) { + option_group = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"deflate")==0) { + option_deflate = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"unlimited")==0) { + option_unlimited = 1; + } else { + fprintf(stderr,"usage: tst_chunking [ [group|deflate|unlimited]]\n"); + exit(1); + } + } + + printf("*** Creating chunkable test file %s...\n", file_name); + if(option_deflate) + printf("\toption: deflate\n"); + else if(option_unlimited) + printf("\toption: unlimited\n"); + else if(option_group) + printf("\toption: group\n"); + + if (nc_create(file_name, mode, &ncid)) LERR; + for(r = 0; r < VAR_RANK; r++) { + if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) LERR; + fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; + } + if(option_unlimited) { + int udims[2]; + if (nc_def_dim(ncid, UNLIM_NAME, 0, &unlimid)) LERR; + udims[0] = unlimid; + udims[1] = ivar_dims[0]; + if (nc_def_var(ncid, IVAR_NAME, NC_INT, 2, udims, &ivarid)) LERR; + } else { + if (option_group) { + if (nc_def_grp(ncid, GRP_NAME, &grpid)) LERR; + } else + grpid = ncid; + if (nc_def_var(grpid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) LERR; + if(option_deflate) { + if(nc_def_var_deflate(grpid,ivarid,NC_NOSHUFFLE, option_deflate, DEFLATE_LEVEL)) LERR; + } + } + /* fvar is unchanged */ + if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) LERR; + if (nc_enddef (ncid)) LERR; + + /* Fill in the data */ + if(option_unlimited) { + int nvals = UNLIM_SIZE * dim_lens[0]; + size_t start[2] = {0,0}; + size_t count[2]; + for(i=0;i +# usage: trim +# Leaves result in variable TRIMMED +trim() { + # trim leading whitespace and remove + TMP=`echo "$1" |tr -d '\r' | sed -e 's/^[ ]*//'` + # trim trailing whitespace + TRIMMED=`echo "$TMP" | sed -e 's/[ ]*$//'` +} + +# usage: checkfvar +checkfvar() { + # Make sure that fvar was not chunked + C5FVAR=`sed -e '/fvar:_ChunkSizes/p' -e d <$1` + if test "x$C5FVAR" != x ; then + echo "***Fail: fvar was chunked" + exit 1 + fi +} + +# usage: verifychunkline line1 line2 +verifychunkline() { + # trim leading whitespace + trim "$1"; L1="$TRIMMED" + trim "$2"; L2="$TRIMMED" + if test "x$L1" != "x$L2" ; then + echo "chunk line mismatch |$L1| |$L2|" + exit 1; + fi +} + +# Remove any temporary files +cleanup() { + rm -f tmp_nc5.nc tmp_nc5a.nc + rm -f tmp_nc5.cdl tmp_nc5a.cdl tmp_nc5b.cdl +} + +# remove all created files +reset() { + cleanup + rm -fr tst_nc5.nc tst_nc5.cdl +} + +reset + +if test "x$T1" = x1 ; then + +# Create a simple classic input file +./tst_chunking tst_nc5.nc + +# Save a .cdl version +${NCDUMP} tst_nc5.nc > tst_nc5.cdl + +echo "*** Test nccopy -c with per-variable chunking; classic->enhanced" +# This should produce same as -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ +${NCCOPY} -c ivar:7,1,2,1,5,1,9 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# Verify that the core cdl is the same +diff tst_nc5.cdl tmp_nc5.cdl + +# Look at the output chunking of ivar +rm -f tmp_nc5a.cdl # reuse +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc deflate +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Look at the output chunking +rm -f tmp_nc5.cdl # reuse +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc group +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c /g/ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Verify chunking +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc unlimited +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c ivar:5,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Verify chunking +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d Date: Sun, 29 Jul 2018 14:53:36 -0500 Subject: [PATCH 5/7] introduce error code NC_EPNETCDF for errors at PnetCDF level --- docs/all-error-codes.md | 1 + include/netcdf.h | 1 + libdispatch/derror.c | 2 + libsrcp/ncpdispatch.c | 96 ++++++++++++++++------------------------- nc_test/util.c | 7 ++- 5 files changed, 46 insertions(+), 61 deletions(-) diff --git a/docs/all-error-codes.md b/docs/all-error-codes.md index b5b0d5864..78829c2ab 100644 --- a/docs/all-error-codes.md +++ b/docs/all-error-codes.md @@ -71,6 +71,7 @@ may occur. #define NC_ENOTFOUND (-90) // No such file #define NC_ECANTREMOVE (-91) // Cannot remove file #define NC_EINTERNAL (-92) // NetCDF Library Internal Error +#define NC_EPNETCDF (-93) // Error at PnetCDF layer ~~~~ # NetCDF-4 Error Codes {#nc4-error-codes} diff --git a/include/netcdf.h b/include/netcdf.h index 58c789e05..a4b62e009 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -426,6 +426,7 @@ by the desired type. */ #define NC_ENOTFOUND (-90) /**< No such file */ #define NC_ECANTREMOVE (-91) /**< Can't remove file */ #define NC_EINTERNAL (-92) /**< NetCDF Library Internal Error */ +#define NC_EPNETCDF (-93) /**< Error at PnetCDF layer */ /* The following was added in support of netcdf-4. Make all netcdf-4 error codes < -100 so that errors can be added to netcdf-3 if diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 4ee2b93b7..6a2b83fc0 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -196,6 +196,8 @@ const char *nc_strerror(int ncerr1) return "NetCDF: cannot delete file"; case NC_EINTERNAL: return "NetCDF: internal library error; Please contact Unidata support"; + case NC_EPNETCDF: + return "NetCDF: PnetCDF error"; case NC_EHDFERR: return "NetCDF: HDF error"; case NC_ECANTREAD: diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index d651abc4f..cc5e8eea1 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -15,14 +15,9 @@ typedef struct NCP_INFO { - /* pnetcdf_file will be true if the file is created/opened with the - * parallel-netcdf library. pnetcdf_access_mode keeps track of - * whether independpent or collective mode is - * desired. pnetcdf_ndims keeps track of how many dims each var - * has, which I need to know to convert start, count, and stride - * arrays from size_t to MPI_Offset. (I can't use an inq function - * to find out the number of dims, because these are collective in - * pnetcdf.) */ + /* pnetcdf_access_mode keeps track of whether independpent or collective + * mode is set currently. + */ int pnetcdf_access_mode; } NCP_INFO; @@ -46,8 +41,8 @@ NCP_create(const char *path, int cmode, { int res, default_format; NCP_INFO* nc5; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; + MPI_Comm comm; + MPI_Info info; /* Check the cmode for only valid flags*/ if(cmode & ~LEGAL_CREATE_FLAGS) @@ -88,23 +83,9 @@ NCP_create(const char *path, int cmode, /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); - /* Fix up the cmode by keeping only essential flags; - these are the flags that are the same in netcf.h and pnetcdf.h - */ - /* It turns out that pnetcdf.h defines a flag called - NC_64BIT_DATA (not to be confused with NC_64BIT_OFFSET). - This flag is essential to getting ncmpi_create to create - a proper pnetcdf format file. - We have set the value of NC_64BIT_DATA to be the same as in pnetcdf.h - (as of pnetcdf version 1.6.0) to avoid conflicts. - In any case, this flag must be set. - */ - /* PnetCDF recognizes the flags below for create and ignores NC_LOCK and NC_SHARE */ - cmode &= (NC_WRITE | NC_NOCLOBBER | NC_SHARE | NC_64BIT_OFFSET | NC_64BIT_DATA); - res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); - if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ + if (res != NC_NOERR) free(nc5); /* reclaim allocated space */ done: return res; } @@ -117,36 +98,19 @@ NCP_open(const char *path, int cmode, { int res; NCP_INFO* nc5; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; + MPI_Comm comm; + MPI_Info info; /* Check the cmode for only valid flags*/ if(cmode & ~LEGAL_OPEN_FLAGS) {res = NC_EINVAL; goto done;} - /* Cannot have both MPIO flags */ - if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX)) - {res = NC_EINVAL; goto done;} + /* No MPI environment initialized */ + if (mpidata == NULL) + {res = NC_ENOPAR; goto done;} - /* Appears that this comment is wrong; allow 64 bit offset*/ - /* Cannot have 64 bit offset flag */ - /* if(cmode & (NC_64BIT_OFFSET)) {res = NC_EINVAL; goto done;} */ - if(mpidata != NULL) { - comm = ((NC_MPI_INFO *)mpidata)->comm; - info = ((NC_MPI_INFO *)mpidata)->info; - } else { - comm = MPI_COMM_WORLD; - info = MPI_INFO_NULL; - } - - /* PnetCDF recognizes the flags NC_WRITE and NC_NOCLOBBER for file open - * and ignores NC_LOCK, NC_SHARE, NC_64BIT_OFFSET, and NC_64BIT_DATA. - * Ignoring the NC_64BIT_OFFSET and NC_64BIT_DATA flags is because the - * file is already in one of the CDF-formats, and setting these 2 flags - * will not change the format of that file. - */ - - cmode &= (NC_WRITE | NC_NOCLOBBER); + comm = ((NC_MPI_INFO *)mpidata)->comm; + info = ((NC_MPI_INFO *)mpidata)->info; /* Create our specific NCP_INFO instance */ nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); @@ -158,7 +122,7 @@ NCP_open(const char *path, int cmode, res = ncmpi_open(comm, path, cmode, info, &(nc->int_ncid)); /* Default to independent access, like netCDF-4/HDF5 files. */ - if(!res) { + if (res == NC_NOERR) { res = ncmpi_begin_indep_data(nc->int_ncid); nc5->pnetcdf_access_mode = NC_INDEPENDENT; } @@ -193,9 +157,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t nc5 = NCP_DATA(nc); assert(nc5); - /* causes implicitly defined warning; may be because of old installed pnetcdf? */ -#if 1 - /* In PnetCDF ncmpi__enddef() is only implemented in v1.5.0 and later */ +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10500) + /* ncmpi__enddef() was first implemented in PnetCDF v1.5.0 */ status = ncmpi__enddef(nc->int_ncid, mpi_h_minfree, mpi_v_align, mpi_v_minfree, mpi_r_align); #else @@ -257,7 +220,12 @@ NCP_set_fill(int ncid, int fillmode, int *old_mode_ptr) NC* nc; int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_set_fill was first implemented in PnetCDF 1.6.1 */ return ncmpi_set_fill(nc->int_ncid,fillmode,old_mode_ptr); +#else + return NC_EPNETCDF; +#endif } static int @@ -314,7 +282,7 @@ NCP_inq_type(int ncid, nc_type typeid, char* name, size_t* size) { /* Assert mode & NC_FORMAT_CDF5 */ if (typeid < NC_BYTE || typeid >= NC_STRING) - return NC_EBADTYPE; + return NC_EBADTYPE; if(name) strcpy(name, NC_atomictypename(typeid)); if(size) @@ -440,15 +408,14 @@ NCP_get_att( { NC* nc; int status; - nc_type xtype; status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; - status = NCP_inq_att(ncid,varid,name,&xtype,NULL); - if(status != NC_NOERR) return status; - - if(memtype == NC_NAT) memtype = xtype; + if (memtype == NC_NAT) { + status = NCP_inq_att(ncid,varid,name,&memtype,NULL); + if (status != NC_NOERR) return status; + } switch (memtype) { case NC_CHAR: @@ -1171,7 +1138,13 @@ NCP_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, if(deflatep) *deflatep = 0; if(fletcher32p) *fletcher32p = 0; if(contiguousp) *contiguousp = NC_CONTIGUOUS; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_inq_var_fill was first implemented in PnetCDF 1.6.1 */ if(no_fill) ncmpi_inq_var_fill(nc->int_ncid, varid, no_fill, fill_valuep); +#else + /* PnetCDF 1.6.0 and priors support NC_NOFILL only */ + if(no_fill) *no_fill = 1; +#endif if(endiannessp) return NC_ENOTNC4; if(idp) return NC_ENOTNC4; if(nparamsp) return NC_ENOTNC4; @@ -1185,7 +1158,12 @@ NCP_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) NC* nc; int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_def_var_fill was first implemented in PnetCDF 1.6.1 */ return ncmpi_def_var_fill(nc->int_ncid, varid, no_fill, fill_value); +#else + return NC_EPNETCDF; +#endif } static int diff --git a/nc_test/util.c b/nc_test/util.c index 811f7eacb..005c76081 100644 --- a/nc_test/util.c +++ b/nc_test/util.c @@ -1284,6 +1284,8 @@ char* nc_err_code_name(int err) case (NC_EAUTH): return "NC_EAUTH"; case (NC_ENOTFOUND): return "NC_ENOTFOUND"; case (NC_ECANTREMOVE): return "NC_ECANTREMOVE"; + case (NC_EINTERNAL): return "NC_EINTERNAL"; + case (NC_EPNETCDF): return "NC_EPNETCDF"; case (NC_EHDFERR): return "NC_EHDFERR"; case (NC_ECANTREAD): return "NC_ECANTREAD"; case (NC_ECANTWRITE): return "NC_ECANTWRITE"; @@ -1315,8 +1317,9 @@ char* nc_err_code_name(int err) case (NC_EDISKLESS): return "NC_EDISKLESS"; case (NC_ECANTEXTEND): return "NC_ECANTEXTEND"; case (NC_EMPI): return "NC_EMPI"; - case (NC_ENULLPAD): return "NC_NULLPAD"; - // case (NC_EURL): return "NC_EURL"; + case (NC_ENULLPAD): return "NC_NULLPAD"; + case (NC_EINMEMORY): return "NC_EINMEMORY"; + // case (NC_EURL): return "NC_EURL"; // case (NC_ECONSTRAINT): return "NC_ECONSTRAINT"; #ifdef USE_PNETCDF case (NC_ESMALL): return "NC_ESMALL"; From 70e454227e59762ab700e6867c2461ae8f1ecd34 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 30 Jul 2018 15:25:45 -0600 Subject: [PATCH 6/7] Corrected an issue on Windows. --- ncdump/tst_nccopy4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index dde77a39d..53797c4d8 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -83,7 +83,7 @@ diff tmp.cdl tmp-chunked.cdl $NCCOPY -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl -$NCCOPY -c / tmp-chunked.nc tmp-unchunked.nc +$NCCOPY -c // tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl echo "*** Test that nccopy -c works as intended for record dimension default (1)" From bf6ae6c59153dd2d36dbdf7793e29f824a98a20d Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 31 Jul 2018 12:51:24 -0600 Subject: [PATCH 7/7] Added ocprint binary to cmakelists. --- ncdump/CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 67b6e67d0..cb338c205 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -5,17 +5,22 @@ ENDIF() SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c) SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) +SET(ocprint_FILES ocprint.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) SET(nccopy_FILES ${nccopy_FILES} XGetopt.c) + SET(ocprint_FILES ${ocprint_FILES} XGetopt.c) + ENDIF() ADD_EXECUTABLE(ncdump ${ncdump_FILES}) ADD_EXECUTABLE(nccopy ${nccopy_FILES}) +ADD_EXECUTABLE(ocprint ${ocprint_FILES}) TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS}) +TARGET_LINK_LIBRARIES(ocprint netcdf ${ALL_TLL_LIBS}) #### # We have to do a little tweaking @@ -37,6 +42,13 @@ IF(MSVC) ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) ENDIF() @@ -236,12 +248,16 @@ IF(MSVC) SET_TARGET_PROPERTIES(nccopy PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" ) + SET_TARGET_PROPERTIES(ocprint + PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" + ) ENDIF() INSTALL(TARGETS ncdump RUNTIME DESTINATION bin COMPONENT utilities) INSTALL(TARGETS nccopy RUNTIME DESTINATION bin COMPONENT utilities) +INSTALL(TARGETS ocprint RUNTIME DESTINATION bin COMPONENT utilities) SET(MAN_FILES nccopy.1 ncdump.1) # Note, the L512.bin file is file containing exactly 512 bytes each of value 0.