From c8f3a647b6538f287a3147bf1ba089e9125b5e1e Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Fri, 13 Jul 2018 16:32:43 -0500 Subject: [PATCH 01/16] add NC_RELAX_COORD_BOUND to netcdf_meta.h.in --- configure.ac | 4 +++- include/netcdf_meta.h.in | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2d2eff41a..f748d002d 100644 --- a/configure.ac +++ b/configure.ac @@ -1397,9 +1397,10 @@ AC_SUBST(HAS_PARALLEL4,[$enable_parallel4]) AC_SUBST(HAS_DISKLESS,[yes]) AC_SUBST(HAS_MMAP,[$enable_mmap]) AC_SUBST(HAS_JNA,[$enable_jna]) -AC_SUBST(RELAX_COORD_BOUND,[$enable_relax_coord_bound]) +AC_SUBST(RELAX_COORD_BOUND,[$enable_zero_length_coord_bound]) AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill]) + # Include some specifics for netcdf on windows. #AH_VERBATIM([_WIN32_STRICMP], AH_BOTTOM( @@ -1466,6 +1467,7 @@ AX_SET_META([NC_HAS_PARALLEL],[$enable_parallel],[yes]) AX_SET_META([NC_HAS_PARALLEL4],[$enable_parallel4],[yes]) AX_SET_META([NC_HAS_CDF5],[$enable_cdf5],[yes]) AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes]) +AX_SET_META([NC_RELAX_COORD_BOUND], [$enable_zero_length_coord_bound],[yes]) # Automake says that this is always run in top_builddir # and that srcdir is defined (== top_srcdir) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 54a2ff0e3..6ad98f1d8 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -53,5 +53,6 @@ #define NC_HAS_CDF5 @NC_HAS_CDF5@ /*!< CDF5 support. */ #define NC_HAS_ERANGE_FILL @NC_HAS_ERANGE_FILL@ /*!< ERANGE_FILL Support */ +#define NC_RELAX_COORD_BOUND @NC_RELAX_COORD_BOUND@ /*!< RELAX_COORD_BOUND */ #endif From d0cb241cfe0457a2b937ea6d7babe314bfcb2c60 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sat, 14 Jul 2018 18:03:42 -0500 Subject: [PATCH 02/16] add NC_HAS_PARALLEL4 to netcdf_meta.h.in --- include/netcdf_meta.h.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 54a2ff0e3..3f93b452a 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -48,8 +48,8 @@ #define NC_HAS_MMAP @NC_HAS_MMAP@ /*!< mmap support. */ #define NC_HAS_JNA @NC_HAS_JNA@ /*!< jna support. */ #define NC_HAS_PNETCDF @NC_HAS_PNETCDF@ /*!< pnetcdf support. */ -#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and -/or pnetcdf. */ +#define NC_HAS_PARALLEL4 @NC_HAS_PARALLEL4@ /*!< parallel IO support via hdf5 */ +#define NC_HAS_PARALLEL @NC_HAS_PARALLEL@ /*!< parallel IO support via hdf5 and/or pnetcdf. */ #define NC_HAS_CDF5 @NC_HAS_CDF5@ /*!< CDF5 support. */ #define NC_HAS_ERANGE_FILL @NC_HAS_ERANGE_FILL@ /*!< ERANGE_FILL Support */ From 739dfa2aa2127cea842a90a75a69f68aa41c34a1 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sun, 15 Jul 2018 16:19:21 -0500 Subject: [PATCH 03/16] PnetCDF does not support per-variable collective/independent data mode change; modify nc_var_par_access to ignore varid --- docs/tutorial.dox | 6 +++- examples/C/parallel_vara.c | 4 +-- libsrcp/ncpdispatch.c | 71 ++++++++++++++++++++++++++++++-------- nc_test/tst_parallel2.c | 5 +++ nc_test/tst_pnetcdf.c | 6 ++-- nc_test/tst_small.c | 6 ++-- nc_test/util.c | 10 ++---- 7 files changed, 74 insertions(+), 34 deletions(-) diff --git a/docs/tutorial.dox b/docs/tutorial.dox index 640204b4f..3d3287353 100644 --- a/docs/tutorial.dox +++ b/docs/tutorial.dox @@ -809,7 +809,11 @@ create/open a netCDF file with parallel access. Parallel file access is either collective (all processors must participate) or independent (any processor may access the data without -waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access(). +waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. +To change from collective to independent mode or vis versa, call +nc_var_par_access() with argument 'access' set to either NC_INDEPENDENT or +NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid' is ignored, as +PnetCDF does not support per-variable collective/independent mode change. \page tutorial_ncids Numbering of NetCDF IDs diff --git a/examples/C/parallel_vara.c b/examples/C/parallel_vara.c index 4aa789c81..757d91769 100644 --- a/examples/C/parallel_vara.c +++ b/examples/C/parallel_vara.c @@ -150,7 +150,7 @@ int main(int argc, char** argv) err = nc_enddef(ncid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* now we are in data mode */ start[0] = 0; @@ -176,7 +176,7 @@ int main(int argc, char** argv) err = nc_inq_varid(ncid, "var", &varid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* each process reads its subarray from the file */ err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index d651abc4f..9e561057c 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -9,10 +9,14 @@ #include #include "nc.h" #include "ncdispatch.h" +#include "fbits.h" /* Must follow netcdf.h */ #include +#define NCP_MODE_DATA 0x0001 +#define NCP_MODE_INDEP 0x0002 + typedef struct NCP_INFO { /* pnetcdf_file will be true if the file is created/opened with the @@ -104,6 +108,9 @@ NCP_create(const char *path, int cmode, res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); + /* Default to independent access, like netCDF-4/HDF5 files. */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ done: return res; @@ -152,6 +159,9 @@ NCP_open(const char *path, int cmode, nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); if(nc5 == NULL) {res = NC_ENOMEM; goto done;} + /* file open automatically enters data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); @@ -160,7 +170,7 @@ NCP_open(const char *path, int cmode, /* Default to independent access, like netCDF-4/HDF5 files. */ if(!res) { res = ncmpi_begin_indep_data(nc->int_ncid); - nc5->pnetcdf_access_mode = NC_INDEPENDENT; + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); } done: return res; @@ -170,8 +180,14 @@ static int NCP_redef(int ncid) { NC* nc; + NCP_INFO* nc5; + int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; + + nc5 = NCP_DATA(nc); + fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + return ncmpi_redef(nc->int_ncid); } @@ -203,7 +219,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t #endif if(!status) { - if (nc5->pnetcdf_access_mode == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) status = ncmpi_begin_indep_data(nc->int_ncid); } return status; @@ -611,7 +628,7 @@ NCP_get_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -705,7 +722,7 @@ NCP_put_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -801,7 +818,7 @@ NCP_get_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -897,7 +914,7 @@ NCP_put_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -995,7 +1012,7 @@ NCP_get_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1093,7 +1110,7 @@ NCP_put_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1198,19 +1215,43 @@ NCP_var_par_access(int ncid, int varid, int par_access) if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE) return NC_EINVAL; +#ifdef _DO_NOT_IGNORE_VARID_ + if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */ + return NC_EINVAL; +#endif + status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; nc5 = NCP_DATA(nc); assert(nc5); - if(par_access == nc5->pnetcdf_access_mode) - return NC_NOERR; - nc5->pnetcdf_access_mode = par_access; - if (par_access == NC_INDEPENDENT) - return ncmpi_begin_indep_data(nc->int_ncid); - else - return ncmpi_end_indep_data(nc->int_ncid); + /* if currently in data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) { + if (par_access == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) + return NC_NOERR; + else { /* currently in collective data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_begin_indep_data(nc->int_ncid); + } + } + else { /* want to enter collective data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_end_indep_data(nc->int_ncid); + } + else + return NC_NOERR; + } + } + else { /* currently in define mode */ + if (par_access == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + else + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + } + return NC_NOERR; } #ifdef USE_NETCDF4 diff --git a/nc_test/tst_parallel2.c b/nc_test/tst_parallel2.c index 92c271d26..45f61a4bc 100644 --- a/nc_test/tst_parallel2.c +++ b/nc_test/tst_parallel2.c @@ -135,8 +135,13 @@ main(int argc, char **argv) sleep(mpi_rank); #endif /* USE_MPE */ +#ifdef USE_PNETCDF +/* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/ + if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR; +#else /* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/ if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR; +#endif if (!mpi_rank) start_time = MPI_Wtime(); diff --git a/nc_test/tst_pnetcdf.c b/nc_test/tst_pnetcdf.c index 812ab5dfd..bcf8d21d5 100644 --- a/nc_test/tst_pnetcdf.c +++ b/nc_test/tst_pnetcdf.c @@ -68,10 +68,8 @@ int main(int argc, char* argv[]) } if (nc_enddef(ncid)) ERR; - for (i=0; i Date: Thu, 19 Jul 2018 07:05:55 -0600 Subject: [PATCH 04/16] split nc4var.c --- libhdf5/CMakeLists.txt | 2 +- libhdf5/Makefile.am | 2 +- libsrc4/nc4var.c | 1034 +--------------------------------------- 3 files changed, 10 insertions(+), 1028 deletions(-) diff --git a/libhdf5/CMakeLists.txt b/libhdf5/CMakeLists.txt index 09d554af5..ff9ce739e 100644 --- a/libhdf5/CMakeLists.txt +++ b/libhdf5/CMakeLists.txt @@ -7,7 +7,7 @@ # The source files for the HDF5 dispatch layer. SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c -hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c) +hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c) # Build the HDF4 dispatch layer as a library that will be included in # the netCDF library. diff --git a/libhdf5/Makefile.am b/libhdf5/Makefile.am index 3d61a4f51..e1f95a70e 100644 --- a/libhdf5/Makefile.am +++ b/libhdf5/Makefile.am @@ -14,7 +14,7 @@ noinst_LTLIBRARIES = libnchdf5.la # The source files. libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \ -hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c +hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5var.c # Package this for cmake build. EXTRA_DIST = CMakeLists.txt diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index c0d95d862..30d1d08cb 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -1,13 +1,13 @@ +/* Copyright 2003-2006, University Corporation for Atmospheric + * Research. See COPYRIGHT file for copying and redistribution + * conditions.*/ /** * @file - * This file is part of netcdf-4, a netCDF-like interface for HDF5, or a - * HDF5 backend for netCDF, depending on your point of view. - - * This file handles the NetCDF-4 variable functions. - - * Copyright 2003-2006, University Corporation for Atmospheric - * Research. See COPYRIGHT file for copying and redistribution - * conditions. + * @internal This file is part of netcdf-4, a netCDF-like interface + * for HDF5, or a HDF5 backend for netCDF, depending on your point of + * view. This file handles the NetCDF-4 variable functions. + * + * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher */ #include @@ -27,42 +27,6 @@ extern int nc4_get_default_fill_value(const NC_TYPE_INFO_T *type_info, void *fill_value); -/** - * @internal If the HDF5 dataset for this variable is open, then close - * it and reopen it, with the perhaps new settings for chunk caching. - * - * @param grp Pointer to the group info. - * @param var Pointer to the var info. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EHDFERR HDF5 error. - * @author Ed Hartnett - */ -int -nc4_reopen_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) -{ - hid_t access_pid; - - if (var->hdf_datasetid) - { - if ((access_pid = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - return NC_EHDFERR; - if (H5Pset_chunk_cache(access_pid, var->chunk_cache_nelems, - var->chunk_cache_size, - var->chunk_cache_preemption) < 0) - return NC_EHDFERR; - if (H5Dclose(var->hdf_datasetid) < 0) - return NC_EHDFERR; - if ((var->hdf_datasetid = H5Dopen2(grp->hdf_grpid, var->hdr.name, - access_pid)) < 0) - return NC_EHDFERR; - if (H5Pclose(access_pid) < 0) - return NC_EHDFERR; - } - - return NC_NOERR; -} - /** * @internal Set chunk cache size for a variable. This is the internal * function called by nc_set_var_chunk_cache(). @@ -396,293 +360,6 @@ nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) return NC_NOERR; } -/** - * @internal This is called when a new netCDF-4 variable is defined - * with nc_def_var(). - * - * @param ncid File ID. - * @param name Name. - * @param xtype Type. - * @param ndims Number of dims. - * @param dimidsp Array of dim IDs. - * @param varidp Gets the var ID. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 - * netcdf-4 file. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EPERM File is read only. - * @returns ::NC_EMAXDIMS Classic model file exceeds ::NC_MAX_VAR_DIMS. - * @returns ::NC_ESTRICTNC3 Attempting to create netCDF-4 type var in - * classic model file - * @returns ::NC_EBADNAME Bad name. - * @returns ::NC_EBADTYPE Bad type. - * @returns ::NC_ENOMEM Out of memory. - * @returns ::NC_EHDFERR Error returned by HDF5 layer. - * @returns ::NC_EINVAL Invalid input - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_def_var(int ncid, const char *name, nc_type xtype, - int ndims, const int *dimidsp, int *varidp) -{ - NC_GRP_INFO_T *grp; - NC_VAR_INFO_T *var; - NC_DIM_INFO_T *dim; - NC_FILE_INFO_T *h5; - NC_TYPE_INFO_T *type_info = NULL; - char norm_name[NC_MAX_NAME + 1]; - int d; - int retval; - - /* Find info for this file and group, and set pointer to each. */ - if ((retval = nc4_find_grp_h5(ncid, &grp, &h5))) - BAIL(retval); - assert(grp && h5); - - /* If it's not in define mode, strict nc3 files error out, - * otherwise switch to define mode. This will also check that the - * file is writable. */ - if (!(h5->flags & NC_INDEF)) - { - if (h5->cmode & NC_CLASSIC_MODEL) - BAIL(NC_ENOTINDEFINE); - if ((retval = NC4_redef(ncid))) - BAIL(retval); - } - assert(!h5->no_write); - - /* Check and normalize the name. */ - if ((retval = nc4_check_name(name, norm_name))) - BAIL(retval); - - /* Not a Type is, well, not a type.*/ - if (xtype == NC_NAT) - BAIL(NC_EBADTYPE); - - /* For classic files, only classic types are allowed. */ - if (h5->cmode & NC_CLASSIC_MODEL && xtype > NC_DOUBLE) - BAIL(NC_ESTRICTNC3); - - /* For classic files */ - if (h5->cmode & NC_CLASSIC_MODEL && ndims > NC_MAX_VAR_DIMS) - BAIL(NC_EMAXDIMS); - - /* cast needed for braindead systems with signed size_t */ - if((unsigned long) ndims > X_INT_MAX) /* Backward compat */ - BAIL(NC_EINVAL); - - /* Check that this name is not in use as a var, grp, or type. */ - if ((retval = nc4_check_dup_name(grp, norm_name))) - BAIL(retval); - - /* For non-scalar vars, dim IDs must be provided. */ - if (ndims && !dimidsp) - BAIL(NC_EINVAL); - - /* Check all the dimids to make sure they exist. */ - for (d = 0; d < ndims; d++) - if ((retval = nc4_find_dim(grp, dimidsp[d], &dim, NULL))) - BAIL(retval); - - /* These degrubbing messages sure are handy! */ - LOG((2, "%s: name %s type %d ndims %d", __func__, norm_name, xtype, ndims)); -#ifdef LOGGING - { - int dd; - for (dd = 0; dd < ndims; dd++) - LOG((4, "dimid[%d] %d", dd, dimidsp[dd])); - } -#endif - - /* If this is a user-defined type, there is a type_info struct with - * all the type information. For atomic types, fake up a type_info - * struct. */ - if (xtype <= NC_STRING) - { - size_t len; - char name[NC_MAX_NAME+1]; - char* atomname = nc4_atomic_name[xtype]; - size_t namelen = strlen(atomname); - memcpy(name,atomname,namelen); - name[namelen] = '\0'; - nc4_get_typelen_mem(h5,xtype,&len); - if((retval = nc4_type_new(grp,len,name,xtype,&type_info))) - BAIL(retval); - type_info->endianness = NC_ENDIAN_NATIVE; - if ((retval = nc4_get_hdf_typeid(h5, xtype, &type_info->hdf_typeid, - type_info->endianness))) - BAIL(retval); - if ((type_info->native_hdf_typeid = H5Tget_native_type(type_info->hdf_typeid, - H5T_DIR_DEFAULT)) < 0) - BAIL(NC_EHDFERR); - if ((retval = nc4_get_typelen_mem(h5, type_info->hdr.id, &type_info->size))) - BAIL(retval); - - /* Set the "class" of the type */ - if (xtype == NC_CHAR) - type_info->nc_type_class = NC_CHAR; - else - { - H5T_class_t class; - - if ((class = H5Tget_class(type_info->hdf_typeid)) < 0) - BAIL(NC_EHDFERR); - switch(class) - { - case H5T_STRING: - type_info->nc_type_class = NC_STRING; - break; - - case H5T_INTEGER: - type_info->nc_type_class = NC_INT; - break; - - case H5T_FLOAT: - type_info->nc_type_class = NC_FLOAT; - break; - - default: - BAIL(NC_EBADTYPID); - } - } - } - /* If this is a user defined type, find it. */ - else - { - if (nc4_find_type(grp->nc4_info, xtype, &type_info)) - BAIL(NC_EBADTYPE); - } - - /* Create a new var and fill in some HDF5 cache setting values. */ - if ((retval = nc4_var_list_add(grp, norm_name, ndims, &var))) - BAIL(retval); - var->is_new_var = NC_TRUE; - - /* Point to the type, and increment its ref. count */ - var->type_info = type_info; - var->type_info->rc++; - type_info = NULL; - - /* Set variables no_fill to match the database default - * unless the variable type is variable length (NC_STRING or NC_VLEN) - * or is user-defined type. - */ - if (var->type_info->nc_type_class < NC_STRING) - var->no_fill = h5->fill_mode; - - /* Assign dimensions to the variable */ - /* At the same time, check to see if this is a coordinate - * variable. If so, it will have the same name as one of its - * dimensions. If it is a coordinate var, is it a coordinate var in - * the same group as the dim? */ - /* Also, check whether we should use contiguous or chunked storage */ - var->contiguous = NC_TRUE; - for (d = 0; d < ndims; d++) - { - NC_GRP_INFO_T *dim_grp; - - /* Look up each dimension */ - if ((retval = nc4_find_dim(grp, dimidsp[d], &dim, &dim_grp))) - BAIL(retval); - - /* Check for dim index 0 having the same name, in the same group */ - if (d == 0 && dim_grp == grp && strcmp(dim->hdr.name, norm_name) == 0) - { - var->dimscale = NC_TRUE; - dim->coord_var = var; - - /* Use variable's dataset ID for the dimscale ID. So delete - * the HDF5 DIM_WITHOUT_VARIABLE dataset that was created for - * this dim. */ - if (dim->hdf_dimscaleid) - { - /* Detach dimscale from any variables using it */ - if ((retval = rec_detach_scales(grp, dimidsp[d], dim->hdf_dimscaleid)) < 0) - BAIL(retval); - - /* Close the HDF5 DIM_WITHOUT_VARIABLE dataset. */ - if (H5Dclose(dim->hdf_dimscaleid) < 0) - BAIL(NC_EHDFERR); - dim->hdf_dimscaleid = 0; - - /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be - * recreated later, if necessary). */ - if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) - BAIL(NC_EDIMMETA); - } - } - - /* Check for unlimited dimension and turn off contiguous storage. */ - if (dim->unlimited) - var->contiguous = NC_FALSE; - - /* Track dimensions for variable */ - var->dimids[d] = dimidsp[d]; - var->dim[d] = dim; - } - - /* Determine default chunksizes for this variable. (Even for - * variables which may be contiguous.) */ - LOG((4, "allocating array of %d size_t to hold chunksizes for var %s", - var->ndims, var->hdr.name)); - if (var->ndims) - if (!(var->chunksizes = calloc(var->ndims, sizeof(size_t)))) - BAIL(NC_ENOMEM); - - if ((retval = nc4_find_default_chunksizes2(grp, var))) - BAIL(retval); - - /* Is this a variable with a chunksize greater than the current - * cache size? */ - if ((retval = nc4_adjust_var_cache(grp, var))) - BAIL(retval); - - /* If the user names this variable the same as a dimension, but - * doesn't use that dimension first in its list of dimension ids, - * is not a coordinate variable. I need to change its HDF5 name, - * because the dimension will cause a HDF5 dataset to be created, - * and this var has the same name. */ - dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); - if (dim && (!var->ndims || dimidsp[0] != dim->hdr.id)) - { - /* Set a different hdf5 name for this variable to avoid name - * clash. */ - if (strlen(norm_name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) - BAIL(NC_EMAXNAME); - if (!(var->hdf5_name = malloc((strlen(NON_COORD_PREPEND) + - strlen(norm_name) + 1) * sizeof(char)))) - BAIL(NC_ENOMEM); - - sprintf(var->hdf5_name, "%s%s", NON_COORD_PREPEND, norm_name); - } - - /* If this is a coordinate var, it is marked as a HDF5 dimension - * scale. (We found dim above.) Otherwise, allocate space to - * remember whether dimension scales have been attached to each - * dimension. */ - if (!var->dimscale && ndims) - if (!(var->dimscale_attached = calloc(ndims, sizeof(nc_bool_t)))) - BAIL(NC_ENOMEM); - - /* Return the varid. */ - if (varidp) - *varidp = var->hdr.id; - LOG((4, "new varid %d", var->hdr.id)); - -exit: - if (type_info) - if ((retval = nc4_type_free(type_info))) - BAIL2(retval); - - return retval; -} - /** * @internal Get all the information about a variable. Pass NULL for * whatever you don't care about. This is the internal function called @@ -869,299 +546,6 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, return NC_NOERR; } -/** - * @internal This functions sets extra stuff about a netCDF-4 variable which - * must be set before the enddef but after the def_var. - * - * @note All pointer parameters may be NULL, in which case they are ignored. - * @param ncid File ID. - * @param varid Variable ID. - * @param shuffle Pointer to shuffle setting. - * @param deflate Pointer to deflate setting. - * @param deflate_level Pointer to deflate level. - * @param fletcher32 Pointer to fletcher32 setting. - * @param contiguous Pointer to contiguous setting. - * @param chunksizes Array of chunksizes. - * @param no_fill Pointer to no_fill setting. - * @param fill_value Pointer to fill value. - * @param endianness Pointer to endianness setting. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 - * netcdf-4 file. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EPERM File is read only. - * @returns ::NC_EINVAL Invalid input - * @returns ::NC_EBADCHUNK Bad chunksize. - * @author Ed Hartnett - */ -static int -nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate, - int *deflate_level, int *fletcher32, int *contiguous, - const size_t *chunksizes, int *no_fill, - const void *fill_value, int *endianness) -{ - NC *nc; - NC_GRP_INFO_T *grp; - NC_FILE_INFO_T *h5; - NC_VAR_INFO_T *var; - int d; - int retval; - - /* All or none of these will be provided. */ - assert((deflate && deflate_level && shuffle) || - (!deflate && !deflate_level && !shuffle)); - - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - - /* Find info for this file and group, and set pointer to each. */ - if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; - assert(nc && grp && h5); - - /* Trying to write to a read-only file? No way, Jose! */ - if (h5->no_write) - return NC_EPERM; - - /* Find the var. */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); - if(!var) - return NC_ENOTVAR; - assert(var && var->hdr.id == varid); - - /* Can't turn on parallel and deflate/fletcher32/szip/shuffle. */ - if (nc->mode & (NC_MPIIO | NC_MPIPOSIX)) { - if (deflate || fletcher32 || shuffle) - return NC_EINVAL; - } - - /* If the HDF5 dataset has already been created, then it is too - * late to set all the extra stuff. */ - if (var->created) - return NC_ELATEDEF; - - /* Check compression options. */ - if (deflate && !deflate_level) - return NC_EINVAL; - - /* Valid deflate level? */ - if (deflate) - { - if (*deflate) - if (*deflate_level < NC_MIN_DEFLATE_LEVEL || - *deflate_level > NC_MAX_DEFLATE_LEVEL) - return NC_EINVAL; - - /* For scalars, just ignore attempt to deflate. */ - if (!var->ndims) - return NC_NOERR; - - /* Well, if we couldn't find any errors, I guess we have to take - * the users settings. Darn! */ - var->contiguous = NC_FALSE; - var->deflate = *deflate; - if (*deflate) - var->deflate_level = *deflate_level; - LOG((3, "%s: *deflate_level %d", __func__, *deflate_level)); - } - - /* Shuffle filter? */ - if (shuffle) - { - var->shuffle = *shuffle; - var->contiguous = NC_FALSE; - } - - /* Fletcher32 checksum error protection? */ - if (fletcher32) - { - var->fletcher32 = *fletcher32; - var->contiguous = NC_FALSE; - } - - /* Does the user want a contiguous dataset? Not so fast! Make sure - * that there are no unlimited dimensions, and no filters in use - * for this data. */ - if (contiguous && *contiguous) - { - if (var->deflate || var->fletcher32 || var->shuffle) - return NC_EINVAL; - - for (d = 0; d < var->ndims; d++) - if (var->dim[d]->unlimited) - return NC_EINVAL; - var->contiguous = NC_TRUE; - } - - /* Chunksizes anyone? */ - if (contiguous && *contiguous == NC_CHUNKED) - { - var->contiguous = NC_FALSE; - - /* If the user provided chunksizes, check that they are not too - * big, and that their total size of chunk is less than 4 GB. */ - if (chunksizes) - { - - if ((retval = check_chunksizes(grp, var, chunksizes))) - return retval; - - /* Ensure chunksize is smaller than dimension size */ - for (d = 0; d < var->ndims; d++) - if(!var->dim[d]->unlimited && var->dim[d]->len > 0 && chunksizes[d] > var->dim[d]->len) - return NC_EBADCHUNK; - - /* Set the chunksizes for this variable. */ - for (d = 0; d < var->ndims; d++) - var->chunksizes[d] = chunksizes[d]; - } - } - - /* Is this a variable with a chunksize greater than the current - * cache size? */ - if (!var->contiguous && (deflate || contiguous)) - { - /* Determine default chunksizes for this variable (do nothing - * for scalar vars). */ - if (var->chunksizes && !var->chunksizes[0]) - if ((retval = nc4_find_default_chunksizes2(grp, var))) - return retval; - - /* Adjust the cache. */ - if ((retval = nc4_adjust_var_cache(grp, var))) - return retval; - } - - /* Are we setting a fill modes? */ - if (no_fill) - { - if (*no_fill) - { - /* NC_STRING types may not turn off fill mode. It's disallowed - * by HDF5 and will cause a HDF5 error later. */ - if (*no_fill) - if (var->type_info->hdr.id == NC_STRING) - return NC_EINVAL; - - /* Set the no-fill mode. */ - var->no_fill = NC_TRUE; - } - else - var->no_fill = NC_FALSE; - } - - /* Are we setting a fill value? */ - if (fill_value && !var->no_fill) - { - /* Copy the fill_value. */ - LOG((4, "Copying fill value into metadata for variable %s", - var->hdr.name)); - - /* If there's a _FillValue attribute, delete it. */ - retval = NC4_del_att(ncid, varid, _FillValue); - if (retval && retval != NC_ENOTATT) - return retval; - - /* Create a _FillValue attribute. */ - if ((retval = nc_put_att(ncid, varid, _FillValue, var->type_info->hdr.id, 1, fill_value))) - return retval; - } - - /* Is the user setting the endianness? */ - if (endianness) - var->type_info->endianness = *endianness; - - return NC_NOERR; -} - -/** - * @internal Set compression settings on a variable. This is called by - * nc_def_var_deflate(). - * - * @param ncid File ID. - * @param varid Variable ID. - * @param shuffle True to turn on the shuffle filter. - * @param deflate True to turn on deflation. - * @param deflate_level A number between 0 (no compression) and 9 - * (maximum compression). - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EINVAL Invalid input - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_def_var_deflate(int ncid, int varid, int shuffle, int deflate, - int deflate_level) -{ - return nc_def_var_extra(ncid, varid, &shuffle, &deflate, - &deflate_level, NULL, NULL, NULL, NULL, NULL, NULL); -} - -/** - * @internal Set checksum on a variable. This is called by - * nc_def_var_fletcher32(). - * - * @param ncid File ID. - * @param varid Variable ID. - * @param fletcher32 Pointer to fletcher32 setting. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EINVAL Invalid input - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_def_var_fletcher32(int ncid, int varid, int fletcher32) -{ - return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, &fletcher32, - NULL, NULL, NULL, NULL, NULL); -} - -/** - * @internal Define chunking stuff for a var. This is called by - * nc_def_var_chunking(). Chunking is required in any dataset with one - * or more unlimited dimensions in HDF5, or any dataset using a - * filter. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param contiguous Pointer to contiguous setting. - * @param chunksizesp Array of chunksizes. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EINVAL Invalid input - * @returns ::NC_EBADCHUNK Bad chunksize. - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_def_var_chunking(int ncid, int varid, int contiguous, const size_t *chunksizesp) -{ - return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, - &contiguous, chunksizesp, NULL, NULL, NULL); -} - /** * @internal Inquire about chunking settings for a var. This is used * by the fortran API. @@ -1277,151 +661,6 @@ nc_def_var_chunking_ints(int ncid, int varid, int contiguous, int *chunksizesp) return retval; } -/** - * @internal This functions sets fill value and no_fill mode for a - * netCDF-4 variable. It is called by nc_def_var_fill(). - * - * @note All pointer parameters may be NULL, in which case they are ignored. - * @param ncid File ID. - * @param varid Variable ID. - * @param no_fill No_fill setting. - * @param fill_value Pointer to fill value. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 - * netcdf-4 file. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EPERM File is read only. - * @returns ::NC_EINVAL Invalid input - * @author Ed Hartnett - */ -int -NC4_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) -{ - return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, - NULL, &no_fill, fill_value, NULL); -} - -/** - * @internal This functions sets endianness for a netCDF-4 - * variable. Called by nc_def_var_endian(). - * - * @note All pointer parameters may be NULL, in which case they are ignored. - * @param ncid File ID. - * @param varid Variable ID. - * @param endianness Endianness setting. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ESTRICTNC3 Attempting netcdf-4 operation on strict nc3 - * netcdf-4 file. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_ENOTINDEFINE Not in define mode. - * @returns ::NC_EPERM File is read only. - * @returns ::NC_EINVAL Invalid input - * @author Ed Hartnett - */ -int -NC4_def_var_endian(int ncid, int varid, int endianness) -{ - return nc_def_var_extra(ncid, varid, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, &endianness); -} - -/** - * @internal Define filter settings. Called by nc_def_var_filter(). - * - * @param ncid File ID. - * @param varid Variable ID. - * @param id Filter ID - * @param nparams Number of parameters for filter. - * @param parms Filter parameters. - * - * @returns ::NC_NOERR for success - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_ENOTNC4 Attempting netcdf-4 operation on file that is - * not netCDF-4/HDF5. - * @returns ::NC_ELATEDEF Too late to change settings for this variable. - * @returns ::NC_EFILTER Filter error. - * @returns ::NC_EINVAL Invalid input - * @author Dennis Heimbigner - */ -int -NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, - const unsigned int* parms) -{ - int retval = NC_NOERR; - NC *nc; - NC_GRP_INFO_T *grp; - NC_FILE_INFO_T *h5; - NC_VAR_INFO_T *var; - - LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); - - /* Find info for this file and group, and set pointer to each. */ - if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; - - assert(nc && grp && h5); - - /* Find the var. */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); - if(!var) - return NC_ENOTVAR; - assert(var->hdr.id == varid); - - /* Can't turn on parallel and filters */ - if (nc->mode & (NC_MPIIO | NC_MPIPOSIX)) { - return NC_EINVAL; - } - - /* If the HDF5 dataset has already been created, then it is too - * late to set all the extra stuff. */ - if (var->created) - return NC_ELATEDEF; - -#ifdef HAVE_H5Z_SZIP - if(id == H5Z_FILTER_SZIP) { - if(nparams != 2) - return NC_EFILTER; /* incorrect no. of parameters */ - } -#else /*!HAVE_H5Z_SZIP*/ - if(id == H5Z_FILTER_SZIP) - return NC_EFILTER; /* Not allowed */ -#endif - -#if 0 - { - unsigned int fcfg = 0; - herr_t herr = H5Zget_filter_info(id,&fcfg); - if(herr < 0) - return NC_EFILTER; - if((H5Z_FILTER_CONFIG_ENCODE_ENABLED & fcfg) == 0 - || (H5Z_FILTER_CONFIG_DECODE_ENABLED & fcfg) == 0) - return NC_EFILTER; - } -#endif /*0*/ - - var->filterid = id; - var->nparams = nparams; - var->params = NULL; - if(parms != NULL) { - var->params = (unsigned int*)calloc(nparams,sizeof(unsigned int)); - if(var->params == NULL) return NC_ENOMEM; - memcpy(var->params,parms,sizeof(unsigned int)*var->nparams); - } - return NC_NOERR; -} - /** * @internal Find the ID of a variable, from the name. This function * is called by nc_inq_varid(). @@ -1468,138 +707,6 @@ NC4_inq_varid(int ncid, const char *name, int *varidp) return NC_ENOTVAR; } -/** - * @internal Rename a var to "bubba," for example. This is called by - * nc_rename_var() for netCDF-4 files. - * - * @param ncid File ID. - * @param varid Variable ID - * @param name New name of the variable. - * - * @returns ::NC_NOERR No error. - * @returns ::NC_EBADID Bad ncid. - * @returns ::NC_ENOTVAR Invalid variable ID. - * @returns ::NC_EBADNAME Bad name. - * @returns ::NC_EMAXNAME Name is too long. - * @returns ::NC_ENAMEINUSE Name in use. - * @returns ::NC_ENOMEM Out of memory. - * @author Ed Hartnett -*/ -int -NC4_rename_var(int ncid, int varid, const char *name) -{ - NC *nc; - NC_GRP_INFO_T *grp; - NC_FILE_INFO_T *h5; - NC_VAR_INFO_T *var, *tmpvar; - int retval = NC_NOERR; - - if (!name) - return NC_EINVAL; - - LOG((2, "%s: ncid 0x%x varid %d name %s", __func__, ncid, varid, - name)); - - /* Find info for this file and group, and set pointer to each. */ - if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) - return retval; - assert(h5 && grp && h5); - - /* Is the new name too long? */ - if (strlen(name) > NC_MAX_NAME) - return NC_EMAXNAME; - - /* Trying to write to a read-only file? No way, Jose! */ - if (h5->no_write) - return NC_EPERM; - - /* Check name validity, if strict nc3 rules are in effect for this - * file. */ - if ((retval = NC_check_name(name))) - return retval; - - /* Get the variable wrt varid */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); - if (!var) - return NC_ENOTVAR; - - /* Check if new name is in use; note that renaming to same name is still an error - according to the nc_test/test_write.c code. Why?*/ - tmpvar = (NC_VAR_INFO_T*)ncindexlookup(grp->vars,name); - if(tmpvar != NULL) - return NC_ENAMEINUSE; - - /* If we're not in define mode, new name must be of equal or - less size, if strict nc3 rules are in effect for this . */ - if (!(h5->flags & NC_INDEF) && strlen(name) > strlen(var->hdr.name) && - (h5->cmode & NC_CLASSIC_MODEL)) - return NC_ENOTINDEFINE; - - /* Change the HDF5 file, if this var has already been created - there. Should we check here to ensure there is not already a - dimscale dataset of name name??? */ - if (var->created) - { - /* Is there an existing dimscale-only dataset of this name? If - * so, it must be deleted. */ - if (var->ndims && var->dim[0]->hdf_dimscaleid) - { - if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, var->dim[0]))) - return retval; - } - - LOG((3, "Moving dataset %s to %s", var->hdr.name, name)); - if (H5Gmove(grp->hdf_grpid, var->hdr.name, name) < 0) - BAIL(NC_EHDFERR); - } - - /* Now change the name in our metadata. */ - free(var->hdr.name); - if (!(var->hdr.name = strdup(name))) - return NC_ENOMEM; - LOG((3, "var is now %s", var->hdr.name)); - var->hdr.hashkey = NC_hashmapkey(var->hdr.name,strlen(var->hdr.name)); /* Fix hash key */ - - if(!ncindexrebuild(grp->vars)) - return NC_EINTERNAL; - - /* Check if this was a coordinate variable previously, but names are different now */ - if (var->dimscale && strcmp(var->hdr.name, var->dim[0]->hdr.name)) - { - /* Break up the coordinate variable */ - if ((retval = nc4_break_coord_var(grp, var, var->dim[0]))) - return retval; - } - - /* Check if this should become a coordinate variable */ - if (!var->dimscale) - { - /* Only variables with >0 dimensions can become coordinate variables */ - if (var->ndims) - { - NC_GRP_INFO_T *dim_grp; - NC_DIM_INFO_T *dim; - - /* Check to see if this is became a coordinate variable. If so, it - * will have the same name as dimension index 0. If it is a - * coordinate var, is it a coordinate var in the same group as the dim? - */ - if ((retval = nc4_find_dim(grp, var->dimids[0], &dim, &dim_grp))) - return retval; - if (!strcmp(dim->hdr.name, name) && dim_grp == grp) - { - /* Reform the coordinate variable */ - if ((retval = nc4_reform_coord_var(grp, var, dim))) - return retval; - var->became_coord_var = NC_TRUE; - } - } - } - -exit: - return retval; -} - /** * @internal * @@ -1657,128 +764,3 @@ NC4_var_par_access(int ncid, int varid, int par_access) #endif /* USE_PARALLEL4 */ } -/** - * @internal Write an array of data to a variable. This is called by - * nc_put_vara() and other nc_put_vara_* functions, for netCDF-4 - * files. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param startp Array of start indices. - * @param countp Array of counts. - * @param op pointer that gets the data. - * @param memtype The type of these data in memory. - * - * @returns ::NC_NOERR for success - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_put_vara(int ncid, int varid, const size_t *startp, - const size_t *countp, const void *op, int memtype) -{ - NC *nc; - - if (!(nc = nc4_find_nc_file(ncid, NULL))) - return NC_EBADID; - - return nc4_put_vars(nc, ncid, varid, startp, countp, NULL, memtype, - (void *)op); -} - -/** - * Read an array of values. This is called by nc_get_vara() for - * netCDF-4 files, as well as all the other nc_get_vara_* - * functions. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param startp Array of start indices. - * @param countp Array of counts. - * @param ip pointer that gets the data. - * @param memtype The type of these data after it is read into memory. - - * @returns ::NC_NOERR for success - * @author Ed Hartnett, Dennis Heimbigner - */ -int -NC4_get_vara(int ncid, int varid, const size_t *startp, - const size_t *countp, void *ip, int memtype) -{ - NC *nc; - NC_FILE_INFO_T* h5; - - LOG((2, "%s: ncid 0x%x varid %d memtype %d", __func__, ncid, varid, - memtype)); - - if (!(nc = nc4_find_nc_file(ncid, &h5))) - return NC_EBADID; - - /* Get the data. */ - return nc4_get_vars(nc, ncid, varid, startp, countp, NULL, memtype, - (void *)ip); -} - -/** - * @internal Write an array of data to a variable. This is called by - * nc_put_vars() and other nc_put_vars_* functions, for netCDF-4 - * files. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param startp Array of start indices. - * @param countp Array of counts. - * @param stridep Array of strides. - * @param op pointer that gets the data. - * @param memtype The type of these data in memory. - * - * @returns ::NC_NOERR for success - * @author Dennis Heimbigner - */ -int -NC4_put_vars(int ncid, int varid, const size_t *startp, - const size_t *countp, const ptrdiff_t* stridep, - const void *op, int memtype) -{ - NC *nc; - - if (!(nc = nc4_find_nc_file(ncid, NULL))) - return NC_EBADID; - - return nc4_put_vars(nc, ncid, varid, startp, countp, stridep, memtype, - (void *)op); -} - -/** - * Read an array of values. This is called by nc_get_vars() for - * netCDF-4 files, as well as all the other nc_get_vars_* - * functions. - * - * @param ncid File ID. - * @param varid Variable ID. - * @param startp Array of start indices. - * @param countp Array of counts. - * @param stridep Array of strides. - * @param ip pointer that gets the data. - * @param memtype The type of these data after it is read into memory. - - * @returns ::NC_NOERR for success - * @author Dennis Heimbigner - */ -int -NC4_get_vars(int ncid, int varid, const size_t *startp, - const size_t *countp, const ptrdiff_t *stridep, - void *ip, int memtype) -{ - NC *nc; - NC_FILE_INFO_T *h5; - - LOG((2, "%s: ncid 0x%x varid %d memtype %d", __func__, ncid, varid, - memtype)); - - if (!(nc = nc4_find_nc_file(ncid, &h5))) - return NC_EBADID; - - /* Get the data. */ - return nc4_get_vars(nc, ncid, varid, startp, countp, stridep, memtype, - (void *)ip); -} From 9b0192fe94c05ab2080b547115d0016b92fb558c Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 19 Jul 2018 07:26:27 -0600 Subject: [PATCH 05/16] moved memfile code to libhdf5 --- libhdf5/CMakeLists.txt | 2 +- libhdf5/Makefile.am | 2 +- {libsrc4 => libhdf5}/nc4mem.c | 0 {libsrc4 => libhdf5}/nc4memcb.c | 0 libsrc4/CMakeLists.txt | 2 +- libsrc4/Makefile.am | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename {libsrc4 => libhdf5}/nc4mem.c (100%) rename {libsrc4 => libhdf5}/nc4memcb.c (100%) diff --git a/libhdf5/CMakeLists.txt b/libhdf5/CMakeLists.txt index bfeb8d0ef..b04150f58 100644 --- a/libhdf5/CMakeLists.txt +++ b/libhdf5/CMakeLists.txt @@ -8,7 +8,7 @@ # The source files for the HDF5 dispatch layer. SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c -hdf5var.c) +hdf5var.c nc4mem.c nc4memcb.c) # Build the HDF4 dispatch layer as a library that will be included in # the netCDF library. diff --git a/libhdf5/Makefile.am b/libhdf5/Makefile.am index 645db023e..c7cb43444 100644 --- a/libhdf5/Makefile.am +++ b/libhdf5/Makefile.am @@ -15,7 +15,7 @@ noinst_LTLIBRARIES = libnchdf5.la # The source files. libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \ hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \ -hdf5var.c +hdf5var.c nc4mem.c nc4memcb.c # Package this for cmake build. EXTRA_DIST = CMakeLists.txt diff --git a/libsrc4/nc4mem.c b/libhdf5/nc4mem.c similarity index 100% rename from libsrc4/nc4mem.c rename to libhdf5/nc4mem.c diff --git a/libsrc4/nc4memcb.c b/libhdf5/nc4memcb.c similarity index 100% rename from libsrc4/nc4memcb.c rename to libhdf5/nc4memcb.c diff --git a/libsrc4/CMakeLists.txt b/libsrc4/CMakeLists.txt index a6dd5d63f..f54d652c3 100644 --- a/libsrc4/CMakeLists.txt +++ b/libsrc4/CMakeLists.txt @@ -1,6 +1,6 @@ # Process these files with m4. -SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c ncindex.c nc4mem.c nc4memcb.c ) +SET(libsrc4_SOURCES nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c nc4type.c nc4var.c ncfunc.c nc4internal.c ncindex.c ) IF(LOGGING) SET(libsrc4_SOURCES ${libsrc4_SOURCES} error4.c) diff --git a/libsrc4/Makefile.am b/libsrc4/Makefile.am index f60b30dc3..0c59d6ae8 100644 --- a/libsrc4/Makefile.am +++ b/libsrc4/Makefile.am @@ -12,7 +12,7 @@ libnetcdf4_la_CPPFLAGS = ${AM_CPPFLAGS} noinst_LTLIBRARIES = libnetcdf4.la libnetcdf4_la_SOURCES = nc4dispatch.c nc4attr.c nc4dim.c nc4grp.c \ nc4internal.c nc4type.c nc4var.c ncfunc.c error4.c nc4printer.c \ -ncindex.c nc4mem.c nc4memcb.c +ncindex.c EXTRA_DIST = CMakeLists.txt From d1198a366731e2d026da86659df8e6c52ba987b4 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Thu, 19 Jul 2018 07:33:48 -0600 Subject: [PATCH 06/16] whitespace cleanup --- libhdf5/hdf5var.c | 52 +++++++++++++++++++++++------------------------ libsrc4/nc4var.c | 15 +++++++------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index a5e8ee2f1..9950f9a98 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -5,7 +5,7 @@ * @file * @internal This file handles the HDF5 variable functions. * - * @author Ed Hartnett, Dennis Heimbigner, Ward Fisher + * @author Ed Hartnett */ #include "config.h" @@ -297,7 +297,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* For non-scalar vars, dim IDs must be provided. */ if (ndims && !dimidsp) - BAIL(NC_EINVAL); + BAIL(NC_EINVAL); /* Check all the dimids to make sure they exist. */ for (d = 0; d < ndims; d++) @@ -327,7 +327,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, name[namelen] = '\0'; nc4_get_typelen_mem(h5,xtype,&len); if((retval = nc4_type_new(grp,len,name,xtype,&type_info))) - BAIL(retval); + BAIL(retval); type_info->endianness = NC_ENDIAN_NATIVE; if ((retval = nc4_get_hdf_typeid(h5, xtype, &type_info->hdf_typeid, type_info->endianness))) @@ -410,7 +410,7 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, { var->dimscale = NC_TRUE; dim->coord_var = var; - + /* Use variable's dataset ID for the dimscale ID. So delete * the HDF5 DIM_WITHOUT_VARIABLE dataset that was created for * this dim. */ @@ -419,12 +419,12 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, /* Detach dimscale from any variables using it */ if ((retval = rec_detach_scales(grp, dimidsp[d], dim->hdf_dimscaleid)) < 0) BAIL(retval); - + /* Close the HDF5 DIM_WITHOUT_VARIABLE dataset. */ if (H5Dclose(dim->hdf_dimscaleid) < 0) BAIL(NC_EHDFERR); dim->hdf_dimscaleid = 0; - + /* Now delete the DIM_WITHOUT_VARIABLE dataset (it will be * recreated later, if necessary). */ if (H5Gunlink(grp->hdf_grpid, dim->hdr.name) < 0) @@ -464,17 +464,17 @@ NC4_def_var(int ncid, const char *name, nc_type xtype, * and this var has the same name. */ dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); if (dim && (!var->ndims || dimidsp[0] != dim->hdr.id)) - { - /* Set a different hdf5 name for this variable to avoid name - * clash. */ - if (strlen(norm_name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) - BAIL(NC_EMAXNAME); - if (!(var->hdf5_name = malloc((strlen(NON_COORD_PREPEND) + - strlen(norm_name) + 1) * sizeof(char)))) - BAIL(NC_ENOMEM); + { + /* Set a different hdf5 name for this variable to avoid name + * clash. */ + if (strlen(norm_name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) + BAIL(NC_EMAXNAME); + if (!(var->hdf5_name = malloc((strlen(NON_COORD_PREPEND) + + strlen(norm_name) + 1) * sizeof(char)))) + BAIL(NC_ENOMEM); - sprintf(var->hdf5_name, "%s%s", NON_COORD_PREPEND, norm_name); - } + sprintf(var->hdf5_name, "%s%s", NON_COORD_PREPEND, norm_name); + } /* If this is a coordinate var, it is marked as a HDF5 dimension * scale. (We found dim above.) Otherwise, allocate space to @@ -504,7 +504,7 @@ exit: * @note All pointer parameters may be NULL, in which case they are ignored. * @param ncid File ID. * @param varid Variable ID. - * @param shuffle Pointer to shuffle setting. + * @param shuffle Pointer to shuffle setting. * @param deflate Pointer to deflate setting. * @param deflate_level Pointer to deflate level. * @param fletcher32 Pointer to fletcher32 setting. @@ -1007,7 +1007,7 @@ NC4_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, * @returns ::NC_ENAMEINUSE Name in use. * @returns ::NC_ENOMEM Out of memory. * @author Ed Hartnett -*/ + */ int NC4_rename_var(int ncid, int varid, const char *name) { @@ -1042,7 +1042,7 @@ NC4_rename_var(int ncid, int varid, const char *name) return retval; /* Get the variable wrt varid */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); if (!var) return NC_ENOTVAR; @@ -1050,7 +1050,7 @@ NC4_rename_var(int ncid, int varid, const char *name) according to the nc_test/test_write.c code. Why?*/ tmpvar = (NC_VAR_INFO_T*)ncindexlookup(grp->vars,name); if(tmpvar != NULL) - return NC_ENAMEINUSE; + return NC_ENAMEINUSE; /* If we're not in define mode, new name must be of equal or less size, if strict nc3 rules are in effect for this . */ @@ -1070,7 +1070,7 @@ NC4_rename_var(int ncid, int varid, const char *name) if ((retval = delete_existing_dimscale_dataset(grp, var->dim[0]->hdr.id, var->dim[0]))) return retval; } - + LOG((3, "Moving dataset %s to %s", var->hdr.name, name)); if (H5Gmove(grp->hdf_grpid, var->hdr.name, name) < 0) BAIL(NC_EHDFERR); @@ -1084,7 +1084,7 @@ NC4_rename_var(int ncid, int varid, const char *name) var->hdr.hashkey = NC_hashmapkey(var->hdr.name,strlen(var->hdr.name)); /* Fix hash key */ if(!ncindexrebuild(grp->vars)) - return NC_EINTERNAL; + return NC_EINTERNAL; /* Check if this was a coordinate variable previously, but names are different now */ if (var->dimscale && strcmp(var->hdr.name, var->dim[0]->hdr.name)) @@ -1127,7 +1127,7 @@ exit: * @internal Write an array of data to a variable. This is called by * nc_put_vara() and other nc_put_vara_* functions, for netCDF-4 * files. - * + * * @param ncid File ID. * @param varid Variable ID. * @param startp Array of start indices. @@ -1188,7 +1188,7 @@ NC4_get_vara(int ncid, int varid, const size_t *startp, * @internal Write an array of data to a variable. This is called by * nc_put_vars() and other nc_put_vars_* functions, for netCDF-4 * files. - * + * * @param ncid File ID. * @param varid Variable ID. * @param startp Array of start indices. @@ -1203,7 +1203,7 @@ NC4_get_vara(int ncid, int varid, const size_t *startp, int NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, const ptrdiff_t* stridep, - const void *op, int memtype) + const void *op, int memtype) { NC *nc; @@ -1233,7 +1233,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, int NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, const ptrdiff_t *stridep, - void *ip, int memtype) + void *ip, int memtype) { NC *nc; NC_FILE_INFO_T *h5; diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index 25617394d..b9d6437b3 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -56,7 +56,7 @@ NC4_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, /* Find the var. */ var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); if(!var) - return NC_ENOTVAR; + return NC_ENOTVAR; assert(var && var->hdr.id == varid); /* Set the values. */ @@ -138,7 +138,7 @@ NC4_get_var_chunk_cache(int ncid, int varid, size_t *sizep, /* Find the var. */ var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); if(!var) - return NC_ENOTVAR; + return NC_ENOTVAR; assert(var && var->hdr.id == varid); /* Give the user what they want. */ @@ -264,7 +264,7 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, if ((retval = nc4_read_grp_atts(grp))) return retval; - *nattsp = ncindexcount(grp->att); + *nattsp = ncindexcount(grp->att); } return NC_NOERR; } @@ -474,10 +474,10 @@ NC4_inq_varid(int ncid, const char *name, int *varidp) /* Find var of this name. */ var = (NC_VAR_INFO_T*)ncindexlookup(grp->vars,norm_name); if(var) - { - *varidp = var->hdr.id; - return NC_NOERR; - } + { + *varidp = var->hdr.id; + return NC_NOERR; + } return NC_ENOTVAR; } @@ -537,4 +537,3 @@ NC4_var_par_access(int ncid, int varid, int par_access) return NC_NOERR; #endif /* USE_PARALLEL4 */ } - From b02703aa24e2a7a75c49b3cba61c1205df6992bb Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 26 Jul 2018 20:16:02 -0600 Subject: [PATCH 07/16] This PR primarily addresses Issue https://github.com/Unidata/netcdf-c/issues/725. After a long discussion, I implemented the rules at the end of that issue. They are documented in nccopy.1. Additionally, I added a new, per-variable, -c flag that allows for the direct setting of the chunking parameters for a variable. The form is -c var:c1,c2,...ck where var is the name of the variable (possibly a fully qualified name) and the ci are the chunksizes for that variable. It must be the case that the rank of the variable is k. If the new form is used as well as the old form, then the new form overrides the old form for the specified variable. Note that multiple occurrences of the new form -c flag may be specified. Misc. Other fixes 1. Added -M option to nccopy to specify the minimum allowable chunksize. 2. Removed the unused variables from bigmeta.c (Issue https://github.com/Unidata/netcdf-c/issues/1079) 3. Fixed failure of nc_test4/tst_filter.sh by using the new -M flag (#1) to allow filter test on a small chunk size. --- cf | 4 +- cf.cmake | 2 +- libdap4/d4data.c | 3 +- nc_test4/Make0 | 8 +- nc_test4/bigmeta.c | 5 +- nc_test4/findplugin.in | 6 + nc_test4/tst_filter.sh | 3 +- ncdump/CMakeLists.txt | 7 +- ncdump/Make0 | 8 +- ncdump/Makefile.am | 10 +- ncdump/chunkspec.c | 249 ++++++++++++++++--- ncdump/chunkspec.h | 17 +- ncdump/list.c | 221 +++++++++++++++++ ncdump/list.h | 62 +++++ ncdump/nccopy.1 | 35 +++ ncdump/nccopy.c | 533 ++++++++++++++++++++++++----------------- ncdump/tst_chunking.c | 130 ++++++++-- ncdump/tst_nccopy5.sh | 167 +++++++++++++ ncdump/utils.c | 132 +++++++++- ncdump/utils.h | 6 + 20 files changed, 1310 insertions(+), 298 deletions(-) create mode 100644 ncdump/list.c create mode 100644 ncdump/list.h create mode 100755 ncdump/tst_nccopy5.sh diff --git a/cf b/cf index a2a3d7c9d..1d123b52d 100644 --- a/cf +++ b/cf @@ -1,6 +1,6 @@ #!/bin/bash #NB=1 -DB=1 +#DB=1 #X=-x #FAST=1 @@ -118,7 +118,7 @@ FLAGS="$FLAGS --disable-diskless" #FLAGS="$FLAGS --enable-jna" #FLAGS="$FLAGS --disable-properties-attribute" #FLAGS="$FLAGS --disable-silent-rules" -FLAGS="$FLAGS --disable-filter-testing" +#FLAGS="$FLAGS --disable-filter-testing" #FLAGS="$FLAGS --enable-metadata-perf" if test "x$TESTSERVERS" != x ; then diff --git a/cf.cmake b/cf.cmake index 9c257a248..1dd4ad581 100644 --- a/cf.cmake +++ b/cf.cmake @@ -60,7 +60,7 @@ FLAGS="$FLAGS -DENABLE_DAP_REMOTE_TESTS=true" FLAGS="$FLAGS -DENABLE_LOGGING=true" #FLAGS="$FLAGS -DENABLE_DOXYGEN=true -DENABLE_INTERNAL_DOCS=true" #FLAGS="$FLAGS -DENABLE_LARGE_FILE_TESTS=true" -#FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" +FLAGS="$FLAGS -DENABLE_FILTER_TESTING=true" # Disables FLAGS="$FLAGS -DENABLE_EXAMPLES=false" diff --git a/libdap4/d4data.c b/libdap4/d4data.c index a60210be0..c448e865c 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -99,8 +99,7 @@ NCD4_processdata(NCD4meta* meta) for(i=0;idata.localchecksum != var->data.remotechecksum) { - fprintf(stderr,"Checksum mismatch: %s\n",var->name); - fflush(stderr); + nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } diff --git a/nc_test4/Make0 b/nc_test4/Make0 index df06ce208..22f7a0a75 100644 --- a/nc_test4/Make0 +++ b/nc_test4/Make0 @@ -1,5 +1,5 @@ # Test c output -T=tst_varsperf +T=bigmeta #SRC=hdf5plugins/H5Zmisc.c @@ -12,7 +12,8 @@ T=tst_varsperf #PAR=1 #SZIP=1 -CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include +#CFLAGS = -Wall -Wno-unused-variable -Wno-unused-function -g -O0 -I.. -I../include +CFLAGS = -Wall -g -O0 -I.. -I../include LDFLAGS = ../liblib/.libs/libnetcdf.a -L/usr/local/lib -lhdf5_hl -lhdf5 -lz -ldl -lcurl -lm -lmfhdf -ldf @@ -31,7 +32,8 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH} all:: cmp export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CMD} ./t ${ARGS} + +# ${CMD} ./t ${ARGS} cmp:: export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ diff --git a/nc_test4/bigmeta.c b/nc_test4/bigmeta.c index 20e8fcfcb..fb2054eb2 100644 --- a/nc_test4/bigmeta.c +++ b/nc_test4/bigmeta.c @@ -134,7 +134,6 @@ static void buildatts(int grpid, int varid) { char name[NC_MAX_NAME+1]; - int attid; int i, count; count = (varid == NC_GLOBAL? ngroupattrs : nvarattrs); @@ -149,7 +148,7 @@ static void buildgroup(int parent, int grpindex, int depth) { char name[NC_MAX_NAME+1]; - int i, grpid, dimid, typid, varid, attid; + int i, grpid, varid; int dimids[NDIMS]; if(depth == 0) return; @@ -193,9 +192,7 @@ buildgroup(int parent, int grpindex, int depth) int main(int argc, char **argv) { - int stat = NC_NOERR; int i, ncid; - int grpid, dimid, varid, typid, attid; time_t starttime, endtime; long long delta; int tag; diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index a7af4d7de..06568234e 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -99,3 +99,9 @@ HDF5_PLUGIN_PATH="$FP_PLUGIN_PATH" return 0 } + +# debug +if test "x$1" != x ; then +findplugin +echo "HDF5_PLUGIN_PATH=|$FP_PLUGIN_PATH|" +fi diff --git a/nc_test4/tst_filter.sh b/nc_test4/tst_filter.sh index f695a8427..6e2201dc9 100755 --- a/nc_test4/tst_filter.sh +++ b/nc_test4/tst_filter.sh @@ -110,7 +110,8 @@ echo " *** Pass: nccopy simple filter" echo " *** Testing pass-thru of filters" rm -f ./tst_filter.txt tst_filter2.txt ./tst_filter2.nc -${NCCOPY} ./filtered.nc ./tst_filter2.nc +# Prevent failure by allowing any chunk size +${NCCOPY} -M0 ./filtered.nc ./tst_filter2.nc ${NCDUMP} -s tst_filter2.nc > ./tst_filter.txt sed -e '/_Filter/p' -e d < ./tst_filter.txt >tst_filter2.txt test -s tst_filter2.txt diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 190d4261e..67b6e67d0 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -4,7 +4,7 @@ IF(BUILD_SHARED_LIBS AND WIN32) ENDIF() SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c) -SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c) +SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) @@ -211,6 +211,11 @@ ENDIF(MSVC) SET_TESTS_PROPERTIES(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE) ENDIF(HAVE_BASH) + add_sh_test(ncdump tst_nccopy5) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(ncdump_tst_nccopy5 PROPERTIES RUN_SERIAL TRUE) + ENDIF(HAVE_BASH) + IF(USE_NETCDF4) add_sh_test(ncdump tst_ncgen4) ENDIF(USE_NETCDF4) diff --git a/ncdump/Make0 b/ncdump/Make0 index c3a594b69..1d02f46ce 100644 --- a/ncdump/Make0 +++ b/ncdump/Make0 @@ -1,12 +1,12 @@ # Test c output -T=tst_unicode +T=tst_chunking #ARGS=./x.nc #TF=test_atomic_array.cdl #CMD=valgrind --leak-check=full -CMD=gdb --args +#CMD=gdb --args #HDF4=1 #PAR=1 @@ -36,11 +36,11 @@ LLP=/usr/local/lib:${LD_LIBRARY_PATH} all:: comp export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CMD} ./t ${ARGS} + ${CMD} ./$T ${ARGS} comp:: export LD_LIBRARY_PATH=${LLP}; export CFLAGS; export LDFLAGS; \ - ${CC} -o t ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \ + ${CC} -o $T ${CFLAGS} ${T}.c ${SRC} ${LDFLAGS}; \ x.nc: x.cdl ncgen -4 x.cdl diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 0629c2087..cf3b4abdf 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -26,7 +26,7 @@ utils.c nciter.h nciter.c nccomps.h # netCDF API bin_PROGRAMS += nccopy nccopy_SOURCES = nccopy.c nciter.c nciter.h chunkspec.h chunkspec.c \ -utils.h utils.c dimmap.h dimmap.c +utils.h utils.c dimmap.h dimmap.c list.c list.h # A simple netcdf-4 metadata -> xml printer. Do not install. if USE_NETCDF4 @@ -80,14 +80,14 @@ tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar # Tests for netCDF-4 behavior. TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \ tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \ -tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \ +tst_nccopy5.sh tst_grp_spec.sh tst_mud.sh tst_h_scalar.sh tst_formatx4.sh \ run_utf8_nc4_tests.sh run_back_comp_tests.sh run_ncgen_nc4_tests.sh \ tst_ncgen4.sh -# The tst_nccopy4.sh test script depends on the output of a bunch of -# other tests. Record dependencies so parallel builds work. +# Record interscript dependencies so parallel builds work. tst_nccopy4.log: run_ncgen_tests.log tst_output.log tst_ncgen4.log \ tst_fillbug.log tst_netcdf4_4.log tst_h_scalar.log +tst_nccopy5.log: tst_nccopy4.log endif #!USE_NETCDF4 TESTS += tst_inmemory_nc3.sh @@ -111,7 +111,7 @@ ref_tst_noncoord.cdl ref_tst_compounds2.nc ref_tst_compounds2.cdl \ ref_tst_compounds3.nc ref_tst_compounds3.cdl ref_tst_compounds4.nc \ ref_tst_compounds4.cdl ref_tst_group_data_v23.cdl tst_mslp.cdl \ tst_bug321.cdl ref_tst_format_att.cdl ref_tst_format_att_64.cdl \ -tst_nccopy3.sh tst_nccopy4.sh ref_nc_test_netcdf4_4_0.nc \ +tst_nccopy3.sh tst_nccopy4.sh tst_nccopy5.sh ref_nc_test_netcdf4_4_0.nc \ run_back_comp_tests.sh ref_nc_test_netcdf4.cdl \ ref_tst_special_atts3.cdl tst_brecs.cdl ref_tst_grp_spec0.cdl \ ref_tst_grp_spec.cdl tst_grp_spec.sh ref_tst_charfill.cdl \ diff --git a/ncdump/chunkspec.c b/ncdump/chunkspec.c index c92dc48fb..62dd0506d 100644 --- a/ncdump/chunkspec.c +++ b/ncdump/chunkspec.c @@ -7,38 +7,80 @@ #include #include #include -#include +#include "netcdf.h" +#include "list.h" #include "utils.h" #include "chunkspec.h" /* Structure mapping dimension IDs to corresponding chunksizes. */ -static struct { +static struct DimChunkSpecs { size_t ndims; /* number of dimensions in chunkspec string */ - int *dimids; /* ids for dimensions in chunkspec string */ + int *idimids; /* (input) ids for dimensions in chunkspec string */ size_t *chunksizes; /* corresponding chunk sizes */ bool_t omit; /* true if chunking to be turned off */ -} chunkspecs; +} dimchunkspecs; + +struct VarChunkSpec { + size_t rank; /* number of dimensions in chunkspec string */ + size_t chunksizes[NC_MAX_VAR_DIMS]; /* corresponding chunk sizes */ + bool_t omit; /* true if chunking to be turned off */ + int igrpid; /* container of the (input) variable */ + int ivarid; /* (input) Variable whose chunks are specified */ +}; + +static List* varchunkspecs = NULL; /* List */ + +/* Forward */ +static int dimchunkspec_parse(int ncid, const char *spec); +static int varchunkspec_parse(int ncid, const char *spec); + +void +chunkspecinit(void) +{ + /* initialization */ + if(varchunkspecs == NULL) + varchunkspecs = listnew(); + memset(&dimchunkspecs,0,sizeof(dimchunkspecs)); +} /* - * Parse chunkspec string and convert into chunkspec_t structure. + * Parse chunkspec string of either kind. + * Returns NC_NOERR if no error, NC_EINVAL if spec was malformed. + */ +int +chunkspec_parse(int igrp, const char *spec) +{ + /* Decide if this is a per-variable or per-dimension chunkspec */ + if (!spec || *spec == '\0') + return NC_NOERR; /* Use defaults */ + if(strchr(spec,':') == NULL) + return dimchunkspec_parse(igrp,spec); + else + return varchunkspec_parse(igrp,spec); +} + +/* + * Parse chunkspec string and convert into dimchunkspec structure. * ncid: location ID of open netCDF file or group in an open file * spec: string of form * dim1/n1,dim2/n2,...,dimk/nk - * - * specifying chunk size (ni) to be used for dimension named - * dimi. Dimension names may be absolute, - * e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be - * omitted, in which case it is assumed to be the entire - * dimension size. That is also the default for dimensions - * not mentioned in the string. - * If the chunkspec string is "/", specifying no dimensions or - * chunk sizes, it indicates chunking to be turned off on output. + * specifying chunk size (ni) to be used for dimension named + * dimi. Dimension names may be absolute, + * e.g. "/grp_a/grp_a1/dim". The "ni" part of the spec may be + * omitted, in which case it is assumed to be the entire + * dimension size. That is also the default for dimensions + * not mentioned in the string. However, for unlimited dimensions, + * the default is a default size: 4 megabytes or the + * existing unlimited size if smaller. + * If the chunkspec string is "/", specifying no dimensions or + * chunk sizes, it indicates chunking to be turned off on output. * * Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive * unescaped commas or no chunksize specified for dimension. */ -int -chunkspec_parse(int ncid, const char *spec) { +static int +dimchunkspec_parse(int igrp, const char *spec) +{ const char *cp; /* character cursor */ const char *pp = spec; /* previous char cursor for detecting escapes */ const char *np; /* beginning of current dimension name */ @@ -47,12 +89,12 @@ chunkspec_parse(int ncid, const char *spec) { int ret; int comma_seen = 0; - chunkspecs.ndims = 0; - chunkspecs.omit = false; + dimchunkspecs.ndims = 0; + dimchunkspecs.omit = false; if (!spec || *spec == '\0') /* default chunking */ return NC_NOERR; if (spec[0] == '/' && spec[1] == '\0') { /* no chunking */ - chunkspecs.omit = true; + dimchunkspecs.omit = true; return NC_NOERR; } /* Count unescaped commas, handle consecutive unescaped commas as error */ @@ -69,9 +111,9 @@ chunkspec_parse(int ncid, const char *spec) { pp = cp; } ndims++; - chunkspecs.ndims = ndims; - chunkspecs.dimids = (int *) emalloc(ndims * sizeof(int)); - chunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t)); + dimchunkspecs.ndims = ndims; + dimchunkspecs.idimids = (int *) emalloc(ndims * sizeof(int)); + dimchunkspecs.chunksizes = (size_t *) emalloc(ndims * sizeof(size_t)); /* Look up dimension ids and assign chunksizes */ pp = spec; np = spec; @@ -97,15 +139,15 @@ chunkspec_parse(int ncid, const char *spec) { } *dp = '\0'; /* look up dimension id from dimension pathname */ - ret = nc_inq_dimid2(ncid, dimname, &dimid); + ret = nc_inq_dimid2(igrp, dimname, &dimid); if(ret != NC_NOERR) break; - chunkspecs.dimids[idim] = dimid; + dimchunkspecs.idimids[idim] = dimid; /* parse and assign corresponding chunksize */ pp++; /* now points to first digit of chunksize, ',', or '\0' */ if(*pp == ',' || *pp == '\0') { /* no size specified, use dim len */ size_t dimlen; - ret = nc_inq_dimlen(ncid, dimid, &dimlen); + ret = nc_inq_dimlen(igrp, dimid, &dimlen); if(ret != NC_NOERR) return(ret); chunksize = dimlen; @@ -120,7 +162,7 @@ chunkspec_parse(int ncid, const char *spec) { return (NC_EINVAL); chunksize = (size_t)val; } - chunkspecs.chunksizes[idim] = chunksize; + dimchunkspecs.chunksizes[idim] = chunksize; idim++; free(dimname); if(*cp == '\0') @@ -135,11 +177,11 @@ chunkspec_parse(int ncid, const char *spec) { /* Return size in chunkspec string specified for dimension corresponding to dimid, 0 if not found */ size_t -chunkspec_size(int dimid) { +dimchunkspec_size(int indimid) { int idim; - for(idim = 0; idim < chunkspecs.ndims; idim++) { - if(dimid == chunkspecs.dimids[idim]) { - return chunkspecs.chunksizes[idim]; + for(idim = 0; idim < dimchunkspecs.ndims; idim++) { + if(indimid == dimchunkspecs.idimids[idim]) { + return dimchunkspecs.chunksizes[idim]; } } return 0; @@ -149,15 +191,154 @@ chunkspec_size(int dimid) { * chunkspec string on command line, 0 if no chunkspec string was * specified. */ int -chunkspec_ndims(void) { - return chunkspecs.ndims; +dimchunkspec_ndims(void) { + return dimchunkspecs.ndims; } /* Return whether chunking should be omitted, due to explicit * command-line specification. */ bool_t -chunkspec_omit(void) { - return chunkspecs.omit; +dimchunkspec_omit(void) { + return dimchunkspecs.omit; +} + + +/* + * Parse per-variable chunkspec string and convert into varchunkspec structure. + * ncid: location ID of open netCDF file or group in an open file + * spec: string of form + * var:n1,n2,...nk + * + * specifying chunk size (ni) to be used for ith dimension of + * variable named var. Variable names may be absolute. + * e.g. "/grp_a/grp_a1/var". + * If no chunk sizes are specified, then the variable is not chunked at all. + * + * Returns NC_NOERR if no error, NC_EINVAL if spec has consecutive + * unescaped commas or no chunksize specified for dimension. + */ +static int +varchunkspec_parse(int igrp, const char *spec0) +{ + int ret = NC_NOERR; + int rank; + int i; + int dimids[NC_MAX_VAR_DIMS]; + struct VarChunkSpec* chunkspec = NULL; + char* spec = NULL; + char* p, *q; /* for walking strings */ + + /* Copy spec so we can modify in place */ + spec = strdup(spec0); + if(spec == NULL) {ret = NC_ENOMEM; goto done;} + + chunkspec = calloc(1,sizeof(struct VarChunkSpec)); + if(chunkspec == NULL) {ret = NC_ENOMEM; goto done;} + + chunkspec->igrpid = igrp; + + /* First, find the end of the variable part */ + p = strchr(spec,':'); + if(p == NULL) + {ret = NC_EINVAL; goto done;} + *p++ = '\0'; + /* Lookup the variable by name */ + ret = nc_inq_varid2(igrp, spec, &chunkspec->ivarid, &chunkspec->igrpid); + if(ret != NC_NOERR) goto done; + + /* Iterate over dimension sizes */ + while(*p) { + unsigned long dimsize; + q = strchr(p,','); + if(q == NULL) + q = p + strlen(p); /* Fake the endpoint */ + else + *q++ = '\0'; + + /* Scan as unsigned long */ + if(sscanf(p,"%lu",&dimsize) != 1) + {ret = NC_EINVAL; goto done;} /* Apparently not a valid dimension size */ + if(chunkspec->rank >= NC_MAX_VAR_DIMS) {ret = NC_EINVAL; goto done;} /* to many chunks */ + chunkspec->chunksizes[chunkspec->rank] = (size_t)dimsize; + chunkspec->rank++; + p = q; + } + /* Now do some validity checking */ + /* Get some info about the var (from input) */ + ret = nc_inq_var(chunkspec->igrpid,chunkspec->ivarid,NULL,NULL,&rank,dimids,NULL); + if(ret != NC_NOERR) goto done; + + /* 1. check # chunksizes == rank of variable */ + if(rank != chunkspec->rank) {ret = NC_EINVAL; goto done;} + + /* 2. check that chunksizes are legal for the given dimension sizes */ + for(i=0;ichunksizes[i] > len) {ret = NC_EBADCHUNK; goto done;} + } + + /* add the chunkspec to our list */ + listpush(varchunkspecs,chunkspec); + chunkspec = NULL; + +done: + if(chunkspec != NULL) + free(chunkspec); + if(spec != NULL) + free(spec); + return ret; +} + +/* Accessors */ + +bool_t +varchunkspec_exists(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return true; + } + return false; +} + +bool_t +varchunkspec_omit(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->omit; + } + return dimchunkspecs.omit; +} + +size_t* +varchunkspec_chunksizes(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->chunksizes; + } + return NULL; +} + +size_t +varchunkspec_rank(int igrpid, int ivarid) +{ + int i; + for(i=0;iigrpid == igrpid && spec->ivarid == ivarid) + return spec->rank; + } + return 0; } diff --git a/ncdump/chunkspec.h b/ncdump/chunkspec.h index e0885dc33..5c7d68d18 100644 --- a/ncdump/chunkspec.h +++ b/ncdump/chunkspec.h @@ -15,17 +15,28 @@ chunkspec_parse(int ncid, const char *spec); /* Return chunk size in chunkspec string specified for dimension * corresponding to dimid, 0 if not found */ extern size_t -chunkspec_size(int dimid); +dimchunkspec_size(int dimid); /* Return number of dimensions for which chunking was specified in * chunkspec string on command line, 0 if no chunkspec string was * specified. */ extern int -chunkspec_ndims(void); +dimchunkspec_ndims(void); /* Return whether chunking should be omitted, due to explicit * command-line specification. */ extern bool_t -chunkspec_omit(void); +dimchunkspec_omit(void); + +extern bool_t varchunkspec_omit(int grpid, int varid); + +extern size_t* varchunkspec_chunksizes(int grpid, int varid); + +extern size_t varchunkspec_ndims(int grpid, int varid); + +extern bool_t varchunkspec_exists(int grpid, int varid); + +extern void chunkspecinit(void); + #endif /* _CHUNKSPEC_H_ */ diff --git a/ncdump/list.c b/ncdump/list.c new file mode 100644 index 000000000..30570a43c --- /dev/null +++ b/ncdump/list.c @@ -0,0 +1,221 @@ +/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ +#include +#include +#include + +#include "list.h" + +int listnull(void* e) {return e == NULL;} + +#ifndef TRUE +#define TRUE 1 +#endif +#ifndef FALSE +#define FALSE 0 +#endif + +#define DEFAULTALLOC 16 +#define ALLOCINCR 16 + +List* listnew(void) +{ + List* l; +/* + if(!initialized) { + memset((void*)&DATANULL,0,sizeof(void*)); + initialized = 1; + } +*/ + l = (List*)malloc(sizeof(List)); + if(l) { + l->alloc=0; + l->length=0; + l->content=NULL; + } + return l; +} + +int +listfree(List* l) +{ + if(l) { + l->alloc = 0; + if(l->content != NULL) {free(l->content); l->content = NULL;} + free(l); + } + return TRUE; +} + +int +listsetalloc(List* l, unsigned long sz) +{ + void** newcontent = NULL; + if(l == NULL) return FALSE; + if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);} + if(l->alloc >= sz) {return TRUE;} + newcontent=(void**)calloc(sz,sizeof(void*)); + if(newcontent != NULL && l->alloc > 0 && l->length > 0 && l->content != NULL) { + memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length); + } + if(l->content != NULL) free(l->content); + l->content=newcontent; + l->alloc=sz; + return TRUE; +} + +int +listsetlength(List* l, unsigned long sz) +{ + if(l == NULL) return FALSE; + if(sz > l->alloc && !listsetalloc(l,sz)) return FALSE; + l->length = sz; + return TRUE; +} + +void* +listget(List* l, unsigned long index) +{ + if(l == NULL || l->length == 0) return NULL; + if(index >= l->length) return NULL; + return l->content[index]; +} + +int +listset(List* l, unsigned long index, void* elem) +{ + if(l == NULL) return FALSE; + if(index >= l->length) return FALSE; + l->content[index] = elem; + return TRUE; +} + +/* Insert at position i of l; will push up elements i..|seq|. */ +int +listinsert(List* l, unsigned long index, void* elem) +{ + int i; /* do not make unsigned */ + if(l == NULL) return FALSE; + if(index > l->length) return FALSE; + listsetalloc(l,0); + for(i=(int)l->length;i>index;i--) l->content[i] = l->content[i-1]; + l->content[index] = elem; + l->length++; + return TRUE; +} + +int +listpush(List* l, void* elem) +{ + if(l == NULL) return FALSE; + if(l->length >= l->alloc) listsetalloc(l,0); + l->content[l->length] = elem; + l->length++; + return TRUE; +} + +void* +listpop(List* l) +{ + if(l == NULL || l->length == 0) return NULL; + l->length--; + return l->content[l->length]; +} + +void* +listtop(List* l) +{ + if(l == NULL || l->length == 0) return NULL; + return l->content[l->length - 1]; +} + +void* +listremove(List* l, unsigned long i) +{ + unsigned long len; + void* elem; + if(l == NULL || (len=l->length) == 0) return NULL; + if(i >= len) return NULL; + elem = l->content[i]; + for(i+=1;icontent[i-1] = l->content[i]; + l->length--; + return elem; +} + +/* Duplicate and return the content (null terminate) */ +void** +listdup(List* l) +{ + void** result = (void**)malloc(sizeof(void*)*(l->length+1)); + memcpy((void*)result,(void*)l->content,sizeof(void*)*l->length); + result[l->length] = (void*)0; + return result; +} + +int +listcontains(List* l, void* elem) +{ + unsigned long i; + for(i=0;ilength) == 0) return 0; + for(i=0;icontent[i]; + if(elem == candidate) { + for(i+=1;icontent[i-1] = l->content[i]; + l->length--; + found = 1; + break; + } + } + return found; +} + + + + +/* Extends list to include a unique operator + which remove duplicate values; NULL values removed + return value is always 1. +*/ + +int +listunique(List* l) +{ + unsigned long i,j,k,len; + void** content; + if(l == NULL || l->length == 0) return 1; + len = l->length; + content = l->content; + for(i=0;ilength = len; + return 1; +} + +List* +listclone(List* l) +{ + List* clone = listnew(); + *clone = *l; + clone->content = listdup(l); + return clone; +} diff --git a/ncdump/list.h b/ncdump/list.h new file mode 100644 index 000000000..2206af37d --- /dev/null +++ b/ncdump/list.h @@ -0,0 +1,62 @@ +/* Copyright 2009, UCAR/Unidata and OPeNDAP, Inc. + See the COPYRIGHT file for more information. */ +#ifndef LIST_H +#define LIST_H 1 + +/* Define the type of the elements in the list*/ + +#if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) +#define EXTERNC extern "C" +#else +#define EXTERNC extern +#endif + +EXTERNC int listnull(void*); + +typedef struct List { + unsigned long alloc; + unsigned long length; + void** content; +} List; + +EXTERNC List* listnew(void); +EXTERNC int listfree(List*); +EXTERNC int listsetalloc(List*,unsigned long); +EXTERNC int listsetlength(List*,unsigned long); + +/* Set the ith element */ +EXTERNC int listset(List*,unsigned long,void*); +/* Get value at position i */ +EXTERNC void* listget(List*,unsigned long);/* Return the ith element of l */ +/* Insert at position i; will push up elements i..|seq|. */ +EXTERNC int listinsert(List*,unsigned long,void*); +/* Remove element at position i; will move higher elements down */ +EXTERNC void* listremove(List* l, unsigned long i); + +/* Tail operations */ +EXTERNC int listpush(List*,void*); /* Add at Tail */ +EXTERNC void* listpop(List*); +EXTERNC void* listtop(List*); + +/* Duplicate and return the content (null terminate) */ +EXTERNC void** listdup(List*); + +/* Look for value match */ +EXTERNC int listcontains(List*, void*); + +/* Remove element by value; only removes first encountered */ +EXTERNC int listelemremove(List* l, void* elem); + +/* remove duplicates */ +EXTERNC int listunique(List*); + +/* Create a clone of a list */ +EXTERNC List* listclone(List*); + +/* Following are always "in-lined"*/ +#define listclear(l) listsetlength((l),0) +#define listextend(l,len) listsetalloc((l),(len)+(l->alloc)) +#define listcontents(l) ((l)==NULL?NULL:(l)->content) +#define listlength(l) ((l)==NULL?0:(int)(l)->length) + +#endif /*LIST_H*/ diff --git a/ncdump/nccopy.1 b/ncdump/nccopy.1 index 783809319..1eda79971 100644 --- a/ncdump/nccopy.1 +++ b/ncdump/nccopy.1 @@ -352,6 +352,41 @@ memory before writing it to disk on close: .HP nccopy \-w \-c time/1000,lat/40,lon/40 slow.nc fast.nc .RE + +.SH "Chunking Rules" +.LP +The complete set of chunking rules is captured here. As a rough +summary, these rules preserve all chunking properties from the +input file. These rules apply only when the selected output +format supports chunking, i.e. for the netcdf-4 variants. +.LP +The following rules are applied in the given order independently +for each variable to be copied from input to output. The rules are +written assuming we are trying to determine the chunking for a given +output variable Vout that comes from an input variable Vin. +.NP +For each dimension of Vout explicitly specified on the command line +using the '-c' option, apply the chunking value for that +dimension. regardless of input format or input properties. + +.NP +For dimensions of V not named on the command line, preserve chunk +sizes from the corresponding input variable. +.NP +If V is netcdf-4 and contiguous, and none of its dimensions are +named on the command line, and chunking is not mandated by other +options, then make V be contiguous. +.NP +If the input variable is contiguous (or is some netcdf-3 +variant) and there are no options requiring chunking, or the '/' +special case for the '-c' option is specified, then the output +variable V is marked as contiguous. +.NP +Handle all remaining cases when some or all chunk sizes are not determined by the command line or the input variable. This includes the non-chunked input cases such as netcdf-3, cdf5, and DAP. In these cases: + Retain all chunk sizes determined by (1) and (2); and + Compute the remaining chunk sizes automatically, with some reasonable + + .SH "SEE ALSO" .LP .BR ncdump(1), ncgen(1), netcdf(3) diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 5a8901ec8..914da437b 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -23,6 +23,7 @@ #include "dimmap.h" #include "nccomps.h" #include "ncfilter.h" +#include "list.h" #undef DEBUGFILTER @@ -48,6 +49,8 @@ int optind; #define ESCAPESD "0123456789" #define ESCAPES " !\"#$%&'()*,:;<=>?[]\\^`{|}~" +#define DFALTUNLIMSIZE (4* 1<<20) /*4 megabytes */ + #ifdef USE_NETCDF4 /* The unique id for a variable requires also the enclosing group id */ @@ -73,20 +76,65 @@ static int suppressfilters = 0; /* 1 => do not apply any output filters unless s #endif +/* table of formats for legal -k values */ +static struct Kvalues { + char* name; + int kind; +} legalkinds[] = { + /* NetCDF-3 classic format (32-bit offsets) */ + {"classic", NC_FORMAT_CLASSIC}, /* canonical format name */ + {"nc3", NC_FORMAT_CLASSIC}, /* short format name */ + {"1", NC_FORMAT_CLASSIC}, /* deprecated, use "-3" or "-k nc3" instead */ + + /* NetCDF-3 64-bit offset format */ + {"64-bit offset", NC_FORMAT_64BIT_OFFSET}, /* canonical format name */ + {"nc6", NC_FORMAT_64BIT_OFFSET}, /* short format name */ + {"2", NC_FORMAT_64BIT_OFFSET}, /* deprecated, use "-6" or "-k nc6" instead */ + {"64-bit-offset", NC_FORMAT_64BIT_OFFSET}, /* deprecated alias */ + + /* NetCDF-4 HDF5-based format */ + {"netCDF-4", NC_FORMAT_NETCDF4}, /* canonical format name */ + {"nc4", NC_FORMAT_NETCDF4}, /* short format name */ + {"3", NC_FORMAT_NETCDF4}, /* deprecated, use "-4" or "-k nc4" instead */ + {"netCDF4", NC_FORMAT_NETCDF4}, /* deprecated aliases */ + {"hdf5", NC_FORMAT_NETCDF4}, + {"enhanced", NC_FORMAT_NETCDF4}, + + /* NetCDF-4 HDF5-based format, restricted to classic data model */ + {"netCDF-4 classic model", NC_FORMAT_NETCDF4_CLASSIC}, /* canonical format name */ + {"nc7", NC_FORMAT_NETCDF4_CLASSIC}, /* short format name */ + {"4", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated, use "-7" or -k nc7" */ + {"netCDF-4-classic", NC_FORMAT_NETCDF4_CLASSIC}, /* deprecated aliases */ + {"netCDF-4_classic", NC_FORMAT_NETCDF4_CLASSIC}, + {"netCDF4_classic", NC_FORMAT_NETCDF4_CLASSIC}, + {"hdf5-nc3", NC_FORMAT_NETCDF4_CLASSIC}, + {"enhanced-nc3", NC_FORMAT_NETCDF4_CLASSIC}, + + /* The 64-bit data (CDF5) kind (5) */ + {"5", NC_FORMAT_CDF5}, + {"64-bit-data", NC_FORMAT_CDF5}, + {"64-bit data", NC_FORMAT_CDF5}, + {"nc5", NC_FORMAT_CDF5}, + {"cdf5", NC_FORMAT_CDF5}, + + /* null terminate*/ + {NULL,0} +}; + /* Global variables for command-line requests */ char *progname; /* for error messages */ static int option_kind = SAME_AS_INPUT; static int option_deflate_level = -1; /* default, compress output only if input compressed */ static int option_shuffle_vars = NC_NOSHUFFLE; /* default, no shuffling on compression */ static int option_fix_unlimdims = 0; /* default, preserve unlimited dimensions */ -static char* option_chunkspec = 0; /* default, no chunk specification */ +static List* option_chunkspecs = NULL; /* default, no chunk specification */ static size_t option_copy_buffer_size = COPY_BUFFER_SIZE; static size_t option_chunk_cache_size = CHUNK_CACHE_SIZE; /* default from config.h */ static size_t option_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /* default from config.h */ static int option_read_diskless = 0; /* default, don't read input into memory on open */ static int option_write_diskless = 0; /* default, don't write output to diskless file */ #ifdef USE_NETCDF4 -static int option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of +static size_t option_min_chunk_bytes = CHUNK_THRESHOLD; /* default, don't chunk variable if prod of * chunksizes of its dimensions is smaller * than this */ #endif @@ -97,9 +145,9 @@ static char** option_lgrps = 0; /* list of group names specified with -g static idnode_t* option_grpids = 0; /* list of grpids matching list specified with -g option */ static bool_t option_grpstruct = false; /* if -g set, copy structure for non-selected groups */ static int option_nlvars = 0; /* Number of variables specified with -v * option on command line */ -static char** option_lvars = 0; /* list of variable names specified with -v - * option on command line */ -static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */ +static char** option_lvars = 0; /* list of variable names specified with -v + * option on command line */ +static bool_t option_varstruct = false; /* if -v set, copy structure for non-selected vars */ static int option_compute_chunkcaches = 0; /* default, don't try still flaky estimate of * chunk cache for each variable */ @@ -186,7 +234,7 @@ computeFQN(VarID vid, char** fqnp) q = escname; for(first=1;*p;first=0) { if((first && strchr(ESCAPESD,*p) != NULL) - || strchr(ESCAPES,*p) != NULL) *q++ = '\\'; + || strchr(ESCAPES,*p) != NULL) *q++ = '\\'; *q++ = *p++; } *q++ = '\0'; /* guarantee */ @@ -196,44 +244,6 @@ done: return stat; } -#if 0 -static int -parseFQN(int ncid, const char* fqn0, VarID* idp) -{ - int stat = NC_NOERR; - char* fqn; - VarID vid; - char* p; - char* q; - char* segment; - - vid.grpid = ncid; - if(fqn0 == NULL || fqn0[1] != '/') - {stat = NC_EBADNAME; goto done;} - fqn = strdup(fqn0+1); /* skip leading '/'*/ - p = fqn; - for(;;) { - int newgrp; - segment = p; - q = p; - while(*p != '\0' && *p != '/') { - if(*p == '\\') p++; - *q++ = *p++; - } - if(*p == '\0') break; - *p++ = '\0'; - if((stat=nc_inq_grp_ncid(vid.grpid,segment,&newgrp))) goto done; - vid.grpid = newgrp; - } - /* Segment should point to the varname */ - if((stat=nc_inq_varid(vid.grpid,segment,&vid.varid))) goto done; -done: - if(fqn) free(fqn); - if(stat == NC_NOERR && idp != NULL) *idp = vid; - return stat; -} -#endif - static int parsefilterspec(const char* optarg0, struct FilterSpec* spec) { @@ -266,9 +276,9 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec) if(optarg[0]=='/') spec->fqn = strdup(optarg); else { - spec->fqn = (char*)malloc(1+strlen(optarg)+1); + spec->fqn = (char*)malloc(1+strlen(optarg)+1); strcpy(spec->fqn,"/"); - strcat(spec->fqn,optarg); + strcat(spec->fqn,optarg); } /* Check for special cases */ @@ -280,10 +290,10 @@ parsefilterspec(const char* optarg0, struct FilterSpec* spec) /* Collect the id+parameters */ if((stat = NC_parsefilterspec(remainder,&id,&nparams,¶ms)) == NC_NOERR) { - if(spec != NULL) { - spec->filterid = id; - spec->nparams = nparams; - spec->params = params; + if(spec != NULL) { + spec->filterid = id; + spec->nparams = nparams; + spec->params = params; } } @@ -336,7 +346,7 @@ static int inq_var_chunking_params(int igrp, int ivarid, int ogrp, int ovarid, size_t* chunkcache_sizep, size_t *chunkcache_nelemsp, - float * chunkcache_preemptionp) + float * chunkcache_preemptionp) { int stat = NC_NOERR; int ndims; @@ -583,7 +593,7 @@ copy_groups(int iroot, int oroot) /* get full group name of input group */ NC_CHECK(nc_inq_grpname(grpids[i], grpname)); if (option_grpstruct || group_wanted(grpids[i], option_nlgrps, option_grpids)) { - NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL)); + NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, NULL)); grpname_full = emalloc(len_name + 1); NC_CHECK(nc_inq_grpname_full(grpids[i], &len_name, grpname_full)); /* Make sure, the parent group is also wanted (root group is always wanted) */ @@ -653,90 +663,10 @@ copy_types(int igrp, int ogrp) return stat; } -/* Copy all netCDF-4 specific variable properties such as chunking, - * endianness, deflation, checksumming, fill, etc. */ -static int -copy_var_specials(int igrp, int varid, int ogrp, int o_varid) -{ - int stat = NC_NOERR; - { /* handle chunking parameters */ - int ndims; - NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); - if (ndims > 0) { /* no chunking for scalar variables */ - int contig = 0; - size_t *chunkp = (size_t *) emalloc(ndims * sizeof(size_t)); - int *dimids = (int *) emalloc(ndims * sizeof(int)); - int idim; - /* size of a chunk: product of dimension chunksizes and size of value */ - size_t csprod = val_size(ogrp, o_varid); - int is_unlimited = 0; - NC_CHECK(nc_inq_var_chunking(igrp, varid, &contig, chunkp)); - NC_CHECK(nc_inq_vardimid(igrp, varid, dimids)); - - for(idim = 0; idim < ndims; idim++) { - int idimid = dimids[idim]; - int odimid = dimmap_odimid(idimid); - size_t chunksize = chunkspec_size(idimid); - if(chunksize > 0) { /* found in chunkspec */ - chunkp[idim] = chunksize; - } - csprod *= chunkp[idim]; - if(dimmap_ounlim(odimid)) - is_unlimited = 1; - } - /* Explicitly set chunking, even if default */ - /* If product of chunksizes is too small and no unlimited - * dimensions used, don't chunk. Also if chunking - * explicitly turned off with chunk spec, don't chunk. */ - if ((csprod < option_min_chunk_bytes && !is_unlimited) || contig == 1 - || chunkspec_omit() == true) { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); - } else { - NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, chunkp)); - } - free(dimids); - free(chunkp); - } - } - { /* handle compression parameters, copying from input, overriding - * with command-line options */ - int shuffle_in=0, deflate_in=0, deflate_level_in=0; - int shuffle_out=0, deflate_out=0, deflate_level_out=0; - if(option_deflate_level != 0) { - NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in)); - if(option_deflate_level == -1) { /* not specified, copy input compression and shuffling */ - shuffle_out = shuffle_in; - deflate_out = deflate_in; - deflate_level_out = deflate_level_in; - } else if(option_deflate_level > 0) { /* change to specified compression, shuffling */ - shuffle_out = option_shuffle_vars; - deflate_out=1; - deflate_level_out = option_deflate_level; - } - NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out)); - } - } - { /* handle checksum parameters */ - int fletcher32 = 0; - NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32)); - if(fletcher32 != 0) { - NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32)); - } - } - { /* handle endianness */ - int endianness = 0; - NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness)); - if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */ - NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness)); - } - } - return stat; -} - /* Copy netCDF-4 specific variable filter properties */ /* Watch out if input is netcdf-3 */ static int -copy_var_filter(int igrp, int varid, int ogrp, int o_varid) +copy_var_filter(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind) { int stat = NC_NOERR; VarID vid = {igrp,varid}; @@ -747,14 +677,11 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid) actualspec = {NULL,0,0,0,NULL}; int i; char* ofqn = NULL; - int format, oformat; int inputdefined, outputdefined, unfiltered; + int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); + int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); - /* Get file format of the input and output */ - if((stat=nc_inq_format(vid.grpid,&format))) goto done; - if((stat=nc_inq_format(ovid.grpid,&oformat))) goto done; - - if(oformat != NC_FORMAT_NETCDF4 && oformat != NC_FORMAT_NETCDF4_CLASSIC) + if(!outnc4) goto done; /* Can only use filter when output is some netcdf4 variant */ /* Compute the output vid's FQN */ @@ -768,7 +695,7 @@ copy_var_filter(int igrp, int varid, int ogrp, int o_varid) /* Is there a filter on the output variable */ outputdefined = 0; /* default is no filter defined */ /* Only bother to look if output is netcdf-4 variant */ - if(oformat == NC_FORMAT_NETCDF4 || oformat == NC_FORMAT_NETCDF4_CLASSIC) { + if(outnc4) { /* See if any output filter spec is defined for this output variable */ for(i=0;i 0) { /* found in chunkspec */ + ochunkp[idim] = chunksize; + ocontig = 0; /* cannot use contiguous */ + goto next; + } + + /* Not specified in -c; Apply defaulting rules as defined in nccopy.1 */ + + /* If input is chunked, then use that chunk size */ + if(!icontig) { + ochunkp[idim] = ichunkp[idim]; + ocontig = 0; + goto next; + } + + /* If input is netcdf-4 then use the input size as the chunk size; + but do not force chunking. + */ + if(!innc4) { + ochunkp[idim] = dimlen; + goto next; + } + + /* Default for unlimited is max(4 megabytes, current dim size) */ + if(is_unlimited) { + size_t mb4dimsize = DFALTUNLIMSIZE / typesize; + ochunkp[idim] = (dimlen > mb4dimsize ? mb4dimsize : dimlen); + } else { + /* final default is the current dimension size */ + ochunkp[idim] = dimlen; + } + +next: + /* compute on-going dimension product */ + csprod *= ochunkp[idim]; + } + + /* Finally, if total chunksize is too small (and dim is not unlimited) => do not chunk */ + if(csprod < option_min_chunk_bytes && !is_unlimited) { + ocontig = 1; /* Force contiguous */ + } + + /* Apply the chunking, if any */ + + if(ocontig) { /* We can use contiguous output */ + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CONTIGUOUS, NULL)); + } else { + NC_CHECK(nc_def_var_chunking(ogrp, o_varid, NC_CHUNKED, ochunkp)); + } + } +done: + return stat; +} + +/* Copy all netCDF-4 specific variable properties such as chunking, + * endianness, deflation, checksumming, fill, etc. */ +static int +copy_var_specials(int igrp, int varid, int ogrp, int o_varid, int inkind, int outkind) +{ + int stat = NC_NOERR; + int innc4 = (inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC); + int outnc4 = (outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC); + + if(!outnc4) + return stat; /* Ignore non-netcdf4 files */ + + { /* handle chunking parameters */ + int ndims; + NC_CHECK(nc_inq_varndims(igrp, varid, &ndims)); + if (ndims > 0) { /* no chunking for scalar variables */ + NC_CHECK(copy_chunking(igrp, varid, ogrp, o_varid, ndims, inkind, outkind)); + } + } + + { /* handle compression parameters, copying from input, overriding + * with command-line options */ + int shuffle_in=0, deflate_in=0, deflate_level_in=0; + int shuffle_out=0, deflate_out=0, deflate_level_out=0; + if(innc4) { /* See if the input variable has deflation applied */ + NC_CHECK(nc_inq_var_deflate(igrp, varid, &shuffle_in, &deflate_in, &deflate_level_in)); + } + if(option_deflate_level == -1) { + /* not specified by -d flag, copy input compression and shuffling */ + shuffle_out = shuffle_in; + deflate_out = deflate_in; + deflate_level_out = deflate_level_in; + } else if(option_deflate_level > 0) { /* change to specified compression, shuffling */ + shuffle_out = option_shuffle_vars; + deflate_out=1; + deflate_level_out = option_deflate_level; + } else if(option_deflate_level == 0) { /* special case; force off */ + shuffle_out = 0; + deflate_out = 0; + deflate_level_out = 0; + } + /* Apply output deflation */ + if(outnc4) { + /* Note that if we invoke this function and even if shuffle and deflate flags are 0, + then default chunking will be turned on; so do a special check for that. */ + if(shuffle_out != 0 || deflate_out != 0) + NC_CHECK(nc_def_var_deflate(ogrp, o_varid, shuffle_out, deflate_out, deflate_level_out)); + } + } + + if(innc4 && outnc4) + { /* handle checksum parameters */ + int fletcher32 = 0; + NC_CHECK(nc_inq_var_fletcher32(igrp, varid, &fletcher32)); + if(fletcher32 != 0) { + NC_CHECK(nc_def_var_fletcher32(ogrp, o_varid, fletcher32)); + } + } + + if(innc4 && outnc4) + { /* handle endianness */ + int endianness = 0; + NC_CHECK(nc_inq_var_endian(igrp, varid, &endianness)); + if(endianness != NC_ENDIAN_NATIVE) { /* native is the default */ + NC_CHECK(nc_def_var_endian(ogrp, o_varid, endianness)); + } + } + + /* handle other general filters */ + NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid, inkind, outkind)); + + return stat; +} + +#if 0 +Subsumed into copy_chunking. /* Set output variable o_varid (in group ogrp) to use chunking * specified on command line, only called for classic format input and * netCDF-4 format output, so no existing chunk lengths to override. */ @@ -850,7 +989,7 @@ set_var_chunked(int ogrp, int o_varid) int odim; size_t chunk_threshold = CHUNK_THRESHOLD; - if(chunkspec_ndims() == 0) /* no chunking specified on command line */ + if(dimchunkspec_ndims() == 0) /* no chunking specified on command line */ return stat; NC_CHECK(nc_inq_varndims(ogrp, o_varid, &ndims)); @@ -878,7 +1017,7 @@ set_var_chunked(int ogrp, int o_varid) if(dimmap_ounlim(odimid)) is_unlimited = 1; /* whether vriable is unlimited */ if(idimid != -1) { - size_t chunksize = chunkspec_size(idimid); /* from chunkspec */ + size_t chunksize = dimchunkspec_size(idimid); /* from chunkspec */ size_t dimlen; NC_CHECK(nc_inq_dimlen(ogrp, odimid, &dimlen)); if( (chunksize > 0) || dimmap_ounlim(odimid)) { @@ -901,7 +1040,7 @@ set_var_chunked(int ogrp, int o_varid) for(odim = 0; odim < ndims; odim++) { int odimid = dimids[odim]; int idimid = dimmap_idimid(odimid); - size_t chunksize = chunkspec_size(idimid); + size_t chunksize = dimchunkspec_size(idimid); if(chunksize > 0) { chunkp[odim] = chunksize; } else { @@ -919,7 +1058,9 @@ set_var_chunked(int ogrp, int o_varid) } return stat; } +#endif +#if 0 /* Set variable to compression specified on command line */ static int set_var_compressed(int ogrp, int o_varid) @@ -931,6 +1072,7 @@ set_var_compressed(int ogrp, int o_varid) } return stat; } +#endif /* Release the variable chunk cache allocated for variable varid in * group grp. This is not necessary, but will save some memory when @@ -1110,19 +1252,11 @@ copy_var(int igrp, int varid, int ogrp) int outkind; NC_CHECK(nc_inq_format(igrp, &inkind)); NC_CHECK(nc_inq_format(ogrp, &outkind)); - if(outkind == NC_FORMAT_NETCDF4 || outkind == NC_FORMAT_NETCDF4_CLASSIC) { - if((inkind == NC_FORMAT_NETCDF4 || inkind == NC_FORMAT_NETCDF4_CLASSIC)) { - /* Copy all netCDF-4 specific variable properties such as - * chunking, endianness, deflation, checksumming, fill, etc. */ - NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid)); - } else { - /* Set chunking if specified in command line option */ - NC_CHECK(set_var_chunked(ogrp, o_varid)); - /* Set compression if specified in command line option */ - NC_CHECK(set_var_compressed(ogrp, o_varid)); - } - NC_CHECK(copy_var_filter(igrp, varid, ogrp, o_varid)); - } + /* Copy all variable properties such as + * chunking, endianness, deflation, checksumming, fill, etc. + * Ok to call if outkind is netcdf-3 + */ + NC_CHECK(copy_var_specials(igrp, varid, ogrp, o_varid, inkind, outkind)); } #endif /* USE_NETCDF4 */ free(idimids); @@ -1667,7 +1801,7 @@ copy(char* infile, char* outfile) || inkind == NC_FORMAT_CDF5) { if (option_deflate_level > 0 || option_shuffle_vars == NC_SHUFFLE || - option_chunkspec) + listlength(option_chunkspecs) > 0) { outkind = NC_FORMAT_NETCDF4_CLASSIC; } @@ -1675,10 +1809,14 @@ copy(char* infile, char* outfile) } #ifdef USE_NETCDF4 - if(option_chunkspec) { - /* Now that input is open, can parse option_chunkspec into binary + if(listlength(option_chunkspecs) > 0) { + int i; + /* Now that input is open, can parse option_chunkspecs into binary * structure. */ - NC_CHECK(chunkspec_parse(igrp, option_chunkspec)); + for(i=0;i= 0) - nc_set_log_level(level); - } + { + int level = atoi(optarg); + if(level >= 0) + nc_set_log_level(level); + } #endif - break; + break; case 'F': /* optional filter spec for a specified variable */ #ifdef USE_NETCDF4 /* If the arg is "none" then suppress all filters @@ -2058,7 +2154,16 @@ main(int argc, char**argv) #else error("-F requires netcdf-4"); #endif + case 'M': /* set min chunk size */ +#ifdef USE_NETCDF4 + option_min_chunk_bytes = atol(optarg); + if(option_min_chunk_bytes < 0) + error("-M value must be non-negative integer"); break; +#else + error("-M requires netcdf-4"); +#endif + default: usage(); } diff --git a/ncdump/tst_chunking.c b/ncdump/tst_chunking.c index 17e59100c..589f1c683 100644 --- a/ncdump/tst_chunking.c +++ b/ncdump/tst_chunking.c @@ -3,49 +3,135 @@ conditions of use. See www.unidata.ucar.edu for more info. Create a chunkable test file for nccopy to test chunking. - - $Id$ */ #include #include #include "err_macros.h" +#define DEBUG + +static int ret = NC_NOERR; + +/* Make trackable ERR macro replacement */ +static int lerr(int stat, const char* file, int lineno) { + fflush(stdout); /* Make sure our stdout is synced with stderr. */ + err++; + fprintf(stderr, "Sorry! Unexpected result(%d), %s, line: %d\n",ret,file,lineno); + fflush(stderr); \ + return 2; \ +} +#define LERR lerr(ret,__FILE__,__LINE__) + #define FILE_NAME "tst_chunking.nc" #define VAR_RANK 7 #define IVAR_NAME "ivar" #define FVAR_NAME "fvar" +#define GRP_NAME "g" +#define UNLIM_NAME "unlimited" +#define UNLIM_SIZE 10 +#define DEFLATE_LEVEL 1 #define NVALS 45360 /* 7 * 4 * 2 * 3 * 5 * 6 * 9 */ +static const char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; +static const size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; + int -main(int argc, char **argv) { - int ncid; +main(int argc, char **argv) +{ + /* mutually exclusive command line options */ + int option_group = 0; + int option_deflate = 0; + int option_unlimited = 0; + /* file metadata */ + int mode = NC_CLOBBER; + int ncid, grpid; int ivarid, fvarid; - char *dim_names[VAR_RANK] = {"dim0", "dim1", "dim2", "dim3", "dim4", "dim5", "dim6"}; - size_t dim_lens[VAR_RANK] = {7, 4, 2, 3, 5, 6, 9}; int ivar_dims[VAR_RANK]; int fvar_dims[VAR_RANK]; int ivar_data[NVALS]; float fvar_data[NVALS]; int r, i; + char* file_name = FILE_NAME; + int unlimid; - printf("*** Creating chunkable test file %s...", FILE_NAME); - - if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR; - for(r = 0; r < VAR_RANK; r++) { - if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) ERR; - fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; - } - if (nc_def_var(ncid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) ERR; - if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) ERR; - if (nc_enddef (ncid)) ERR; - for(i=0; i < NVALS; i++) { - ivar_data[i] = i; - fvar_data[i] = NVALS - i; + /* Parse command line */ + if(argc >= 2) { + file_name = argv[1]; } - if (nc_put_var(ncid, ivarid, ivar_data)) ERR; - if (nc_put_var(ncid, fvarid, fvar_data)) ERR; - if (nc_close(ncid)) ERR; + if(argc >= 3) { + if(strcmp(argv[2],"group")==0) { + option_group = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"deflate")==0) { + option_deflate = 1; + mode |= NC_NETCDF4; + } else if(strcmp(argv[2],"unlimited")==0) { + option_unlimited = 1; + } else { + fprintf(stderr,"usage: tst_chunking [ [group|deflate|unlimited]]\n"); + exit(1); + } + } + + printf("*** Creating chunkable test file %s...\n", file_name); + if(option_deflate) + printf("\toption: deflate\n"); + else if(option_unlimited) + printf("\toption: unlimited\n"); + else if(option_group) + printf("\toption: group\n"); + + if (nc_create(file_name, mode, &ncid)) LERR; + for(r = 0; r < VAR_RANK; r++) { + if (nc_def_dim(ncid, dim_names[r], dim_lens[r], &ivar_dims[r])) LERR; + fvar_dims[VAR_RANK - 1 - r] = ivar_dims[r]; + } + if(option_unlimited) { + int udims[2]; + if (nc_def_dim(ncid, UNLIM_NAME, 0, &unlimid)) LERR; + udims[0] = unlimid; + udims[1] = ivar_dims[0]; + if (nc_def_var(ncid, IVAR_NAME, NC_INT, 2, udims, &ivarid)) LERR; + } else { + if (option_group) { + if (nc_def_grp(ncid, GRP_NAME, &grpid)) LERR; + } else + grpid = ncid; + if (nc_def_var(grpid, IVAR_NAME, NC_INT, VAR_RANK, ivar_dims, &ivarid)) LERR; + if(option_deflate) { + if(nc_def_var_deflate(grpid,ivarid,NC_NOSHUFFLE, option_deflate, DEFLATE_LEVEL)) LERR; + } + } + /* fvar is unchanged */ + if (nc_def_var(ncid, FVAR_NAME, NC_FLOAT, VAR_RANK, fvar_dims, &fvarid)) LERR; + if (nc_enddef (ncid)) LERR; + + /* Fill in the data */ + if(option_unlimited) { + int nvals = UNLIM_SIZE * dim_lens[0]; + size_t start[2] = {0,0}; + size_t count[2]; + for(i=0;i +# usage: trim +# Leaves result in variable TRIMMED +trim() { + # trim leading whitespace and remove + TMP=`echo "$1" |tr -d '\r' | sed -e 's/^[ ]*//'` + # trim trailing whitespace + TRIMMED=`echo "$TMP" | sed -e 's/[ ]*$//'` +} + +# usage: checkfvar +checkfvar() { + # Make sure that fvar was not chunked + C5FVAR=`sed -e '/fvar:_ChunkSizes/p' -e d <$1` + if test "x$C5FVAR" != x ; then + echo "***Fail: fvar was chunked" + exit 1 + fi +} + +# usage: verifychunkline line1 line2 +verifychunkline() { + # trim leading whitespace + trim "$1"; L1="$TRIMMED" + trim "$2"; L2="$TRIMMED" + if test "x$L1" != "x$L2" ; then + echo "chunk line mismatch |$L1| |$L2|" + exit 1; + fi +} + +# Remove any temporary files +cleanup() { + rm -f tmp_nc5.nc tmp_nc5a.nc + rm -f tmp_nc5.cdl tmp_nc5a.cdl tmp_nc5b.cdl +} + +# remove all created files +reset() { + cleanup + rm -fr tst_nc5.nc tst_nc5.cdl +} + +reset + +if test "x$T1" = x1 ; then + +# Create a simple classic input file +./tst_chunking tst_nc5.nc + +# Save a .cdl version +${NCDUMP} tst_nc5.nc > tst_nc5.cdl + +echo "*** Test nccopy -c with per-variable chunking; classic->enhanced" +# This should produce same as -c dim0/,dim1/1,dim2/,dim3/1,dim4/,dim5/1,dim6/ +${NCCOPY} -c ivar:7,1,2,1,5,1,9 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# Verify that the core cdl is the same +diff tst_nc5.cdl tmp_nc5.cdl + +# Look at the output chunking of ivar +rm -f tmp_nc5a.cdl # reuse +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc deflate +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Look at the output chunking +rm -f tmp_nc5.cdl # reuse +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc group +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c /g/ivar:4,1,2,1,5,2,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Verify chunking +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d enhanced" +reset +./tst_chunking tst_nc5.nc unlimited +${NCDUMP} -n tst_nc5 tst_nc5.nc > tst_nc5.cdl +${NCCOPY} -c ivar:5,3 tst_nc5.nc tmp_nc5.nc +${NCDUMP} -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +diff tst_nc5.cdl tmp_nc5.cdl + +# Verify chunking +${NCDUMP} -hs -n tst_nc5 tmp_nc5.nc > tmp_nc5.cdl +# extract the chunking line +TESTLINE=`sed -e '/ivar:_ChunkSizes/p' -e d Date: Sun, 29 Jul 2018 14:53:36 -0500 Subject: [PATCH 08/16] introduce error code NC_EPNETCDF for errors at PnetCDF level --- docs/all-error-codes.md | 1 + include/netcdf.h | 1 + libdispatch/derror.c | 2 + libsrcp/ncpdispatch.c | 96 ++++++++++++++++------------------------- nc_test/util.c | 7 ++- 5 files changed, 46 insertions(+), 61 deletions(-) diff --git a/docs/all-error-codes.md b/docs/all-error-codes.md index b5b0d5864..78829c2ab 100644 --- a/docs/all-error-codes.md +++ b/docs/all-error-codes.md @@ -71,6 +71,7 @@ may occur. #define NC_ENOTFOUND (-90) // No such file #define NC_ECANTREMOVE (-91) // Cannot remove file #define NC_EINTERNAL (-92) // NetCDF Library Internal Error +#define NC_EPNETCDF (-93) // Error at PnetCDF layer ~~~~ # NetCDF-4 Error Codes {#nc4-error-codes} diff --git a/include/netcdf.h b/include/netcdf.h index 58c789e05..a4b62e009 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -426,6 +426,7 @@ by the desired type. */ #define NC_ENOTFOUND (-90) /**< No such file */ #define NC_ECANTREMOVE (-91) /**< Can't remove file */ #define NC_EINTERNAL (-92) /**< NetCDF Library Internal Error */ +#define NC_EPNETCDF (-93) /**< Error at PnetCDF layer */ /* The following was added in support of netcdf-4. Make all netcdf-4 error codes < -100 so that errors can be added to netcdf-3 if diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 4ee2b93b7..6a2b83fc0 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -196,6 +196,8 @@ const char *nc_strerror(int ncerr1) return "NetCDF: cannot delete file"; case NC_EINTERNAL: return "NetCDF: internal library error; Please contact Unidata support"; + case NC_EPNETCDF: + return "NetCDF: PnetCDF error"; case NC_EHDFERR: return "NetCDF: HDF error"; case NC_ECANTREAD: diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index d651abc4f..cc5e8eea1 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -15,14 +15,9 @@ typedef struct NCP_INFO { - /* pnetcdf_file will be true if the file is created/opened with the - * parallel-netcdf library. pnetcdf_access_mode keeps track of - * whether independpent or collective mode is - * desired. pnetcdf_ndims keeps track of how many dims each var - * has, which I need to know to convert start, count, and stride - * arrays from size_t to MPI_Offset. (I can't use an inq function - * to find out the number of dims, because these are collective in - * pnetcdf.) */ + /* pnetcdf_access_mode keeps track of whether independpent or collective + * mode is set currently. + */ int pnetcdf_access_mode; } NCP_INFO; @@ -46,8 +41,8 @@ NCP_create(const char *path, int cmode, { int res, default_format; NCP_INFO* nc5; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; + MPI_Comm comm; + MPI_Info info; /* Check the cmode for only valid flags*/ if(cmode & ~LEGAL_CREATE_FLAGS) @@ -88,23 +83,9 @@ NCP_create(const char *path, int cmode, /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); - /* Fix up the cmode by keeping only essential flags; - these are the flags that are the same in netcf.h and pnetcdf.h - */ - /* It turns out that pnetcdf.h defines a flag called - NC_64BIT_DATA (not to be confused with NC_64BIT_OFFSET). - This flag is essential to getting ncmpi_create to create - a proper pnetcdf format file. - We have set the value of NC_64BIT_DATA to be the same as in pnetcdf.h - (as of pnetcdf version 1.6.0) to avoid conflicts. - In any case, this flag must be set. - */ - /* PnetCDF recognizes the flags below for create and ignores NC_LOCK and NC_SHARE */ - cmode &= (NC_WRITE | NC_NOCLOBBER | NC_SHARE | NC_64BIT_OFFSET | NC_64BIT_DATA); - res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); - if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ + if (res != NC_NOERR) free(nc5); /* reclaim allocated space */ done: return res; } @@ -117,36 +98,19 @@ NCP_open(const char *path, int cmode, { int res; NCP_INFO* nc5; - MPI_Comm comm = MPI_COMM_WORLD; - MPI_Info info = MPI_INFO_NULL; + MPI_Comm comm; + MPI_Info info; /* Check the cmode for only valid flags*/ if(cmode & ~LEGAL_OPEN_FLAGS) {res = NC_EINVAL; goto done;} - /* Cannot have both MPIO flags */ - if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX)) - {res = NC_EINVAL; goto done;} + /* No MPI environment initialized */ + if (mpidata == NULL) + {res = NC_ENOPAR; goto done;} - /* Appears that this comment is wrong; allow 64 bit offset*/ - /* Cannot have 64 bit offset flag */ - /* if(cmode & (NC_64BIT_OFFSET)) {res = NC_EINVAL; goto done;} */ - if(mpidata != NULL) { - comm = ((NC_MPI_INFO *)mpidata)->comm; - info = ((NC_MPI_INFO *)mpidata)->info; - } else { - comm = MPI_COMM_WORLD; - info = MPI_INFO_NULL; - } - - /* PnetCDF recognizes the flags NC_WRITE and NC_NOCLOBBER for file open - * and ignores NC_LOCK, NC_SHARE, NC_64BIT_OFFSET, and NC_64BIT_DATA. - * Ignoring the NC_64BIT_OFFSET and NC_64BIT_DATA flags is because the - * file is already in one of the CDF-formats, and setting these 2 flags - * will not change the format of that file. - */ - - cmode &= (NC_WRITE | NC_NOCLOBBER); + comm = ((NC_MPI_INFO *)mpidata)->comm; + info = ((NC_MPI_INFO *)mpidata)->info; /* Create our specific NCP_INFO instance */ nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); @@ -158,7 +122,7 @@ NCP_open(const char *path, int cmode, res = ncmpi_open(comm, path, cmode, info, &(nc->int_ncid)); /* Default to independent access, like netCDF-4/HDF5 files. */ - if(!res) { + if (res == NC_NOERR) { res = ncmpi_begin_indep_data(nc->int_ncid); nc5->pnetcdf_access_mode = NC_INDEPENDENT; } @@ -193,9 +157,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t nc5 = NCP_DATA(nc); assert(nc5); - /* causes implicitly defined warning; may be because of old installed pnetcdf? */ -#if 1 - /* In PnetCDF ncmpi__enddef() is only implemented in v1.5.0 and later */ +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10500) + /* ncmpi__enddef() was first implemented in PnetCDF v1.5.0 */ status = ncmpi__enddef(nc->int_ncid, mpi_h_minfree, mpi_v_align, mpi_v_minfree, mpi_r_align); #else @@ -257,7 +220,12 @@ NCP_set_fill(int ncid, int fillmode, int *old_mode_ptr) NC* nc; int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_set_fill was first implemented in PnetCDF 1.6.1 */ return ncmpi_set_fill(nc->int_ncid,fillmode,old_mode_ptr); +#else + return NC_EPNETCDF; +#endif } static int @@ -314,7 +282,7 @@ NCP_inq_type(int ncid, nc_type typeid, char* name, size_t* size) { /* Assert mode & NC_FORMAT_CDF5 */ if (typeid < NC_BYTE || typeid >= NC_STRING) - return NC_EBADTYPE; + return NC_EBADTYPE; if(name) strcpy(name, NC_atomictypename(typeid)); if(size) @@ -440,15 +408,14 @@ NCP_get_att( { NC* nc; int status; - nc_type xtype; status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; - status = NCP_inq_att(ncid,varid,name,&xtype,NULL); - if(status != NC_NOERR) return status; - - if(memtype == NC_NAT) memtype = xtype; + if (memtype == NC_NAT) { + status = NCP_inq_att(ncid,varid,name,&memtype,NULL); + if (status != NC_NOERR) return status; + } switch (memtype) { case NC_CHAR: @@ -1171,7 +1138,13 @@ NCP_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, if(deflatep) *deflatep = 0; if(fletcher32p) *fletcher32p = 0; if(contiguousp) *contiguousp = NC_CONTIGUOUS; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_inq_var_fill was first implemented in PnetCDF 1.6.1 */ if(no_fill) ncmpi_inq_var_fill(nc->int_ncid, varid, no_fill, fill_valuep); +#else + /* PnetCDF 1.6.0 and priors support NC_NOFILL only */ + if(no_fill) *no_fill = 1; +#endif if(endiannessp) return NC_ENOTNC4; if(idp) return NC_ENOTNC4; if(nparamsp) return NC_ENOTNC4; @@ -1185,7 +1158,12 @@ NCP_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value) NC* nc; int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; +#if (PNETCDF_VERSION_MAJOR*10000 + PNETCDF_VERSION_MINOR*100 + PNETCDF_VERSION_SUB >= 10601) + /* ncmpi_def_var_fill was first implemented in PnetCDF 1.6.1 */ return ncmpi_def_var_fill(nc->int_ncid, varid, no_fill, fill_value); +#else + return NC_EPNETCDF; +#endif } static int diff --git a/nc_test/util.c b/nc_test/util.c index 811f7eacb..005c76081 100644 --- a/nc_test/util.c +++ b/nc_test/util.c @@ -1284,6 +1284,8 @@ char* nc_err_code_name(int err) case (NC_EAUTH): return "NC_EAUTH"; case (NC_ENOTFOUND): return "NC_ENOTFOUND"; case (NC_ECANTREMOVE): return "NC_ECANTREMOVE"; + case (NC_EINTERNAL): return "NC_EINTERNAL"; + case (NC_EPNETCDF): return "NC_EPNETCDF"; case (NC_EHDFERR): return "NC_EHDFERR"; case (NC_ECANTREAD): return "NC_ECANTREAD"; case (NC_ECANTWRITE): return "NC_ECANTWRITE"; @@ -1315,8 +1317,9 @@ char* nc_err_code_name(int err) case (NC_EDISKLESS): return "NC_EDISKLESS"; case (NC_ECANTEXTEND): return "NC_ECANTEXTEND"; case (NC_EMPI): return "NC_EMPI"; - case (NC_ENULLPAD): return "NC_NULLPAD"; - // case (NC_EURL): return "NC_EURL"; + case (NC_ENULLPAD): return "NC_NULLPAD"; + case (NC_EINMEMORY): return "NC_EINMEMORY"; + // case (NC_EURL): return "NC_EURL"; // case (NC_ECONSTRAINT): return "NC_ECONSTRAINT"; #ifdef USE_PNETCDF case (NC_ESMALL): return "NC_ESMALL"; From 70e454227e59762ab700e6867c2461ae8f1ecd34 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 30 Jul 2018 15:25:45 -0600 Subject: [PATCH 09/16] Corrected an issue on Windows. --- ncdump/tst_nccopy4.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index dde77a39d..53797c4d8 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -83,7 +83,7 @@ diff tmp.cdl tmp-chunked.cdl $NCCOPY -c dim0/,dim1/,dim2/,dim3/,dim4/,dim5/,dim6/ tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl -$NCCOPY -c / tmp-chunked.nc tmp-unchunked.nc +$NCCOPY -c // tmp-chunked.nc tmp-unchunked.nc ${NCDUMP} -n tmp tmp-unchunked.nc > tmp-unchunked.cdl diff tmp.cdl tmp-unchunked.cdl echo "*** Test that nccopy -c works as intended for record dimension default (1)" From bf6ae6c59153dd2d36dbdf7793e29f824a98a20d Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 31 Jul 2018 12:51:24 -0600 Subject: [PATCH 10/16] Added ocprint binary to cmakelists. --- ncdump/CMakeLists.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 67b6e67d0..cb338c205 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -5,17 +5,22 @@ ENDIF() SET(ncdump_FILES ncdump.c vardata.c dumplib.c indent.c nctime0.c utils.c nciter.c) SET(nccopy_FILES nccopy.c nciter.c chunkspec.c utils.c dimmap.c list.c) +SET(ocprint_FILES ocprint.c) IF(USE_X_GETOPT) SET(ncdump_FILES ${ncdump_FILES} XGetopt.c) SET(nccopy_FILES ${nccopy_FILES} XGetopt.c) + SET(ocprint_FILES ${ocprint_FILES} XGetopt.c) + ENDIF() ADD_EXECUTABLE(ncdump ${ncdump_FILES}) ADD_EXECUTABLE(nccopy ${nccopy_FILES}) +ADD_EXECUTABLE(ocprint ${ocprint_FILES}) TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS}) +TARGET_LINK_LIBRARIES(ocprint netcdf ${ALL_TLL_LIBS}) #### # We have to do a little tweaking @@ -37,6 +42,13 @@ IF(MSVC) ${CMAKE_CURRENT_BINARY_DIR}) SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) ENDIF() @@ -236,12 +248,16 @@ IF(MSVC) SET_TARGET_PROPERTIES(nccopy PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" ) + SET_TARGET_PROPERTIES(ocprint + PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" + ) ENDIF() INSTALL(TARGETS ncdump RUNTIME DESTINATION bin COMPONENT utilities) INSTALL(TARGETS nccopy RUNTIME DESTINATION bin COMPONENT utilities) +INSTALL(TARGETS ocprint RUNTIME DESTINATION bin COMPONENT utilities) SET(MAN_FILES nccopy.1 ncdump.1) # Note, the L512.bin file is file containing exactly 512 bytes each of value 0. From 6cf9c2e6c59e406e762072f271328b5a52a1dcd9 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 1 Aug 2018 14:15:01 -0600 Subject: [PATCH 11/16] Fence-posted ocprint behind a DAP conditional, in support of https://github.com/Unidata/netcdf-c/commit/bf6ae6c59153dd2d36dbdf7793e29f824a98a20d#commitcomment-29912584 --- ncdump/CMakeLists.txt | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index cb338c205..b4d4bbdfa 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -16,11 +16,17 @@ ENDIF() ADD_EXECUTABLE(ncdump ${ncdump_FILES}) ADD_EXECUTABLE(nccopy ${nccopy_FILES}) -ADD_EXECUTABLE(ocprint ${ocprint_FILES}) + +IF(ENABLE_DAP) + ADD_EXECUTABLE(ocprint ${ocprint_FILES}) +ENDIF(ENABLE_DAP) TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS}) -TARGET_LINK_LIBRARIES(ocprint netcdf ${ALL_TLL_LIBS}) + +IF(ENABLE_DAP) + TARGET_LINK_LIBRARIES(ocprint netcdf ${ALL_TLL_LIBS}) +ENDIF(ENABLE_DAP) #### # We have to do a little tweaking @@ -43,12 +49,15 @@ IF(MSVC) SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + IF(ENABLE_DAP) SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) + ENDIF(ENABLE_DAP) + ENDIF() @@ -248,16 +257,19 @@ IF(MSVC) SET_TARGET_PROPERTIES(nccopy PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" ) - SET_TARGET_PROPERTIES(ocprint - PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" - ) + + IF(ENABLE_DAP) + SET_TARGET_PROPERTIES(ocprint + PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT" + ) + ENDIF(ENABLE_DAP) ENDIF() INSTALL(TARGETS ncdump RUNTIME DESTINATION bin COMPONENT utilities) INSTALL(TARGETS nccopy RUNTIME DESTINATION bin COMPONENT utilities) -INSTALL(TARGETS ocprint RUNTIME DESTINATION bin COMPONENT utilities) + SET(MAN_FILES nccopy.1 ncdump.1) # Note, the L512.bin file is file containing exactly 512 bytes each of value 0. From ed1b7089db7f8c4eff5ad11da42bf3ff83ef48b6 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 1 Aug 2018 14:19:50 -0600 Subject: [PATCH 12/16] Modified ocprint to not be installed --- ncdump/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index cf3b4abdf..554b7528c 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -36,7 +36,7 @@ endif # Conditionally build the ocprint program, but do not install if ENABLE_DAP -bin_PROGRAMS += ocprint +noinst_PROGRAMS += ocprint ocprint_SOURCES = ocprint.c endif From 16bc7d63f6146418025b2b308d334c3e368231eb Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 1 Aug 2018 14:27:09 -0600 Subject: [PATCH 13/16] Modified ocprint to not install. --- ncdump/Makefile.am | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index 554b7528c..1df11844a 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -11,7 +11,7 @@ # Put together AM_CPPFLAGS and AM_LDFLAGS. include $(top_srcdir)/lib_flags.am LDADD = ${top_builddir}/liblib/libnetcdf.la - +noinst_PROGRAMS= # Note which tests depend on other tests. Necessary for make -j check. TEST_EXTENSIONS = .sh XFAIL_TESTS="" @@ -30,7 +30,7 @@ utils.h utils.c dimmap.h dimmap.c list.c list.h # A simple netcdf-4 metadata -> xml printer. Do not install. if USE_NETCDF4 -noinst_PROGRAMS = nc4print +noinst_PROGRAMS += nc4print nc4print_SOURCES = nc4print.c endif From 504867ab673a696ba1b00b4ecdc10fc2e4a7b17e Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 2 Aug 2018 09:47:36 -0600 Subject: [PATCH 14/16] Corrected issue with strings.h. --- ncdump/ocprint.c | 66 +++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 9bf5cc3c9..4f346dd4e 100644 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -15,7 +15,9 @@ #endif #include #include +#ifdef HAVE_STRINGS_H #include +#endif #include "oc.h" #include "ocx.h" @@ -181,7 +183,7 @@ main(int argc, char **argv) #ifdef OCDEBUG { int i; fprintf(stderr,"argv ="); - for(i=0;i=0;i--) { - indices[i]++; - if(indices[i] < dimsizes[i]) break; - if(i > 0) indices[i] = 0; - } +odom_next(size_t rank, size_t* indices, size_t* dimsizes) +{ + int i; + for(i=rank-1;i>=0;i--) { + indices[i]++; + if(indices[i] < dimsizes[i]) break; + if(i > 0) indices[i] = 0; + } } /* Return 0 if we have exhausted the indices, 1 otherwise */ static int -odom_more(size_t rank, size_t* indices, size_t* dimsizes) +odom_more(size_t rank, size_t* indices, size_t* dimsizes) { if(indices[0] >= dimsizes[0]) return 0; return 1; -} +} /* Compute total # of elements if dimensioned */ static size_t @@ -1172,7 +1174,7 @@ static void printstack(char* msg) FAIL(oc_dds_dimensionsizes(glink,entry->node,edges)); FAIL(oc_dds_name(glink,node,&name)); fprintf(stderr," [%d] (%s)",(int)i,name) - for(j=0;j Date: Thu, 2 Aug 2018 11:33:21 -0600 Subject: [PATCH 15/16] Correcting an appveyor failure. --- ncdump/ocprint.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 4f346dd4e..f10c90335 100644 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -34,6 +34,12 @@ #define strcasecmp stricmp #endif +#ifdef _MSC_VER +#include "XGetopt.h" +int opterr; +int optind; +#endif + #ifndef nulldup #define nulldup(s) (s==NULL?NULL:strdup(s)) #endif From 81c91eb5197723340b420d18993c41f7bc5086ea Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 2 Aug 2018 14:58:47 -0600 Subject: [PATCH 16/16] Made some changes to export symbols on Windows builds. --- include/ncuri.h | 20 +++++- ncdump/ocprint.c | 2 +- oc2/oc.h | 170 +++++++++++++++++++++++++---------------------- oc2/ocdebug.h | 5 +- oc2/ocx.h | 23 +++++-- 5 files changed, 129 insertions(+), 91 deletions(-) diff --git a/include/ncuri.h b/include/ncuri.h index 72fbe1a42..a60febf52 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -52,21 +52,35 @@ typedef struct NCURI { #endif } NCURI; +/* Declaration modifiers for DLL support (MSC et al) */ +#if defined(DLL_NETCDF) /* define when library is a DLL */ +# if defined(DLL_EXPORT) /* define when building the library */ +# define MSC_EXTRA __declspec(dllexport) +# else +# define MSC_EXTRA __declspec(dllimport) +# endif +# include +#else +#define MSC_EXTRA /**< Needed for DLL build. */ +#endif /* defined(DLL_NETCDF) */ + +#define EXTERNL MSC_EXTRA extern /**< Needed for DLL build. */ + #if defined(_CPLUSPLUS_) || defined(__CPLUSPLUS__) || defined(__CPLUSPLUS) extern "C" { #endif -extern int ncuriparse(const char* s, NCURI** ncuri); +EXTERNL int ncuriparse(const char* s, NCURI** ncuri); extern void ncurifree(NCURI* ncuri); /* Replace the protocol */ extern int ncurisetprotocol(NCURI*,const char* newprotocol); /* Replace the constraints */ -extern int ncurisetquery(NCURI*,const char* query); +EXTERNL int ncurisetquery(NCURI*,const char* query); /* Construct a complete NC URI; caller frees returned string */ -extern char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags); +EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags); /*! Search the fragment for a given parameter Null result => entry not found; !NULL=>found; diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index f10c90335..f2ffdb88a 100644 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -60,7 +60,7 @@ int optind; /*Mnemonic*/ #define TOPLEVEL 1 -extern int ocdebug; +int ocdebug; static OCerror ocstat; static OClink glink; diff --git a/oc2/oc.h b/oc2/oc.h index a51dcc9e1..9f6ebc53d 100644 --- a/oc2/oc.h +++ b/oc2/oc.h @@ -197,34 +197,46 @@ typedef OCobject OClink; /**@}*/ /**************************************************/ -/* External API */ +/* extern API */ +/* Declaration modifiers for DLL support (MSC et al) */ +#if defined(DLL_NETCDF) /* define when library is a DLL */ +# if defined(DLL_EXPORT) /* define when building the library */ +# define MSC_EXTRA __declspec(dllexport) +# else +# define MSC_EXTRA __declspec(dllimport) +# endif +# include +#else +#define MSC_EXTRA /**< Needed for DLL build. */ +#endif /* defined(DLL_NETCDF) */ +#define EXTERNL MSC_EXTRA extern /**< Needed for DLL build. */ #ifdef __cplusplus -extern "C" { +external "C" { #endif /**************************************************/ /* Link management */ -extern OCerror oc_open(const char* url, OClink*); -extern OCerror oc_close(OClink); +EXTERNL OCerror oc_open(const char* url, OClink*); +EXTERNL OCerror oc_close(OClink); /**************************************************/ /* Tree Management */ -extern OCerror oc_fetch(OClink, +EXTERNL OCerror oc_fetch(OClink, const char* constraint, OCdxd, OCflags, OCddsnode*); -extern OCerror oc_root_free(OClink, OCddsnode root); -extern const char* oc_tree_text(OClink, OCddsnode root); +EXTERNL OCerror oc_root_free(OClink, OCddsnode root); +EXTERNL const char* oc_tree_text(OClink, OCddsnode root); /**************************************************/ /* Node Management */ -extern OCerror oc_dds_properties(OClink, OCddsnode, +EXTERNL OCerror oc_dds_properties(OClink, OCddsnode, char** namep, OCtype* typep, OCtype* atomictypep, /* if octype == OC_Atomic */ @@ -235,14 +247,14 @@ extern OCerror oc_dds_properties(OClink, OCddsnode, /* Define some individual accessors for convenience */ -extern OCerror oc_dds_name(OClink,OCddsnode,char**); -extern OCerror oc_dds_class(OClink,OCddsnode,OCtype*); -extern OCerror oc_dds_atomictype(OClink,OCddsnode,OCtype*); -extern OCerror oc_dds_nsubnodes(OClink,OCddsnode,size_t*); -extern OCerror oc_dds_rank(OClink,OCddsnode,size_t*); -extern OCerror oc_dds_attr_count(OClink,OCddsnode,size_t*); -extern OCerror oc_dds_root(OClink,OCddsnode,OCddsnode*); -extern OCerror oc_dds_container(OClink,OCddsnode,OCddsnode*); +EXTERNL OCerror oc_dds_name(OClink,OCddsnode,char**); +EXTERNL OCerror oc_dds_class(OClink,OCddsnode,OCtype*); +EXTERNL OCerror oc_dds_atomictype(OClink,OCddsnode,OCtype*); +EXTERNL OCerror oc_dds_nsubnodes(OClink,OCddsnode,size_t*); +EXTERNL OCerror oc_dds_rank(OClink,OCddsnode,size_t*); +EXTERNL OCerror oc_dds_attr_count(OClink,OCddsnode,size_t*); +EXTERNL OCerror oc_dds_root(OClink,OCddsnode,OCddsnode*); +EXTERNL OCerror oc_dds_container(OClink,OCddsnode,OCddsnode*); /* Aliases */ #define oc_dds_octype oc_dds_class @@ -252,44 +264,44 @@ extern OCerror oc_dds_container(OClink,OCddsnode,OCddsnode*); if there is no such node; return OC_EBADTYPE if node is not a container */ -extern OCerror oc_dds_ithfield(OClink, OCddsnode, size_t index, OCddsnode* ithfieldp); +EXTERNL OCerror oc_dds_ithfield(OClink, OCddsnode, size_t index, OCddsnode* ithfieldp); /* Alias for oc_dds_ithfield */ -extern OCerror oc_dds_ithsubnode(OClink, OCddsnode, size_t, OCddsnode*); +EXTERNL OCerror oc_dds_ithsubnode(OClink, OCddsnode, size_t, OCddsnode*); /* Convenience functions that are just combinations of ithfield with other functions */ /* Return the grid array dds node from the specified grid node*/ -extern OCerror oc_dds_gridarray(OClink, OCddsnode grid, OCddsnode* arrayp); +EXTERNL OCerror oc_dds_gridarray(OClink, OCddsnode grid, OCddsnode* arrayp); /* Return the i'th grid map dds node from the specified grid dds node. NOTE: Map indices start at ZERO. */ -extern OCerror oc_dds_gridmap(OClink, OCddsnode grid, size_t index, OCddsnode* mapp); +EXTERNL OCerror oc_dds_gridmap(OClink, OCddsnode grid, size_t index, OCddsnode* mapp); /* Retrieve a dds node by name from a dds structure or dataset node. return OC_EBADTYPE if node is not a container, return OC_EINDEX if no field by the given name is found. */ -extern OCerror oc_dds_fieldbyname(OClink, OCddsnode, const char* name, OCddsnode* fieldp); +EXTERNL OCerror oc_dds_fieldbyname(OClink, OCddsnode, const char* name, OCddsnode* fieldp); /* Return the dimension nodes, if any, associated with a given DDS node */ /* Caller must allocate and free the vector for dimids */ /* If the node is scalar, then return OC_ESCALAR. */ -extern OCerror oc_dds_dimensions(OClink, OCddsnode, OCddsnode* dimids); +EXTERNL OCerror oc_dds_dimensions(OClink, OCddsnode, OCddsnode* dimids); /* Return the i'th dimension node, if any, associated with a given object */ /* If there is no such dimension, then return OC_EINVAL */ -extern OCerror oc_dds_ithdimension(OClink,OCddsnode, size_t, OCddsnode*); +EXTERNL OCerror oc_dds_ithdimension(OClink,OCddsnode, size_t, OCddsnode*); /* Return the size and name associated with a given dimension object as defined in the DDS */ -extern OCerror oc_dimension_properties(OClink,OCddsnode,size_t*,char**); +EXTERNL OCerror oc_dimension_properties(OClink,OCddsnode,size_t*,char**); /* For convenience, return only the dimension sizes */ -extern OCerror oc_dds_dimensionsizes(OClink,OCddsnode,size_t* dimsizes); +EXTERNL OCerror oc_dds_dimensionsizes(OClink,OCddsnode,size_t* dimsizes); /* Attribute Management */ @@ -299,13 +311,13 @@ extern OCerror oc_dds_dimensionsizes(OClink,OCddsnode,size_t* dimsizes); must do any required conversion based on the octype. The strings vector must be allocated and freed by caller, The contents of the strings vector must also be reclaimed - using oc_attr_reclaim(see below). + using oc_attr_reclaim(see below). Standard practice is to call twice, once with the strings argument == NULL so we get the number of values, then the second time with an allocated char** vector. */ -extern OCerror oc_dds_attr(OClink,OCddsnode, size_t i, +EXTERNL OCerror oc_dds_attr(OClink,OCddsnode, size_t i, char** name, OCtype* octype, size_t* nvalues, char** strings); @@ -318,19 +330,19 @@ extern OCerror oc_dds_attr(OClink,OCddsnode, size_t i, Caller must free returned string. */ -extern OCerror oc_das_attr_count(OClink, OCddsnode, size_t* countp); +EXTERNL OCerror oc_das_attr_count(OClink, OCddsnode, size_t* countp); -extern OCerror oc_das_attr(OClink,OCddsnode, size_t, OCtype*, char**); +EXTERNL OCerror oc_das_attr(OClink,OCddsnode, size_t, OCtype*, char**); /**************************************************/ /* Free up a ddsnode that is no longer being used */ -extern OCerror oc_dds_free(OClink, OCddsnode); +EXTERNL OCerror oc_dds_free(OClink, OCddsnode); /**************************************************/ /* Data Procedures */ /* Given the DDS tree root, get the root data of datadds */ -extern OCerror oc_dds_getdataroot(OClink, OCddsnode treeroot, OCdatanode* rootp); +EXTERNL OCerror oc_dds_getdataroot(OClink, OCddsnode treeroot, OCdatanode* rootp); /* Alias for oc_dds_getdataroot */ #define oc_data_getroot oc_dds_getdataroot @@ -339,13 +351,13 @@ extern OCerror oc_dds_getdataroot(OClink, OCddsnode treeroot, OCdatanode* rootp) If it does not exist, then return NULL. In effect this procedure allows one to walk up the datatree one level. */ -extern OCerror oc_data_container(OClink, OCdatanode data, OCdatanode* containerp); +EXTERNL OCerror oc_data_container(OClink, OCdatanode data, OCdatanode* containerp); /* Return the root node of the data tree that contains the specified data node. In effect this procedure allows one to walk to the root of the datatree containing the specified datanode. */ -extern OCerror oc_data_root(OClink, OCdatanode data, OCdatanode* rootp); +EXTERNL OCerror oc_data_root(OClink, OCdatanode data, OCdatanode* rootp); /* There are multiple ways to walk down a level in a data tree @@ -373,33 +385,33 @@ or oc_data_readscalar. */ /* Return the data node for the i'th field of the specified container data */ -extern OCerror oc_data_ithfield(OClink, OCdatanode container, size_t index, +EXTERNL OCerror oc_data_ithfield(OClink, OCdatanode container, size_t index, OCdatanode* fieldp); /* Retrieve the data node by name from a container data node */ -extern OCerror oc_dat_fieldbyname(OClink, OCdatanode, const char* name, OCdatanode* fieldp); +EXTERNL OCerror oc_dat_fieldbyname(OClink, OCdatanode, const char* name, OCdatanode* fieldp); /* Return the grid array data for the specified grid data */ -extern OCerror oc_data_gridarray(OClink, OCdatanode grid, OCdatanode* arrayp); +EXTERNL OCerror oc_data_gridarray(OClink, OCdatanode grid, OCdatanode* arrayp); /* Return the i'th grid map data for the specified grid data. NOTE: Map indices start at ZERO. */ -extern OCerror oc_data_gridmap(OClink, OCdatanode grid, size_t index, OCdatanode* mapp); +EXTERNL OCerror oc_data_gridmap(OClink, OCdatanode grid, size_t index, OCdatanode* mapp); /* Return the data of a dimensioned Structure corresponding to the element specified by the indices. */ -extern OCerror oc_data_ithelement(OClink, OCdatanode data, size_t* indices, OCdatanode* elementp); +EXTERNL OCerror oc_data_ithelement(OClink, OCdatanode data, size_t* indices, OCdatanode* elementp); /* Return the i'th record data of a Sequence data. */ -extern OCerror oc_data_ithrecord(OClink, OCdatanode data, size_t index, OCdatanode* recordp); +EXTERNL OCerror oc_data_ithrecord(OClink, OCdatanode data, size_t index, OCdatanode* recordp); /* Free up an data that is no longer being used */ -extern OCerror oc_data_free(OClink, OCdatanode data); +EXTERNL OCerror oc_data_free(OClink, OCdatanode data); /* Count the records associated with a sequence */ -extern OCerror oc_data_recordcount(OClink, OCdatanode, size_t*); +EXTERNL OCerror oc_data_recordcount(OClink, OCdatanode, size_t*); /* Return the actual data values associated with the specified leaf data. The OCdatanode is assumed to be referencing a leaf node that is @@ -409,14 +421,14 @@ extern OCerror oc_data_recordcount(OClink, OCdatanode, size_t*); and free'ing it. See also oc_dds_read(). */ -extern OCerror oc_data_read(OClink, OCdatanode, size_t*, size_t*, size_t, void*); +EXTERNL OCerror oc_data_read(OClink, OCdatanode, size_t*, size_t*, size_t, void*); /* Like oc_data_read, but for reading a scalar. Caller is responsible for allocating memory(of proper size) and free'ing it. See also oc_dds_readscalar(). */ -extern OCerror oc_data_readscalar(OClink, OCdatanode, size_t, void*); +EXTERNL OCerror oc_data_readscalar(OClink, OCdatanode, size_t, void*); /* Like oc_data_read, but caller provides a starting set of indices and count of the number of elements to read. @@ -424,81 +436,81 @@ extern OCerror oc_data_readscalar(OClink, OCdatanode, size_t, void*); and free'ing it. See also oc_dds_readn(). */ -extern OCerror oc_data_readn(OClink, OCdatanode, size_t*, size_t, size_t, void*); +EXTERNL OCerror oc_data_readn(OClink, OCdatanode, size_t*, size_t, size_t, void*); /* Return the indices for this datas; Assumes the data was obtained using oc_data_ithelement or oc_data_ithrecord; if not, then an error is returned. */ -extern OCerror oc_data_position(OClink, OCdatanode data, size_t* indices); +EXTERNL OCerror oc_data_position(OClink, OCdatanode data, size_t* indices); /* Return the pattern dds node for an data */ -extern OCerror oc_data_ddsnode(OClink, OCdatanode data, OCddsnode*); +EXTERNL OCerror oc_data_ddsnode(OClink, OCdatanode data, OCddsnode*); /* Return the octype of the data (convenience) */ -extern OCerror oc_data_octype(OClink, OCdatanode data, OCtype*); +EXTERNL OCerror oc_data_octype(OClink, OCdatanode data, OCtype*); /* Return 1 if the specified data has a valid index, 0 otherwise. Valid index means it was created using oc_data_ithelement or oc_data_ithrecord. */ -extern int oc_data_indexed(OClink link, OCdatanode datanode); +EXTERNL int oc_data_indexed(OClink link, OCdatanode datanode); /* Return 1 if the specified data has a valid index, 0 otherwise. Valid index means it was created using oc_data_ithelement or oc_data_ithrecord. */ -extern int oc_data_indexed(OClink, OCdatanode); +EXTERNL int oc_data_indexed(OClink, OCdatanode); /* Return 1 if the specified data can be indexed Indexable means the data is pointing to an indexed structure or to a sequence. */ -extern int oc_data_indexable(OClink, OCdatanode); +EXTERNL int oc_data_indexable(OClink, OCdatanode); /**************************************************/ -/* +/* For top-level, atomic variables, it is possible to directly read the associated data without having to use the oc_data_XXX procedures. Provide special procedures to support this. */ /* See oc_data_read for semantics */ -extern OCerror oc_dds_read(OClink, OCddsnode, size_t*, size_t*, size_t, void*); +EXTERNL OCerror oc_dds_read(OClink, OCddsnode, size_t*, size_t*, size_t, void*); /* See oc_data_readscalar for semantics */ -extern OCerror oc_dds_readscalar(OClink, OCddsnode, size_t, void*); +EXTERNL OCerror oc_dds_readscalar(OClink, OCddsnode, size_t, void*); /* See oc_data_readn for semantics */ -extern OCerror oc_dds_readn(OClink, OCddsnode, size_t*, size_t, size_t, void*); +EXTERNL OCerror oc_dds_readn(OClink, OCddsnode, size_t*, size_t, size_t, void*); /**************************************************/ /* Misc. OCtype-related functions */ /* Return size of the given type(Atomic only) */ -extern size_t oc_typesize(OCtype); +EXTERNL size_t oc_typesize(OCtype); /* Return a canonical printable string describing a given type: e.g. Byte, Int16, etc. */ -extern const char* oc_typetostring(OCtype); +EXTERNL const char* oc_typetostring(OCtype); /* Given a value of a atomic OC type, provide a canonical string representing that value; mostly for debugging. */ -extern OCerror oc_typeprint(OCtype, void* value, size_t bufsize, char* buf); +EXTERNL OCerror oc_typeprint(OCtype, void* value, size_t bufsize, char* buf); /**************************************************/ /* Logging */ -extern void oc_loginit(void); -extern int oc_setlogging(int onoff); /* 1=>start logging 0=>stop */ -extern int oc_logopen(const char* logfilename); -extern void oc_logclose(void); +EXTERNL void oc_loginit(void); +EXTERNL int oc_setlogging(int onoff); /* 1=>start logging 0=>stop */ +EXTERNL int oc_logopen(const char* logfilename); +EXTERNL void oc_logclose(void); -extern void oc_log(int tag, const char* fmt, ...); +EXTERNL void oc_log(int tag, const char* fmt, ...); -extern void oc_logtext(int tag, const char* text); +EXTERNL void oc_logtext(int tag, const char* text); /**************************************************/ /* Miscellaneous */ @@ -509,41 +521,41 @@ extern void oc_logtext(int tag, const char* text); Note that only the strings are reclaimed, the string vector is not reclaimed because it was presumably allocated by the client. */ -extern void oc_reclaim_strings(size_t n, char** svec); +EXTERNL void oc_reclaim_strings(size_t n, char** svec); /* Convert an OCerror to a human readable string */ -extern const char* oc_errstring(OCerror err); +EXTERNL const char* oc_errstring(OCerror err); /* Get client parameters from the URL DO NOT free the result */ -extern const char* oc_clientparam_get(OClink, const char* param); +EXTERNL const char* oc_clientparam_get(OClink, const char* param); /**************************************************/ /* Merging operations */ /* Merge a specified DAS into a specified DDS or DATADDS */ -extern OCerror oc_merge_das(OClink, OCddsnode dasroot, OCddsnode ddsroot); +EXTERNL OCerror oc_merge_das(OClink, OCddsnode dasroot, OCddsnode ddsroot); /**************************************************/ /* Debugging */ /* When a server error is detected, then it is possible to get DODS supplied server error info using this procedure */ -extern OCerror oc_svcerrordata(OClink link, char** codep, +EXTERNL OCerror oc_svcerrordata(OClink link, char** codep, char** msgp, long* httpp); /* Get the HTTP return code from the last call; note that this may or may not be the same as returned by oc_svcerrordata. */ -extern int oc_httpcode(OClink); +EXTERNL int oc_httpcode(OClink); /* (Re-)initialize the oc library as if nothing had been called. This is primarily for debugging of rc files. */ -extern OCerror oc_initialize(void); +EXTERNL OCerror oc_initialize(void); /**************************************************/ /* Curl options */ @@ -552,37 +564,37 @@ extern OCerror oc_initialize(void); */ /*Cause the curl library to be verbose and save error messages*/ -extern OCerror oc_trace_curl(OClink link); +EXTERNL OCerror oc_trace_curl(OClink link); /* Allow specification of the rc file */ -extern OCerror oc_set_rcfile(const char* filepath); +EXTERNL OCerror oc_set_rcfile(const char* filepath); /* Allow specification of the netrc file */ -extern OCerror oc_set_netrc(OClink*, const char* filepath); +EXTERNL OCerror oc_set_netrc(OClink*, const char* filepath); /* Set arbitrary curl option */ -extern OCerror oc_set_curlopt(OClink link, const char* option, void* value); +EXTERNL OCerror oc_set_curlopt(OClink link, const char* option, void* value); /**************************************************/ /* Experimental/Undocumented */ /* Given an arbitrary OCnode, return the connection of which it is a part */ -extern OCerror oc_get_connection(OCobject ocnode, OCobject* linkp); +EXTERNL OCerror oc_get_connection(OCobject ocnode, OCobject* linkp); /* Resend a url as a head request to check the Last-Modified time */ -extern OCerror oc_update_lastmodified_data(OClink); +EXTERNL OCerror oc_update_lastmodified_data(OClink); /* Get last known modification time; -1 => data unknown */ -extern long oc_get_lastmodified_data(OClink); +EXTERNL long oc_get_lastmodified_data(OClink); /* Test if a given url responds to a DAP protocol request */ -extern OCerror oc_ping(const char* url); +EXTERNL OCerror oc_ping(const char* url); /* Return the size of the in-memory or on-disk data chunk returned by the server for a given tree. Zero implies it is not defined. */ -extern OCerror oc_raw_xdrsize(OClink,OCddsnode,off_t*); +EXTERNL OCerror oc_raw_xdrsize(OClink,OCddsnode,off_t*); #ifdef __cplusplus } diff --git a/oc2/ocdebug.h b/oc2/ocdebug.h index 0bae593f6..ae64f085d 100644 --- a/oc2/ocdebug.h +++ b/oc2/ocdebug.h @@ -58,8 +58,8 @@ about how IO is getting along. */ #undef OCPROGRESS -extern int ocdebug; -extern int cedebug; +EXTERNL int ocdebug; +EXTERNL int cedebug; /*extern char* dent2(int n);*/ /*/extern char* dent(int n);*/ @@ -103,4 +103,3 @@ extern CURLcode ocreportcurlerror(struct OCstate* state, CURLcode cstat); #define OCTHROWCHK(e) OCCATCHCHK(e) #endif /*OCOCDBG_H*/ - diff --git a/oc2/ocx.h b/oc2/ocx.h index 29c82bc92..d00c16478 100644 --- a/oc2/ocx.h +++ b/oc2/ocx.h @@ -9,6 +9,20 @@ Internal library debugging interface #ifndef OCX_H #define OCX_H +/* Declaration modifiers for DLL support (MSC et al) */ +#if defined(DLL_NETCDF) /* define when library is a DLL */ +# if defined(DLL_EXPORT) /* define when building the library */ +# define MSC_EXTRA __declspec(dllexport) +# else +# define MSC_EXTRA __declspec(dllimport) +# endif +# include +#else +#define MSC_EXTRA /**< Needed for DLL build. */ +#endif /* defined(DLL_NETCDF) */ + +#define EXTERNL MSC_EXTRA extern /**< Needed for DLL build. */ + /**************************************************/ /* Flags defining the structure of an OCdata object */ @@ -24,10 +38,9 @@ typedef unsigned int OCDT; /* Return mode for this data */ extern OCerror oc_data_mode(OClink, OCdatanode, OCDT* modep); -extern OCerror oc_dds_dd(OClink, OCddsnode, int); -extern OCerror oc_dds_ddnode(OClink, OCddsnode); -extern OCerror oc_data_ddpath(OClink, OCdatanode, char**); -extern OCerror oc_data_ddtree(OClink, OCdatanode root); +EXTERNL OCerror oc_dds_dd(OClink, OCddsnode, int); +EXTERNL OCerror oc_dds_ddnode(OClink, OCddsnode); +EXTERNL OCerror oc_data_ddpath(OClink, OCdatanode, char**); +EXTERNL OCerror oc_data_ddtree(OClink, OCdatanode root); #endif /*OCX_H*/ -