mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-27 07:30:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1657 from NOAA-GSD/ejh_fix_master

Browse Source 
Fix largefile issue, szip parallel test, and some documentation updates
This commit is contained in:
Ward Fisher 2020-03-02 23:40:57 -07:00 committed by GitHub
commit de4c487fdc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 63 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASE_NOTES.md
View File

@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
## 4.7.4 - TBD
* [Bug Fix] Attempts to set chunking on scalar vars will now return NC_EINVAL. Scalar vars cannot be chunked. Previously the library ignored these attempts and scalar vars were always stored as contiguous. See [https://github.com/Unidata/netcdf-c/issues/1644].
* [Enhancement] Support has been added for multiple filters per variable. See [https://github.com/Unidata/netcdf-c/issues/1584].
* [Enhancement] Now nc_inq_var_szip retuns 0 for parameter values if szip is not in use for var. See [https://github.com/Unidata/netcdf-c/issues/1618].
* [Enhancement] Now allow parallel I/O with filters, for HDF5-1.10.3 and later. See [https://github.com/Unidata/netcdf-c/issues/1473].

7
h5_test/tst_h_files.c
View File

@ -246,7 +246,6 @@ main()
char file_name[STR_LEN + 1];
char dimscale_wo_var[STR_LEN];
void *bufr;
hsize_t num_obj;
void *fillp = NULL;
sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
@ -361,12 +360,6 @@ main()
if (H5Gclose(grpid) < 0) ERR;
if (H5Fclose(fileid) < 0) ERR;
/* Reopen the file and check it. */
if ((fileid = H5Fopen(file_name, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) ERR;
if (H5Gget_num_objs(fileid, &num_obj) < 0) ERR;
if (num_obj) ERR;
if (H5Fclose(fileid) < 0) ERR;
/* Delete the huge data file we created. */
(void) remove(file_name);
}

83
libdispatch/dparallel.c
View File

@ -289,50 +289,58 @@ nc_open_par_fortran(const char *path, int omode, int comm,
#endif
}
/**\ingroup datasets
/**
@ingroup datasets
This function will change the parallel access of a variable from independent to
collective and vice versa. Note when file is opened/created to use PnetCDF
library to perform parallel I/O underneath, argument varid is ignored and the
mode changed by this function applies to all variables. This is because PnetCDF
does not support access mode change for individual variables. In this case,
users may use NC_GLOBAL in varid argument for better program readability. This
function is collective, i.e. must be called by all MPI processes defined in the
MPI communicator used in nc_create_par() or nc_open_par(). In addition, values
of arguments of this function must be the same among all MPI processes.
This function will change the parallel access of a variable from
independent to collective and vice versa.
To obtain a good I/O performance, users are recommended to use collective mode.
In addition, switching between collective and independent I/O mode can be
expensive.
This function is collective, i.e. must be called by all MPI
processes defined in the MPI communicator used in nc_create_par()
or nc_open_par(). In addition, values of arguments of this function
must be the same among all MPI processes.
In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib and
fletcher32 filters may be used when writing data with parallel
I/O. The use of these filters require collective access. Turning on
the zlib (deflate) or fletcher32 filter for a variable will
automatically set its access to collective. Attempts to set access to
independent will return ::NC_EINVAL.
To obtain a good I/O performance, users are recommended to use
collective mode. In addition, switching between collective and
independent I/O mode can be expensive.
\param ncid NetCDF or group ID, from a previous call to nc_open_par(),
nc_create_par(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib,
szip, fletcher32, and other filters may be used when writing data
with parallel I/O. The use of these filters require collective
access. Turning on the zlib (deflate) or fletcher32 filter for a
variable will automatically set its access to collective if the
file has been opened for parallel I/O. Attempts to set access to
independent will return ::NC_EINVAL.
\param varid Variable ID
@note When the library is build with --enable-pnetcdf, and when
file is opened/created to use PnetCDF library to perform parallel
I/O underneath, argument varid is ignored and the mode changed by
this function applies to all variables. This is because PnetCDF
does not support access mode change for individual variables. In
this case, users may use NC_GLOBAL in varid argument for better
program readability.
\param par_access NC_COLLECTIVE or NC_INDEPENDENT.
@param ncid NetCDF or group ID, from a previous call to
nc_open_par(), nc_create_par(), nc_def_grp(), or associated inquiry
functions such as nc_inq_ncid().
\returns ::NC_NOERR No error.
\returns ::NC_EBADID Invalid ncid passed.
\returns ::NC_ENOTVAR Invalid varid passed.
\returns ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
\returns ::NC_EINVAL Invalid par_access specified, or attempt to set
filtered variable to independent access.
@param varid Variable ID
<h1>Example</h1>
@param par_access NC_COLLECTIVE or NC_INDEPENDENT.
Here is an example from examples/C/parallel_vara.c which changes the
parallel access of a variable and then writes to it.
@return ::NC_NOERR No error.
@return ::NC_EBADID Invalid ncid passed.
@return ::NC_ENOTVAR Invalid varid passed.
@return ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
@return ::NC_EINVAL Invalid par_access specified, or attempt to set
filtered variable to independent access.
\code
<h1>Example</h1>
Here is an example from examples/C/parallel_vara.c which changes
the parallel access of a variable and then writes to it.
@code
#define NY 10
#define NX 4
@ -366,10 +374,9 @@ parallel access of a variable and then writes to it.
err = nc_put_vara_int(ncid, varid, start, count, &buf[0][0]); ERR
\endcode
\author Ed Hartnett, Dennis Heimbigner
\ingroup datasets
*/
@endcode
@author Ed Hartnett, Dennis Heimbigner
*/
int
nc_var_par_access(int ncid, int varid, int par_access)
{

35
libdispatch/dvar.c
View File

@ -470,10 +470,8 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
}
/**
Define chunking parameters for a variable
The function nc_def_var_chunking sets the storage and, optionally,
the chunking parameters for a variable in a netCDF-4 file.
Define storage and, if chunked storage is used, chunking parameters
for a variable
The storage may be set to NC_CONTIGUOUS, NC_COMPACT, or NC_CHUNKED.
@ -490,7 +488,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
Chunked storage means the data are stored as chunks, of
user-configurable size. Chunked storage is required for variable
with one or more unlimted dimensions, or variable which use
compression.
compression, or any other filter.
The total size of a chunk must be less than 4 GiB. That is, the
product of all chunksizes and the size of the data (or the size of
@ -500,16 +498,14 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
before nc_enddef is called. Once the chunking parameters are set for a
variable, they cannot be changed.
Note scalar variables may have a storage of NC_CONTIGUOUS or
NC_COMPACT. Attempts to set chunking on a scalare variable will be
ignored by the library (but no error code is returned). Only
non-scalar variables can have chunking.
@note Scalar variables may have a storage of NC_CONTIGUOUS or
NC_COMPACT. Attempts to set chunking on a scalare variable will
cause ::NC_EINVEL to be returned. Only non-scalar variables can
have chunking.
@param ncid NetCDF ID, from a previous call to nc_open() or
nc_create().
@param varid Variable ID.
@param storage If ::NC_CONTIGUOUS or ::NC_COMPACT, then contiguous
or compact storage is used for this variable. Variables with one or
more unlimited dimensions cannot use contiguous or compact
@ -518,20 +514,19 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
storage is used for this variable. Chunk sizes may be specified
with the chunksizes parameter or default sizes will be used if that
parameter is NULL.
@param chunksizesp A pointer to an array list of chunk sizes. The
array must have one chunksize for each dimension of the variable. If
::NC_CONTIGUOUS storage is set, then the chunksizes parameter is
ignored.
ignored. Ignored if NULL.
@return ::NC_NOERR No error.
@return ::NC_EBADID Bad ID.
@return ::NC_ENOTNC4 Not a netCDF-4 file.
@return ::NC_ELATEDEF This variable has already been the subject of a
nc_enddef call. In netCDF-4 files nc_enddef will be called
@return ::NC_ELATEDEF This variable has already been the subject of
a nc_enddef call. In netCDF-4 files nc_enddef will be called
automatically for any data read or write. Once nc_enddef has been
called after the nc_def_var call for a variable, it is impossible to
set the chunking for that variable.
called after the nc_def_var call for a variable, it is impossible
to set the chunking for that variable.
@return ::NC_ENOTINDEFINE Not in define mode. This is returned for
netCDF classic or 64-bit offset files, or for netCDF-4 files, when
they wwere created with ::NC_CLASSIC_MODEL flag by nc_create().
@ -542,7 +537,8 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
@return ::NC_EVARSIZE Compact storage attempted for variable bigger
than 64 KB.
@return ::NC_EINVAL Attempt to set contiguous or compact storage
for var with one or more unlimited dimensions.
for var with one or more unlimited dimensions, or chunking for a
scalar var.
@section nc_def_var_chunking_example Example
@ -585,8 +581,7 @@ nc_def_var_fletcher32(int ncid, int varid, int fletcher32)
@author Ed Hartnett, Dennis Heimbigner
*/
int
nc_def_var_chunking(int ncid, int varid, int storage,
const size_t *chunksizesp)
nc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
{
NC* ncp;
int stat = NC_check_id(ncid, &ncp);

3
nc_test4/tst_parallel5.c
View File

@ -336,7 +336,8 @@ main(int argc, char **argv)
&ncid)) ERR;
if (nc_def_dim(ncid, SZIP_DIM_NAME, SZIP_DIM_LEN, &dimid)) ERR;
if (nc_def_var(ncid, SZIP_VAR_NAME, NC_FLOAT, NDIMS1, &dimid, &varid)) ERR;
if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, SZIP_PIXELS_PER_BLOCK)) ERR;
if (nc_def_var_szip(ncid, varid, H5_SZIP_NN_OPTION_MASK,
SZIP_PIXELS_PER_BLOCK)) ERR;
if (nc_enddef(ncid)) ERR;
start[0] = mpi_rank * elements_per_pe;
count[0] = elements_per_pe;

1
unit_test/tst_nclist.c
View File

@ -112,7 +112,6 @@ main(int argc, char **argv)
{
NC *ncp;
int mode = 0;
NCmodel model;
int max_num_nc = 65535;
int i;
int ret;