mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-31 17:50:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix contributed by Nath Gopalaswamy to large file problem reading

Browse Source 
netCDF classic or 64-bit offset files that have a UINT32_MAX flag for
large last record size of a variable that has values larger than 1
byte.  This problem had previously been fixed for *writing* such data,
but was only tested with an ncbyte variable.  Fixed test to
demonstrate problem and the fix.

More updates to chunking documentation, cosmetic fixes for some
"--option=" documentation that doxygen turns into mdash.
This commit is contained in:
Russ Rew 2013-05-31 20:18:18 +00:00
parent 3152893da0
commit 946a7ab8b5
4 changed files with 64 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libsrc/v1hpg.c
View File

@ -1117,8 +1117,8 @@ NC_computeshapes(NC3_INFO* ncp)
{
if(first_rec == NULL)
first_rec = *vpp;
if((*vpp)->len == UINT32_MAX)
ncp->recsize += (*vpp)->dsizes[0];
if((*vpp)->len == UINT32_MAX) /* Flag for large last record */
ncp->recsize += (*vpp)->dsizes[0] * (*vpp)->xsz;
else
ncp->recsize += (*vpp)->len;
}

5
man4/guide.doc
View File

@ -2558,14 +2558,11 @@ function at any time on an open file. C programmers see
nc_set_var_chunk_cache(), Fortran 77 programmers see
NF_SET_VAR_CHUNK_CACHE, ).
\page default_chunking_4_1 The Default Chunking Scheme in version 4.1
(and 4.1.1)
\page default_chunking_4_1 The Default Chunking Scheme
When the data writer does not specify chunk sizes for variable, the
netCDF library has to come up with some default values.
The C code below determines the default chunks sizes.
For unlimited dimensions, a chunk size of one is always used. For
large datasets, where the size of fixed dimensions is small compared
to the unlimited dimensions, users are advised to avoid unlimited

16
man4/install.doc
View File

@ -111,7 +111,7 @@ Then you build HDF5, specifying the location of the zlib library:
make check install
\endverbatim
In all cases, the installation location specified with the --prefix
In all cases, the installation location specified with the <CODE>--prefix</CODE>
option must be different from the source directory where the software
is being built.
@ -120,7 +120,7 @@ directory to the LD_LIBRARY_PATH environment variable. See
the <a href="http://www.unidata.ucar.edu/netcdf/docs/faq.html#Shared%20Libraries">netCDF
FAQ</a> for more details on using shared libraries.
If you are building HDF5 with szip, then include the --with-szlib=
If you are building HDF5 with szip, then include the <CODE>--with-szlib=</CODE>
option, with the directory holding the szip library.
After HDF5 is done, build netcdf, specifying the location of the
@ -133,14 +133,14 @@ make check install
\endverbatim
The configure script will try to find necessary tools in your
path. When you run configure you may optionally use the --prefix
path. When you run configure you may optionally use the <CODE>--prefix</CODE>
argument to change the default installation directory. The above
examples install the zlib, HDF5, and netCDF-4 libraries in
/home/ed/local/lib, the header file in /home/ed/local/include, and the
utilities in /home/ed/local/bin. If you don't provide a --prefix
utilities in /home/ed/local/bin. If you don't provide a <CODE>--prefix</CODE>
option, installation will be in /usr/local/, in subdirectories lib/,
include/, and bin/. The installation location specified with the
--prefix option must be different from the source directory where the
<CODE>--prefix</CODE> option must be different from the source directory where the
software is being built.
\page build_classic Building NetCDF with Classic Library Only
@ -163,7 +163,7 @@ make check install
(Replace ``/home/ed/local'' with the name of the directory where
netCDF is to be installed. The installation location specified with
the --prefix option must be different from the source directory where
the <CODE>--prefix</CODE> option must be different from the source directory where
the software is being built.)
Starting with version 4.1.1 the netCDF C libraries and utilities have
@ -248,8 +248,8 @@ If the HDF5 used by netCDF has been built with parallel I/O, then
netCDF will also be built with support for parallel I/O. This allows
parallel I/O access to netCDF-4/HDF5 files. Note that shared libraries
are not supported for parallel HDF5, which makes linking more
difficult to get right. For some reason, "LIBS=-ldl" seems to
sometimes be needed to link successfully with parallel HDF5 libraries.
difficult to get right. "LIBS=-ldl" is also sometimes needed to link
successfully with parallel HDF5 libraries.
(See /ref netcdf_formats for more information about the netCDF format
variants.)

113
nc_test/large_files.c
View File

@ -37,7 +37,7 @@ main(int argc, char **argv) {
char file_name[NC_MAX_NAME + 1];
int ncid; /* netCDF id */
int rec, i, j, k;
signed char x[] = {42, 21};
int x[] = {42, 21};
/* dimension ids */
int rec_dim;
@ -47,8 +47,9 @@ main(int argc, char **argv) {
int n_dim;
#define NUMRECS 2
#define I_LEN 4106
#define J_LEN 1023
#define I_LEN 5
//#define J_LEN 214700000
#define J_LEN 500000000
#define K_LEN 1023
#define N_LEN 2
@ -64,7 +65,7 @@ main(int argc, char **argv) {
int x_id;
/* rank (number of dimensions) for each variable */
# define RANK_var1 4
# define RANK_var1 3
# define RANK_x 2
/* variable shapes */
@ -96,14 +97,14 @@ main(int argc, char **argv) {
x_dims[0] = rec_dim;
x_dims[1] = n_dim;
stat = nc_def_var(ncid, "x", NC_BYTE, RANK_x, x_dims, &x_id);
stat = nc_def_var(ncid, "x", NC_INT, RANK_x, x_dims, &x_id);
check_err(stat,__LINE__,__FILE__);
var1_dims[0] = rec_dim;
var1_dims[1] = i_dim;
var1_dims[2] = j_dim;
var1_dims[3] = k_dim;
stat = nc_def_var(ncid, "var1", NC_BYTE, RANK_var1, var1_dims, &var1_id);
//var1_dims[3] = k_dim;
stat = nc_def_var(ncid, "var1", NC_FLOAT, RANK_var1, var1_dims, &var1_id);
check_err(stat,__LINE__,__FILE__);
/* don't initialize variables with fill values */
@ -115,30 +116,26 @@ main(int argc, char **argv) {
check_err(stat,__LINE__,__FILE__);
{ /* store var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static size_t var1_start[RANK_var1] = {0, 0, 0, 0};
static size_t var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static size_t x_start[RANK_x] = {0, 0};
static size_t x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
var1[j][k] = (signed char) n;
n++;
n %= 256;
}
}
var1_start[1] = i;
stat = nc_put_vara_schar(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
}
stat = nc_put_vara_schar(ncid, x_id, x_start, x_count, x);
check_err(stat,__LINE__,__FILE__);
static float var1[J_LEN];
static size_t var1_start[RANK_var1] = {0, 0, 0};
static size_t var1_count[RANK_var1] = {1, 1, J_LEN};
static size_t x_start[RANK_x] = {0, 0};
static size_t x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
for(j=0; j<J_LEN; j++) {
var1[j] = (float)(j + (rec+1) * i);
}
var1_start[1] = i;
stat = nc_put_vara_float(ncid, var1_id, var1_start, var1_count, var1);
check_err(stat,__LINE__,__FILE__);
}
x[0] += rec; x[1] += rec;
stat = nc_put_vara_int(ncid, x_id, x_start, x_count, x);
check_err(stat,__LINE__,__FILE__);
}
}
stat = nc_close(ncid);
@ -151,37 +148,33 @@ main(int argc, char **argv) {
check_err(stat,__LINE__,__FILE__);
{ /* read var1 */
int n = 0;
static signed char var1[J_LEN][K_LEN];
static size_t var1_start[RANK_var1] = {0, 0, 0, 0};
static size_t var1_count[RANK_var1] = {1, 1, J_LEN, K_LEN};
static size_t x_start[RANK_x] = {0, 0};
static size_t x_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
var1_start[0] = rec;
x_start[0] = rec;
for(i=0; i<I_LEN; i++) {
var1_start[1] = i;
stat = nc_get_vara_schar(ncid, var1_id, var1_start, var1_count, &var1[0][0]);
check_err(stat,__LINE__,__FILE__);
for(j=0; j<J_LEN; j++) {
for (k=0; k<K_LEN; k++) {
if (var1[j][k] != (signed char) n) {
printf("Error on read, var1[%d, %d, %d, %d] = %d wrong, "
"should be %d !\n", rec, i, j, k, var1[j][k], (signed char) n);
return 1;
}
n++;
n %= 256;
}
}
}
nc_get_vara_schar(ncid, x_id, x_start, x_count, x);
if(x[0] != 42 || x[1] != 21) {
printf("Error on read, x[] = %d, %d\n", x[0], x[1]);
return 1;
}
static float avar1[J_LEN];
static size_t avar1_start[RANK_var1] = {0, 0, 0};
static size_t avar1_count[RANK_var1] = {1, 1, J_LEN};
static size_t ax_start[RANK_x] = {0, 0};
static size_t ax_count[RANK_x] = {1, N_LEN};
for(rec=0; rec<NUMRECS; rec++) {
avar1_start[0] = rec;
ax_start[0] = rec;
for(i=0; i<I_LEN; i++) {
avar1_start[1] = i;
stat = nc_get_vara_float(ncid, var1_id, avar1_start, avar1_count, avar1);
check_err(stat,__LINE__,__FILE__);
for(j=0; j<J_LEN; j++)
{
if (avar1[j] != (float)(j + (rec + 1) * i)) {
printf("Error on read, var1[%d, %d, %d] = %g wrong, "
"should be %g !\n", rec, i, j, avar1[j], (float) (j + (rec + 1)* i));
return 1;
}
}
}
nc_get_vara_int(ncid, x_id, ax_start, ax_count, x);
if(x[0] != (42 + rec) || x[1] != (21+rec)) {
printf("Error on read, x[] = %d, %d\n", x[0], x[1]);
return 1;
}
}
}
stat = nc_close(ncid);
check_err(stat,__LINE__,__FILE__);