mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2024-11-21 03:13:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

merged master

Browse Source
This commit is contained in:
Ed Hartnett 2018-07-23 13:18:21 -06:00
commit 12b76d2dc5
41 changed files with 470 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
CMakeLists.txt
View File

@ -761,12 +761,6 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
ENDIF(USE_HDF5 OR ENABLE_NETCDF_4)
# Option to turn on CDF5 support.
OPTION(ENABLE_CDF5 "Enable CDF5 Support." ON)
IF(ENABLE_CDF5)
SET(USE_CDF5 ON CACHE BOOL "")
ENDIF(ENABLE_CDF5)
# Option to Build DAP2+DAP4 Clients
OPTION(ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
IF(ENABLE_DAP)
@ -1292,6 +1286,26 @@ CHECK_TYPE_SIZE("_Bool" SIZEOF__BOOL)
CHECK_TYPE_SIZE("size_t" SIZEOF_SIZE_T)
# Check whether to turn on or off CDF5 support.
SET(ENABLE_CDF5 AUTO CACHE STRING "AUTO")
OPTION(ENABLE_CDF5 "Enable CDF5 support" ON)
IF(SIZEOF_SIZE_T EQUAL 4)
IF(ENABLE_CDF5) # enable or auto
STRING(TOUPPER ${ENABLE_CDF5} ENABLE_CDF5)
IF(ENABLE_CDF5 AND NOT ENABLE_CDF5 STREQUAL "AUTO") # explicitly enabled
MESSAGE(FATAL_ERROR "Unable to support CDF5 feature because size_t is less than 8 bytes")
ENDIF(ENABLE_CDF5 AND NOT ENABLE_CDF5 STREQUAL "AUTO")
SET(ENABLE_CDF5 OFF) # cannot support CDF5
SET(USE_CDF5 OFF CACHE BOOL "") # cannot support CDF5
ENDIF(ENABLE_CDF5)
ELSE(SIZEOF_SIZE_T EQUAL 4)
IF(ENABLE_CDF5) # explicitly set by user or not set
SET(USE_CDF5 ON CACHE BOOL "")
ELSE(ENABLE_CDF5) # explicitly disabled by user
SET(USE_CDF5 OFF CACHE BOOL "")
ENDIF(ENABLE_CDF5)
ENDIF(SIZEOF_SIZE_T EQUAL 4)
CHECK_TYPE_SIZE("ssize_t" SIZEOF_SSIZE_T)
IF(SIZEOF_SSIZE_T)
SET(HAVE_SSIZE_T TRUE)

1
config.h.cmake.in
View File

@ -109,7 +109,6 @@ are set when opening a binary file on Windows. */
/* if true, enable CDF5 Support */
#cmakedefine ENABLE_CDF5 1
#cmakedefine USE_CDF5 1
/* if true, enable strict null byte header padding. */
#cmakedefine USE_STRICT_NULL_BYTE_HEADER_PADDING 1

19
configure.ac
View File

@ -896,29 +896,28 @@ $SLEEPCMD
AC_CHECK_SIZEOF(unsigned long long)
# Check whether we want to enable CDF5 support.
AC_MSG_CHECKING([whether CDF5 support should be enabled])
AC_MSG_CHECKING([whether CDF5 support should be disabled])
AC_ARG_ENABLE([cdf5],
[AS_HELP_STRING([--enable-cdf5],
[AS_HELP_STRING([--disable-cdf5],
[build without CDF5 support.])],
[enable_cdf5=${enableval}], [enable_cdf5=auto]
)
if test "x${enable_cdf5}" = xyes && test "$ac_cv_sizeof_size_t" -lt "8" ; then
dnl unable to support CDF5, but --enable-cdf5 is explicitly set
AC_MSG_ERROR([Unable to support CDF5 feature because size_t is less than 4 bytes])
fi
if test "$ac_cv_sizeof_size_t" -lt "8" ; then
if test "x${enable_cdf5}" = xyes ; then
dnl unable to support CDF5, but --enable-cdf5 is explicitly set
AC_MSG_ERROR([Unable to support CDF5 feature because size_t is less than 8 bytes])
fi
enable_cdf5=no
else
enable_cdf5=yes
if test "x${enable_cdf5}" != xno ; then
enable_cdf5=yes
fi
fi
AC_MSG_RESULT($enable_cdf5)
if test "x${enable_cdf5}" = xyes; then
AC_DEFINE([USE_CDF5], [1], [if true, enable CDF5 Support])
AC_DEFINE([ENABLE_CDF5], [1], [if true, enable CDF5 Support])
fi
AM_CONDITIONAL(USE_CDF5, [test x$enable_cdf5 = xyes ])
AM_CONDITIONAL(ENABLE_CDF5, [test x$enable_cdf5 = xyes ])
$SLEEPCMD

147
docs/FAQ.md
View File

@ -547,6 +547,7 @@ There are four netCDF format variants:
- the classic format
- the 64-bit offset format
- the 64-bit data format
- the netCDF-4 format
- the netCDF-4 classic model format
@ -556,13 +557,13 @@ though these are not usually thought of as formats: CDL and NcML.)
The **classic format** was the only format for netCDF data created
between 1989 and 2004 by the reference software from Unidata. It is
still the default format for new netCDF data files, and the form in
which most netCDF data is stored.
which most netCDF data is stored. This format is also referred as CDF-1 format.
In 2004, the **64-bit offset format** variant was added. Nearly
identical to netCDF classic format, it allows users to create and access
far larger datasets than were possible with the original format. (A
64-bit platform is not required to write or read 64-bit offset netCDF
files.)
files.) This format is also referred as CDF-2 format.
In 2008, the **netCDF-4 format** was added to support per-variable
compression, multiple unlimited dimensions, more complex data types, and
@ -574,6 +575,14 @@ format**, was added for users who needed the performance benefits of the
new format (such as compression) without the complexity of a new
programming interface or enhanced data model.
In 2016, the **64-bit data format** variant was added. To support large
variables with more than 4-billion array elements, it replaces most of the
32-bit integers used in the format specification with 64-bit integers. It also
adds support for several new data types including unsigned byte, unsigned
short, unsigned int, signed 64-bit int and unsigned 64-bit int. A 64-bit
platform is required to write or read 64-bit data netCDF files. This format is
also referred as CDF-5 format.
With each additional format variant, the C-based reference software from
Unidata has continued to support access to data stored in previous
formats transparently, and to also support programs written using
@ -581,15 +590,18 @@ previous programming interfaces.
Although strictly speaking, there is no single "netCDF-3 format", that
phrase is sometimes used instead of the more cumbersome but correct
"netCDF classic or 64-bit offset format" to describe files created by
the netCDF-3 (or netCDF-1 or netCDF-2) libraries. Similarly "netCDF-4
format" is sometimes used informally to mean "either the general
netCDF-4 format or the restricted netCDF-4 classic model format". We
will use these shorter phrases in FAQs below when no confusion is
likely.
"netCDF classic CDF-1, 64-bit offset CDF-2, or 64-bit data CDF-5 format" to
describe files created by the netCDF-3 (or netCDF-1 or netCDF-2) libraries.
Similarly "netCDF-4 format" is sometimes used informally to mean "either the
general netCDF-4 format or the restricted netCDF-4 classic model format". We
will use these shorter phrases in FAQs below when no confusion is likely.
A more extensive description of the netCDF formats and a formal
specification of the classic and 64-bit formats is available as a [NASA ESDS community standard](https://earthdata.nasa.gov/sites/default/files/esdswg/spg/rfc/esds-rfc-011/ESDS-RFC-011v2.00.pdf).
A more extensive description of the netCDF formats and a formal specification
of the classic and 64-bit formats is available as a [NASA ESDS community
standard](https://earthdata.nasa.gov/sites/default/files/esdswg/spg/rfc/esds-rfc-011/ESDS-RFC-011v2.00.pdf).
The 64-bit data CDF-5 format specification is available in
http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html.
How can I tell which format a netCDF file uses? {#How-can-I-tell-which-format-a-netCDF-file-uses}
-----------------
@ -598,8 +610,9 @@ How can I tell which format a netCDF file uses? {#How-can-I-tell-which-format-a-
The short answer is that under most circumstances, you should not care,
if you use version 4.0 or later of the netCDF library to access data in
the file. But the difference is indicated in the first four bytes of the
file, which are 'C', 'D', 'F', '\\001' for the classic netCDF format;
'C', 'D', 'F', '\\002' for the 64-bit offset format; or '\\211', 'H',
file, which are 'C', 'D', 'F', '\\001' for the classic netCDF CDF-1 format;
'C', 'D', 'F', '\\002' for the 64-bit offset CDF-2 format;
'C', 'D', 'F', '\\005' for the 64-bit data CDF-5 format; or '\\211', 'H',
'D', 'F' for an HDF5 file, which could be either a netCDF-4 file or a
netCDF-4 classic model file. (HDF5 files may also begin with a
user-block of 512, 1024, 2048, ... bytes before what is actually an
@ -635,22 +648,37 @@ which will output
C D F 002
~~~~
~~~~ {.boldcode}
C D F 005
~~~~
~~~~ {.boldcode}
211 H D F
~~~~
depending on whether foo.nc is a classic, 64-bit offset, or netCDF-4
file, respectively. This method cannot be used to distinguish between
netCDF-4 and netCDF-4 classic model variants, or between a netCDF-4 file
and a different kind of HDF5 file.
depending on whether foo.nc is a classic CDF-1, 64-bit offset CDF-2, 64-bit
data CDF-5, or netCDF-4 file, respectively. This method cannot be used to
distinguish between netCDF-4 and netCDF-4 classic model variants, or between a
netCDF-4 file and a different kind of HDF5 file.
----------
How many netCDF data models are there? {#How-many-netCDF-data-models-are-there}
-----------------
There are only two netCDF data models, the [classic model](/netcdf/workshops/2008/datamodel/NcClassicModel.html) and the [enhanced model](/netcdf/workshops/2008/netcdf4/Nc4DataModel.html) (also called the netCDF-4 data model). The classic model is the simpler of the two, and is used for all data stored in classic format, 64-bit offset format, or netCDF-4 classic model format. The enhanced model (sometimes also referred to as the netCDF-4 data model) is an extension of the classic model that adds more powerful forms of data representation and
data types at the expense of some additional complexity. Although data represented with the classic model can also be represented using the enhanced model, datasets that use enhanced model features, such as user-defined data types, cannot be represented with the classic model. Use of the enhanced model requires storage in the netCDF-4 format.
There are only two netCDF data models, the [classic
model](/netcdf/workshops/2008/datamodel/NcClassicModel.html) and the [enhanced
model](/netcdf/workshops/2008/netcdf4/Nc4DataModel.html) (also called the
netCDF-4 data model). The classic model is the simpler of the two, and is used
for all data stored in classic CDF-1 format, 64-bit offset CDF-2 format, 64-bit
data CDF-5 format, or netCDF-4 classic model format. The enhanced model
(sometimes also referred to as the netCDF-4 data model) is an extension of the
classic model that adds more powerful forms of data representation and data
types at the expense of some additional complexity. Although data represented
with the classic model can also be represented using the enhanced model,
datasets that use enhanced model features, such as user-defined data types,
cannot be represented with the classic model. Use of the enhanced model
requires storage in the netCDF-4 format.
How many releases of the C-based netCDF software are supported? {#How-many-releases-of-the-C-based-netCDF-software-are-supported}
-----------------
@ -664,15 +692,17 @@ the netCDF-4 and netCDF-4 classic model formats, if built using a
previously installed HDF5 library and using the "--enable-netcdf-4"
configure option. Software built from the netCDF-4.0 release without
specifying "--enable-netcdf-4" (the default) was identical to software
built with netCDF-3.6.3.
built with netCDF-3.6.3. Starting from version 4.4.0, netCDF added support
for CDF-5 format.
Both netCDF-3 and netCDF-4 C libraries are part of a single software
release. The netCDF software may be built to support just the classic
and 64-bit offset formats (the default) or to also support the netCDF-4
and netCDF-4 classic model formats, if the HDF5-1.8.x library is
installed. Unidata no longer supports a separate netCDF-3-only version
of the software, but instead supports both the classic and enhanced data
models and all four format variants in a single source distribution.
CDF-1 and 64-bit offset CDF-2 formats (the default), 64-bit data CDF-5 format,
or to also support the netCDF-4 and netCDF-4 classic model formats, if the
HDF5-1.8.x library is installed. Unidata no longer supports a separate
netCDF-3-only version of the software, but instead supports both the classic
and enhanced data models and all four format variants in a single source
distribution.
This does not indicate any plan to drop support for netCDF-3 or the
formats associated with netCDF-3. Support for earlier formats and APIs
@ -681,14 +711,13 @@ will continue with all future versions of netCDF software from Unidata.
Should I get netCDF-3 or netCDF-4? {#Should-I-get-netCDF-3-or-netCDF-4}
-----------------
By downloading a current version of netCDF-4, you have the choice to
build either
- the default netCDF-3 libraries, which support classic and 64-bit
offset formats, and the classic data model; or
- the default netCDF-3 libraries, which support classic CDF-1, 2, and 5
formats, and the classic data model; or
- the netCDF-4 libraries, which support netCDF-4 and netCDF-4 classic
model formats, as well as classic and 64-bit offset formats, and the
model formats, as well as classic formats, and the
enhanced data model.
Which version to build depends on how you will use the software.
@ -696,8 +725,8 @@ Which version to build depends on how you will use the software.
Installing the simpler netCDF-3 version of the software is recommended
if the following situations apply:
- all the data you need to access is available in netCDF classic or
64-bit offset formats
- all the data you need to access is available in netCDF classic
formats
- you are installing netCDF in order to support another software
package that uses only netCDF-3 features
- you plan to only write data in a form that netCDF-3 software and
@ -845,13 +874,12 @@ How can I convert netCDF-3 files into netCDF-4 files? {#How-can-I-convert-netCDF
-----------------
Every netCDF-3 file can be read or written by a netCDF version 4
library, so in that respect netCDF-3 files are already netCDF-4 files
and need no conversion. But if you want to convert a classic or 64-bit
offset format file into a netCDF-4 format or netCDF-4 classic model
format file, the easiest way is to use the **nccopy** utility. For example
to convert a classic format file foo3.nc to a netCDF-4 format file
foo4.nc, use:
Every netCDF-3 file can be read or written by a netCDF version 4 library, so in
that respect netCDF-3 files are already netCDF-4 files and need no conversion.
But if you want to convert a classic format file (CDF-1, 2, or 5) into a
netCDF-4 format or netCDF-4 classic model format file, the easiest way is to
use the **nccopy** utility. For example to convert a classic format file
foo3.nc to a netCDF-4 format file foo4.nc, use:
~~~~~~~~~~~~~~~~~~~~~~~~~ {.boldcode}
nccopy -k netCDF-4 foo3.nc foo4.nc
@ -872,10 +900,9 @@ utility "ncks" can be used to accomplish the same task, as follows:
~~~~~~~~~~~~~~~~~~~~~~~~
Another method is available for relatively small files, using the **ncdump**
and **ncgen** utilities (built with a netCDF-4 library). Assuming
"small3.nc" is a small classic format or 64-bit offset format netCDF
file, you can create an equivalent netCDF-4 file named
"small4.nc" as follows:
and **ncgen** utilities (built with a netCDF-4 library). Assuming "small3.nc"
is a small classic format netCDF file, you can create an equivalent netCDF-4
file named "small4.nc" as follows:
~~~~ {.boldcode}
ncdump small3.nc > small.cdl
@ -1219,11 +1246,15 @@ reference to the format label at the start of a file.
With netCDF version 3.6 and later, a second variant of netCDF format is
supported in addition to the classic format. The new variant is referred
to as the *64-bit offset* format, *version 2*, or *CDF2*. The primary
to as the *64-bit offset* format, *version 2*, or *CDF-2*. The primary
difference from the classic format is the use of 64-bit file offsets
instead of 32-bit offsets, but it also supports larger variable and
record sizes.
Starting from version 4.4.0, netCDF added support for CDF-5 format, which
allows multiple large variables with more than 4-billion array elements defined
in the file. This format is only supported on 64-bit machine platforms.
----------
Do I have to know which netCDF file format variant is used in order to access or modify a netCDF file? {#Do-I-have-to-know-which-netCDF-file-format-variant-is-used-in-order-to-access-or-modify-a-netCDF-file}
@ -1235,10 +1266,10 @@ detect which variant of the format is used for each file when it is
opened for reading or writing, so it is not necessary to know which
variant of the format is used. The version of the format will be
preserved by the library on writing. If you want to modify a classic
format file to use the 64-bit offset format so you can make it much
format file to use the CDF-2 or CDF-5 format so you can make it much
larger, you will have to create a new file and copy the data to it. The
**nccopy** utility available in version 4.1 can copy a classic file to a
64-bit offset file.
CDF-2 or CDF-5 file.
----------
@ -1247,13 +1278,13 @@ Will future versions of the netCDF library continue to support accessing files i
Yes, the 3.6 library and all planned future versions of the library will
continue to support reading and writing files using the classic (32-bit
offset) format as well as the 64-bit offset format. There is no need to
continue to support reading and writing files using the classic CDF-1 (32-bit
offset), 64-bit offset CDF-2, and 64-bit data CDF-5 format. There is no need to
convert existing archives from the classic to the 64-bit offset format.
Even netCDF-4, which introduces a third variant of the netCDF format
based on HDF5, continues to support accessing classic format netCDF
files as well as 64-bit offset netCDF files. NetCDF-4 HDF5 files have
even fewer restrictions on size than 64-bit offset netCDF files.
based on HDF5, continues to support accessing classic CDF-1, 2, and 5 format
files. NetCDF-4 HDF5 files have even fewer restrictions on size than CDF-1 and
CDF-2 files.
----------
@ -1272,17 +1303,17 @@ the classic format.
----------
How can I tell if a netCDF file uses the classic format or 64-bit offset format? {#How-can-I-tell-if-a-netCDF-file-uses-the-classic-format-or-64-bit-offset-format}
How can I tell if a netCDF file uses the classic format (CDF-1), 64-bit offset format (CDF-2) or 64-bit data format (CDF-5)? {#How-can-I-tell-if-a-netCDF-file-uses-the-classic-format-or-64-bit-offset-format}
-----------------
The short answer is that under most circumstances, you should not care,
if you use version 3.6.0 or later of the netCDF library. But the
difference is indicated in the first four bytes of the file, which are
'C', 'D', 'F', '\\001' for the classic netCDF format and 'C', 'D', 'F',
'\\002' for the 64-bit offset format. On a Unix system, one way to
display the first four bytes of a file, say foo.nc, is to run the
following command:
'C', 'D', 'F', '\\001' for the classic CDF-1 format, 'C', 'D', 'F',
'\\002' for the 64-bit offset CDF-2 format, and 'C', 'D', 'F', '\\005' for the
64-bit data CDF-5 format. On a Unix system, one way to display the first four
bytes of a file, say foo.nc, is to run the following command:
~~~~ {.boldcode}
od -An -c -N4 foo.nc
@ -1300,7 +1331,13 @@ or
C D F 002
~~~~
depending on whether foo.nc is a classic or 64-bit offset netCDF file,
or
~~~~ {.boldcode}
C D F 005
~~~~
depending on whether foo.nc is a CDF-1, CDF-2, or CDF-5 netCDF file,
respectively.
With netCDF version 3.6.2 or later, there is an easier way, using the

159
docs/all-error-codes.md
View File

@ -42,31 +42,7 @@ NetCDF Error Code Listing {#nc-error-codes}
#define NC_EVARSIZE (-62) // One or more variable sizes violate format constraints
#define NC_EDIMSIZE (-63) // Invalid dimension size
#define NC_ETRUNC (-64) // File likely truncated or possibly corrupted
~~~~
# NetCDF-4 Error Codes {#nc4-error-codes}
NetCDF-4 uses all error codes from NetCDF-3 (see section [NetCDF-3 Error
Codes](#NetCDF_002d3-Error-Codes)). The following additional error codes
were added for new errors unique to netCDF-4.
~~~~
#define NC_EHDFERR (-101)
#define NC_ECANTREAD (-102)
#define NC_ECANTWRITE (-103)
#define NC_ECANTCREATE (-104)
#define NC_EFILEMETA (-105)
#define NC_EDIMMETA (-106)
#define NC_EATTMETA (-107)
#define NC_EVARMETA (-108)
#define NC_ENOCOMPOUND (-109)
#define NC_EATTEXISTS (-110)
#define NC_ENOTNC4 (-111) // Attempting netcdf-4 operation on netcdf-3 file.
#define NC_ESTRICTNC3 (-112) // Attempting netcdf-4 operation on strict nc3 netcdf-4 file.
#define NC_EBADGRPID (-113) // Bad group id. Bad!
#define NC_EBADTYPEID (-114) // Bad type id.
#define NC_EBADFIELDID (-115) // Bad field id.
#define NC_EUNKNAME (-116)
#define NC_EAXISTYPE (-65) // Unknown axis type
~~~~
# DAP Error Codes {#dap-error-codes}
@ -85,14 +61,127 @@ may occur.
#define NC_EDATADDS (-73) // Malformed or inaccessible DATADDS
#define NC_EDAPURL (-74) // Malformed DAP URL
#define NC_EDAPCONSTRAINT (-75) // Malformed DAP Constraint
#define NC_EDAP (-66) // Generic DAP error
#define NC_ECURL (-67) // Generic libcurl error
#define NC_EIO (-68) // Generic IO error
#define NC_ENODATA (-69) // Attempt to access variable with no data
#define NC_EDAPSVC (-70) // DAP Server side error
#define NC_EDAS (-71) // Malformed or inaccessible DAS
#define NC_EDDS (-72) // Malformed or inaccessible DDS
#define NC_EDATADDS (-73) // Malformed or inaccessible DATADDS
#define NC_EDAPURL (-74) // Malformed DAP URL
#define NC_EDAPCONSTRAINT (-75) // Malformed DAP Constraint
#define NC_ETRANSLATION (-76) // Untranslatable construct
#define NC_EACCESS (-77) // Access Failure
#define NC_EAUTH (-78) // Authorization Failure
~~~~
# Misc. additional errors
~~~~
#define NC_ENOTFOUND (-90) // No such file
#define NC_ECANTREMOVE (-91) // Cannot remove file
#define NC_EINTERNAL (-92) // NetCDF Library Internal Error
~~~~
# NetCDF-4 Error Codes {#nc4-error-codes}
NetCDF-4 uses all error codes from NetCDF-3 (see section [NetCDF-3 Error
Codes](#NetCDF_002d3-Error-Codes)). The following additional error codes
were added for new errors unique to netCDF-4.
~~~~
#define NC_EHDFERR (-101) // Error at HDF5 layer.
#define NC_ECANTREAD (-102) // Cannot read.
#define NC_ECANTWRITE (-103) // Cannot write.
#define NC_ECANTCREATE (-104) // Cannot create.
#define NC_EFILEMETA (-105) // Problem with file metadata.
#define NC_EDIMMETA (-106) // Problem with dimension metadata.
#define NC_EATTMETA (-107) // Problem with attribute metadata.
#define NC_EVARMETA (-108) // Problem with variable metadata.
#define NC_ENOCOMPOUND (-109) // Not a compound type.
#define NC_EATTEXISTS (-110) // Attribute already exists.
#define NC_ENOTNC4 (-111) // Attempting netcdf-4 operation on netcdf-3 file.
#define NC_ESTRICTNC3 (-112) // Attempting netcdf-4 operation on strict nc3 netcdf-4 file.
#define NC_ENOTNC3 (-113) // Attempting netcdf-3 operation on netcdf-4 file.
#define NC_ENOPAR (-114) // Parallel operation on file opened for non-parallel access.
#define NC_EPARINIT (-115) // Error initializing for parallel access.
#define NC_EBADGRPID (-116) // Bad group ID.
#define NC_EBADTYPID (-117) // Bad type ID.
#define NC_ETYPDEFINED (-118) // Type has already been defined and may not be edited.
#define NC_EBADFIELD (-119) // Bad field ID.
#define NC_EBADCLASS (-120) // Bad class.
#define NC_EMAPTYPE (-121) // Mapped access for atomic types only.
#define NC_ELATEFILL (-122) // Attempt to define fill value when data already exists.
#define NC_ELATEDEF (-123) // Attempt to define var properties, like deflate, after enddef.
#define NC_EDIMSCALE (-124) // Problem with HDF5 dimscales.
#define NC_ENOGRP (-125) // No group found.
#define NC_ESTORAGE (-126) // Cannot specify both contiguous and chunking.
#define NC_EBADCHUNK (-127) // Bad chunksize.
#define NC_ENOTBUILT (-128) // Attempt to use feature that was not turned on when netCDF was built.
#define NC_EDISKLESS (-129) // Error in using diskless access.
#define NC_ECANTEXTEND (-130) // Attempt to extend dataset during ind. I/O operation.
#define NC_EMPI (-131) // MPI operation failed.
#define NC_EFILTER (-132) // Filter operation failed.
#define NC_ERCFILE (-133) // RC file failure
#define NC_ENULLPAD (-134) // Header Bytes not Null-Byte padded
#define NC_EINMEMORY (-135) // In-memory file error
~~~~
# PnetCDF Error Codes {#pnetcdf-error-codes}
~~~~
#define NC_ESMALL (-201) // size of MPI_Offset too small for format
#define NC_ENOTINDEP (-202) // Operation not allowed in collective data mode
#define NC_EINDEP (-203) // Operation not allowed in independent data mode
#define NC_EFILE (-204) // Unknown error in file operation
#define NC_EREAD (-205) // Unknown error in reading file
#define NC_EWRITE (-206) // Unknown error in writing to file
#define NC_EOFILE (-207) // file open/creation failed
#define NC_EMULTITYPES (-208) // Multiple etypes used in MPI datatype
#define NC_EIOMISMATCH (-209) // Input/Output data amount mismatch
#define NC_ENEGATIVECNT (-210) // Negative count is specified
#define NC_EUNSPTETYPE (-211) // Unsupported etype in memory MPI datatype
#define NC_EINVAL_REQUEST (-212) // invalid nonblocking request ID
#define NC_EAINT_TOO_SMALL (-213) // MPI_Aint not large enough to hold requested value
#define NC_ENOTSUPPORT (-214) // feature is not yet supported
#define NC_ENULLBUF (-215) // trying to attach a NULL buffer
#define NC_EPREVATTACHBUF (-216) // previous attached buffer is found
#define NC_ENULLABUF (-217) // no attached buffer is found
#define NC_EPENDINGBPUT (-218) // pending bput is found, cannot detach buffer
#define NC_EINSUFFBUF (-219) // attached buffer is too small
#define NC_ENOENT (-220) // File does not exist
#define NC_EINTOVERFLOW (-221) // Overflow when type cast to 4-byte integer
#define NC_ENOTENABLED (-222) // feature is not enabled
#define NC_EBAD_FILE (-223) // Invalid file name (e.g., path name too long)
#define NC_ENO_SPACE (-224) // Not enough space
#define NC_EQUOTA (-225) // Quota exceeded
#define NC_ENULLSTART (-226) // argument start is a NULL pointer
#define NC_ENULLCOUNT (-227) // argument count is a NULL pointer
#define NC_EINVAL_CMODE (-228) // Invalid file create mode
#define NC_ETYPESIZE (-229) // MPI derived data type size error (bigger than the variable size)
#define NC_ETYPE_MISMATCH (-230) // element type of the MPI derived data type mismatches the variable type
#define NC_ETYPESIZE_MISMATCH (-231) // file type size mismatches buffer type size
#define NC_ESTRICTCDF2 (-232) // Attempting CDF-5 operation on CDF-2 file
#define NC_ENOTRECVAR (-233) // Attempting operation only for record variables
#define NC_ENOTFILL (-234) // Attempting to fill a variable when its fill mode is off
#define NC_EINVAL_OMODE (-235) // Invalid file open mode
#define NC_EPENDING (-236) // Pending nonblocking request is found at file close
#define NC_EMAX_REQ (-237) // Size of I/O request exceeds INT_MAX
#define NC_EBADLOG (-238) // Unrecognized log file format
#define NC_EFLUSHED (-239) // Nonblocking request has already been flushed to the PFS. It is too late to cancel
#define NC_EMULTIDEFINE (-250) // NC definitions inconsistent among processes
#define NC_EMULTIDEFINE_OMODE (-251) // inconsistent file open modes among processes
#define NC_EMULTIDEFINE_DIM_NUM (-252) // inconsistent number of dimensions
#define NC_EMULTIDEFINE_DIM_SIZE (-253) // inconsistent size of dimension
#define NC_EMULTIDEFINE_DIM_NAME (-254) // inconsistent dimension names
#define NC_EMULTIDEFINE_VAR_NUM (-255) // inconsistent number of variables
#define NC_EMULTIDEFINE_VAR_NAME (-256) // inconsistent variable name
#define NC_EMULTIDEFINE_VAR_NDIMS (-257) // inconsistent variable number of dimensions
#define NC_EMULTIDEFINE_VAR_DIMIDS (-258) // inconsistent variable dimension IDs
#define NC_EMULTIDEFINE_VAR_TYPE (-259) // inconsistent variable data type
#define NC_EMULTIDEFINE_VAR_LEN (-260) // inconsistent variable size
#define NC_EMULTIDEFINE_NUMRECS (-261) // inconsistent number of records
#define NC_EMULTIDEFINE_VAR_BEGIN (-262) // inconsistent variable file begin offset (internal use)
#define NC_EMULTIDEFINE_ATTR_NUM (-263) // inconsistent number of attributes
#define NC_EMULTIDEFINE_ATTR_SIZE (-264) // inconsistent memory space used by attribute (internal use)
#define NC_EMULTIDEFINE_ATTR_NAME (-265) // inconsistent attribute name
#define NC_EMULTIDEFINE_ATTR_TYPE (-266) // inconsistent attribute type
#define NC_EMULTIDEFINE_ATTR_LEN (-267) // inconsistent attribute length
#define NC_EMULTIDEFINE_ATTR_VAL (-268) // inconsistent attribute value
#define NC_EMULTIDEFINE_FNC_ARGS (-269) // inconsistent function arguments used in collective API
#define NC_EMULTIDEFINE_FILL_MODE (-270) // inconsistent dataset fill mode
#define NC_EMULTIDEFINE_VAR_FILL_MODE (-271) // inconsistent variable fill mode
#define NC_EMULTIDEFINE_VAR_FILL_VALUE (-272) // inconsistent variable fill value
#define NC_EMULTIDEFINE_CMODE (-273) // inconsistent file create modes among processes
~~~~

224
docs/guide.dox
View File

@ -115,19 +115,30 @@ is the default format for all versions of netCDF.
In version 3.6.0 a new binary format was introduced, 64-bit offset
format. Nearly identical to netCDF classic format, it uses 64-bit
offsets (hence the name), and allows users to create far larger
datasets.
datasets. This format is also referred as CDF-2, because it bears the signature
string "CDF2" in the file header. After this extension, the classic file
format (i.e. not supporting 64-bit offsets) is now referred as CDF-1.
In version 4.0.0 a third binary format was introduced: the HDF5
format. Starting with this version, the netCDF library can use HDF5
files as its base format. (Only HDF5 files created with netCDF-4 can
be understood by netCDF-4).
By default, netCDF uses the classic format. To use the 64-bit offset
or netCDF-4/HDF5 format, set the appropriate constant when creating
the file.
Starting from version 4.4.0, netCDF included the support of CDF-5 format. In
order to allows defining large array variables with more than 4-billion
elements, CDF-5 replaces most of the 32-bit integers used to describe metadata
in file header with 64-bit integers. In addition, it supports the following new
external data types: NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, and NC_UINT64. The
CDF-5 format specifications can be found in
(http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html).
The classic file formats are now referring to the collection of CDF-1, 2 and 5
formats. By default, netCDF uses the classic format (CDF-1). To use the CDF-2,
CDF-5, or netCDF-4/HDF5 format, set the appropriate constant in the file mode
argument when creating the file.
To achieve network-transparency (machine-independence), netCDF classic
and 64-bit offset formats are implemented in terms of an external
formats are implemented in terms of an external
representation much like XDR (eXternal Data Representation, see
http://www.ietf.org/rfc/rfc1832.txt), a standard for describing and
encoding data. This representation provides encoding of data into
@ -137,11 +148,10 @@ be encoded and decoded in a consistent way. The IEEE 754
floating-point standard is used for floating-point data
representation.
Descriptions of the overall structure of netCDF classic and 64-bit
offset files are provided later in this manual. See
\ref file_structure_and_performance.
Descriptions of the overall structure of netCDF classic files are provided
later in this manual. See \ref file_structure_and_performance.
The details of the classic and 64-bit offset formats are described in
The details of the CDF-1 and CDF-2 formats are described in
an appendix. See \ref netcdf_format. However, users are discouraged from
using the format specification to develop independent low-level
software for reading and writing netCDF files, because this could lead
@ -149,57 +159,70 @@ to compatibility problems if the format is ever modified.
\subsection select_format How to Select the Format
With three different base formats, care must be taken in creating data
With four different base formats, care must be taken in creating data
files to choose the correct base format.
The format of a netCDF file is determined at create time.
When opening an existing netCDF file the netCDF library will
transparently detect its format and adjust accordingly. However,
netCDF library versions earlier than 3.6.0 cannot read 64-bit offset
format files, and library versions before 4.0 can't read netCDF-4/HDF5
files. NetCDF classic format files (even if created by version 3.6.0
netCDF library versions earlier than 3.6.0 cannot read CDF-2
format files, library versions before 4.0 can't read netCDF-4/HDF5
files, and versions before 4.4.0 cannot read CDF-5 files.
NetCDF classic format files (even if created by version 3.6.0
or later) remain compatible with older versions of the netCDF library.
Users are encouraged to use netCDF classic format to distribute data,
Users are encouraged to use netCDF classic CDF-1 format to distribute data,
for maximum portability.
To select 64-bit offset or netCDF-4 format files, C programmers should
use flag NC_64BIT_OFFSET or NC_NETCDF4 in function nc_create().
To select CDF-2, CDF-5 or netCDF-4 format files, C programmers should use flag
NC_64BIT_OFFSET, NC_64BIT_DATA, or NC_NETCDF4 respectively in function
nc_create().
In Fortran, use flag nf_64bit_offset or nf_format_netcdf4 in function
NF_CREATE. See NF_CREATE.
In Fortran, use flag nf_64bit_offset, nf_64bit_data, or nf_format_netcdf4 in
function NF_CREATE. See NF_CREATE.
It is also possible to change the default creation format, to convert
a large body of code without changing every create call. C programmers
see nc_set_default_format(). Fortran programs see NF_SET_DEFAULT_FORMAT.
\subsection classic_format NetCDF Classic Format
\subsection classic_format NetCDF Classic Format (CDF-1)
The original netCDF format is identified using four bytes in the file
header. All files in this format have “CDF\001” at the beginning of
the file. In this documentation this format is referred to as “netCDF
classic format.”
header. All files in this format have "CDF\001" at the beginning of
the file. In this documentation this format is referred to as CDF-1
format.
NetCDF classic format is identical to the format used by every
NetCDF CDF-1 format is identical to the format used by every
previous version of netCDF. It has maximum portability, and is still
the default netCDF format.
\subsection netcdf_64bit_offset_format NetCDF 64-bit Offset Format (CDF-2)
For some users, the various 2 GiB format limitations of the classic
format become a problem. (see \ref limitations).
\subsection netcdf_64bit_offset_format NetCDF 64-bit Offset Format
For these users, 64-bit offset format is a natural choice. It greatly
eases the size restrictions of netCDF classic files (see \ref
limitations).
Files with the 64-bit offsets are identified with a “CDF\002” at the
Files with the 64-bit offsets are identified with a "CDF\002" at the
beginning of the file. In this documentation this format is called
“64-bit offset format.”
CDF-2 format.
Since 64-bit offset format was introduced in version 3.6.0, earlier
versions of the netCDF library can't read 64-bit offset files.
Since CDF-2 format was introduced in version 3.6.0, earlier
versions of the netCDF library can't read CDF-2 files.
\subsection netcdf_64bit_data_format NetCDF 64-bit Data Format (CDF-5)
To allow large variables with more than 4-billion array elements,
64-bit data format is develop to support such I/O requests.
Files with the 64-bit data are identified with a "CDF\005" at the
beginning of the file. In this documentation this format is called
CDF-5 format.
Since CDF-5 format was introduced in version 4.4.0, earlier
versions of the netCDF library can't read CDF-5 files.
\subsection netcdf_4_format NetCDF-4 Format
@ -303,17 +326,17 @@ its implementation in software. Some of these limitations have been
removed or relaxed in netCDF-4 files, but still apply to netCDF
classic and netCDF 64-bit offset files.
Currently, netCDF classic and 64-bit offset formats offer a limited
NetCDF classic CDF-1 and CDF-2 formats offer a limited
number of external numeric data types: 8-, 16-, 32-bit integers, or
32- or 64-bit floating-point numbers. (The netCDF-4 format adds 64-bit
integer types and unsigned integer types.)
32- or 64-bit floating-point numbers. The CDF-5 and netCDF-4 formats add 64-bit
integer types and unsigned integer types.
With the netCDF-4/HDF5 format, new unsigned integers (of various
sizes), 64-bit integers, and the string type allow improved expression
of meaning in scientific data. The new VLEN (variable length) and
COMPOUND types allow users to organize data in new ways.
With the classic netCDF file format, there are constraints that limit
With the classic CDF-1 file format, there are constraints that limit
how a dataset is structured to store more than 2 GiBytes (a GiByte is
2^30 or 1,073,741,824 bytes, as compared to a Gbyte, which is
1,000,000,000 bytes.) of data in a single netCDF dataset. (see \ref
@ -326,7 +349,7 @@ possible to create and access netCDF files larger than 2 GiB on
platforms that provide support for such files (see
\ref large_file_support).
The new 64-bit offset format allows large files, and makes it easy to
The CDF-2 format allows large files, and makes it easy to
create to create fixed variables of about 4 GiB, and record variables
of about 4 GiB per record. (see \ref netcdf_64bit_offset_format). However,
old netCDF applications will not be able to read the 64-bit offset
@ -338,7 +361,11 @@ and files can be as large as the underlying file system
supports. NetCDF-4/HDF5 files are unreadable to the netCDF library
before version 4.0.
Another limitation of the classic (and 64-bit offset) model is that
Similarly, CDF-5 format uses 64-bit integers to allow users to define
large variables. CDF-5 files are not unreadable to the netCDF library
before version 4.4.0.
Another limitation of the classic formats (CDF-1, 2 and 5) is that
only one unlimited (changeable) dimension is permitted for each netCDF
data set. Multiple variables can share an unlimited dimension, but
then they must all grow together. Hence the classic netCDF model does
@ -373,8 +400,8 @@ kinds of data). A data model tailored to capture the shared context
among researchers within one discipline may not be appropriate for
sharing or combining data from multiple disciplines.
The classic netCDF data model (which is used for classic-format and
64-bit offset format data) does not support nested data structures
The classic netCDF data model (which is used for classic CDF-1, 2 and 5 format
data) does not support nested data structures
such as trees, nested arrays, or other recursive structures. Through
use of indirection and conventions it is possible to represent some
kinds of nested structures, but the result may fall short of the
@ -386,20 +413,15 @@ of types. The VLEN type allows efficient storage of ragged arrays, and
the introduction of hierarchical groups allows users new ways to
organize data.
Finally, using the netCDF-3 programming interfaces, concurrent access
to a netCDF dataset is limited. One writer and multiple readers may
access data in a single dataset simultaneously, but there is no
support for multiple concurrent writers.
NetCDF-4 supports parallel read/write access to netCDF-4/HDF5 files,
using the underlying HDF5 library and parallel read/write access to
classic and 64-bit offset files using the parallel-netcdf library.
classic files using the PnetCDf library.
For more information about HDF5, see the HDF5 web site:
http://hdfgroup.org/HDF5/.
For more information about parallel-netcdf, see their web site:
http://www.mcs.anl.gov/parallel-netcdf.
For more information about PnetCDF, see their web site:
https://parallel-netcdf.github.io/.
\page netcdf_data_set_components The Components of a NetCDF Data Set
@ -448,8 +470,7 @@ and variable may not have the same name within the same group, and
similarly for sub-groups of that group.)
Groups and user-defined types are only available in files created in
the netCDF-4/HDF5 format. They are not available for classic or 64-bit
offset format files.
the netCDF-4/HDF5 format. They are not available for classic format files.
\section dimensions Dimensions
@ -460,8 +481,8 @@ model-run-number.
A netCDF dimension has both a name and a length.
A dimension length is an arbitrary positive integer, except that one
dimension in a classic or 64-bit offset netCDF dataset can have the
A dimension length is an arbitrary positive integer, except that only one
dimension in a classic netCDF dataset can have the
length UNLIMITED. In a netCDF-4 dataset, any number of unlimited
dimensions can be used.
@ -470,7 +491,7 @@ dimension. A variable with an unlimited dimension can grow to any
length along that dimension. The unlimited dimension index is like a
record number in conventional record-oriented files.
A netCDF classic or 64-bit offset dataset can have at most one
A netCDF classic dataset can have at most one
unlimited dimension, but need not have any. If a variable has an
unlimited dimension, that dimension must be the most significant
(slowest changing) one. Thus any unlimited dimension must be the first
@ -524,15 +545,16 @@ associated attributes, which may be added, deleted or changed after
the variable is created.
A variable external data type is one of a small set of netCDF
types. In classic and 64-bit offset files, only the original six types
types. In classic CDF-1 and 2 files, only the original six types
are available (byte, character, short, int, float, and
double). Variables in netCDF-4 files may also use unsigned short,
unsigned int, 64-bit int, unsigned 64-bit int, or string. Or the user
double). CDF-5 adds unsigned byte, unsigned short, unsigned int, 64-bit int,
and unsigned 64-bit int. In netCDF-4, variables may also use these additional
data types, plus the string data type. Or the user
may define a type, as an opaque blob of bytes, as an array of variable
length arrays, or as a compound type, which acts like a C struct. (See
\ref data_type).
In the CDL notation, classic and 64-bit offset type can be used. They
In the CDL notation, classic data type can be used. They
are given the simpler names byte, char, short, int, float, and
double. The name real may be used as a synonym for float in the CDL
notation. The name long is a deprecated synonym for int. For the exact
@ -694,8 +716,8 @@ attributes, see \ref attribute_conventions.
Attributes may be added to a netCDF dataset long after it is first
defined, so you don't have to anticipate all potentially useful
attributes. However adding new attributes to an existing classic or
64-bit offset format dataset can incur the same expense as copying the
attributes. However adding new attributes to an existing classic
format dataset can incur the same expense as copying the
dataset. For a more extensive discussion see \ref file_structure_and_performance.
\section differences_atts_vars Differences between Attributes and Variables
@ -771,7 +793,7 @@ information.
\section archival Is NetCDF a Good Archive Format?
NetCDF classic or 64-bit offset formats can be used as a
NetCDF classic formats can be used as a
general-purpose archive format for storing arrays. Compression of data
is possible with netCDF (e.g., using arrays of eight-bit or 16-bit
integers to encode low-resolution floating-point numbers instead of
@ -994,8 +1016,8 @@ servers, full support for the enhanced netCDF-4 data model in the
ncgen utility, a new nccopy utility for copying and conversion among
netCDF format variants, ability to read some HDF4/HDF5 data archives
through the netCDF C or Fortran interfaces, support for parallel I/O
on netCDF classic and 64-bit offset files using the parallel-netcdf
(formerly pnetcdf) library from Argonne/Northwestern, a new nc-config
on netCDF classic files (CDF-1, 2, and 5) using the PnetCDF
library from Argonne/Northwestern, a new nc-config
utility to help compile and link programs that use netCDF, inclusion
of the UDUNITS library for handling “units” attributes, and inclusion
of libcf to assist in creating data compliant with the Climate and
@ -1004,7 +1026,7 @@ Forecast (CF) metadata conventions.
In September, 2010, the Netcdf-Java/CDM (Common Data Model) version
4.2 library was declared stable and made available to users. This
100%-Java implementation provided a read-write interface to netCDF-3
classic and 64-bit offset data, as well as a read-only interface to
classic format files, as well as a read-only interface to
netCDF-4 enhanced model data and many other formats of scientific data
through a common (CDM) interface. More recent releases support
writing netCDF-4 data. The NetCDF-Java library also
@ -1030,7 +1052,7 @@ a different representation from the external type of the variable, a
conversion between the internal type and external type will take place
when the data is read or written.
Access to data in classic and 64-bit offset format is direct. Access
Access to data in classic formats is direct. Access
to netCDF-4 data is buffered by the HDF5 layer. In either case you can
access a small subset of data from a large dataset efficiently,
without first accessing all the data that precedes it.
@ -1234,8 +1256,8 @@ array section access instead.
\section classic_file_parts Parts of a NetCDF Classic File
A netCDF classic or 64-bit offset dataset is stored as a single file
comprising two parts:
A netCDF classic dataset (including CDF-1, 2, and 5 formats) is stored as a
single file comprising two parts:
- a header, containing all the information about dimensions, attributes,
and variables except for the variable data;
- a data part, comprising fixed-size data, containing the data for
@ -1338,8 +1360,8 @@ dimension ids.
XDR is a standard for describing and encoding data and a library of
functions for external data representation, allowing programmers to
encode data structures in a machine-independent way. Classic or 64-bit
offset netCDF employs an extended form of XDR for representing
encode data structures in a machine-independent way. Classic
netCDF employs an extended form of XDR for representing
information in the header part and the data parts. This extended XDR
is used to write portable data that can be read on any other machine
for which the library has been implemented.
@ -1370,10 +1392,20 @@ limitations. See \ref limitations.
In version 3.6.0, netCDF included its first-ever variant of the
underlying data format. The new format introduced in 3.6.0 uses 64-bit
file offsets in place of the 32-bit offsets. There are still some
file offsets in place of the 32-bit offsets. The new format is also referred as
CDF-2 format as it bears a signature string of "CDF2" in the file header.
There are still some
limits on the sizes of variables, but the new format can create very
large datasets. See \ref netcdf_64bit_offset_format.
Starting from version 4.4.0, netCDF included the support of CDF-5 format. In
order to allows defining large array variables with more than 4-billion
elements, CDF-5 replaces most of the 32-bit integers used to describe metadata
with 64-bit integers. In addition, it supports the following new external data
types: NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, and NC_UINT64. The CDF-5 format
specifications can be found in
(http://cucis.ece.northwestern.edu/projects/PnetCDF/CDF-5.html).
NetCDF-4 variables and files can be any size supported by the
underlying file system.
@ -1387,7 +1419,7 @@ about 1.15e+18 bytes. Note also that all sizes are really 4 bytes less
than the ones given below. For example the maximum size of a fixed
variable in netCDF 3.6 classic format is really 2 GiB - 4 bytes.
Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4
Limits | No LFS | v3.5 | v3.6/classic | v3.6/64-bit offset | v4.0/netCDF-4 and CDF-5
-------------------------------------------|------------|---------|---------------|---------------------|-----------------
Max File Size | 2 GiB | 8 EiB | 8 EiB | 8 EiB | unlimited
Max Number of Fixed Vars > 2 GiB | 0 | 1 (last)| 1 (last) | 2^32 | unlimited
@ -1400,14 +1432,14 @@ For more information about the different file formats of netCDF see
\section offset_format_limitations NetCDF 64-bit Offset Format Limitations
Although the 64-bit offset format allows the creation of much larger
Although the 64-bit offset format (CDF-2) allows the creation of much larger
netCDF files than was possible with the classic format, there are
still some restrictions on the size of variables.
It's important to note that without Large File Support (LFS) in the
operating system, it's impossible to create any file larger than 2
GiBytes. Assuming an operating system with LFS, the following
restrictions apply to the netCDF 64-bit offset format.
restrictions apply to the CDF-2 format.
No fixed-size variable can require more than 2^32 - 4 bytes (i.e. 4GiB
- 4 bytes, or 4,294,967,292 bytes) of storage for its data, unless it
@ -1415,13 +1447,13 @@ is the last fixed-size variable and there are no record
variables. When there are no record variables, the last fixed-size
variable can be any size supported by the file system, e.g. terabytes.
A 64-bit offset format netCDF file can have up to 2^32 - 1 fixed sized
A CDF-2 file can have up to 2^32 - 1 fixed sized
variables, each under 4GiB in size. If there are no record variables
in the file the last fixed variable can be any size.
No record variable can require more than 2^32 - 4 bytes of storage for
each record's worth of data, unless it is the last record variable. A
64-bit offset format netCDF file can have up to 2^32 - 1 records, of
CDF-2 file can have up to 2^32 - 1 records, of
up to 2^32 - 1 variables, as long as the size of one record's data for
each record variable except the last is less than 4 GiB - 4.
@ -1432,12 +1464,12 @@ boundaries.
There are important constraints on the structure of large netCDF
classic files that result from the 32-bit relative offsets that are
part of the netCDF classic file format:
part of the netCDF classic file format (CDF-1):
The maximum size of a record in the classic format in versions 3.5.1
and earlier is 2^32 - 4 bytes, or about 4 GiB. In versions 3.6.0 and
later, there is no such restriction on total record size for the
classic format or 64-bit offset format.
classic formats.
If you don't use the unlimited dimension, only one variable can exceed
2 GiB in size, but it can be as large as the underlying file system
@ -1491,10 +1523,10 @@ the data in a 2.4 Tbyte file might be something like:
\section netcdf_3_io The NetCDF-3 I/O Layer
The following discussion applies only to netCDF classic and 64-bit
offset files. For netCDF-4 files, the I/O layer is the HDF5 library.
The following discussion applies only to netCDF classic files (i.e. CDF-1, 2,
and 5 formats). For netCDF-4 files, the I/O layer is the HDF5 library.
For netCDF classic and 64-bit offset files, an I/O layer implemented
For netCDF classic offset files, an I/O layer implemented
much like the C standard I/O (stdio) library is used by netCDF to read
and write portable data to netCDF datasets. Hence an understanding of
the standard I/O library provides answers to many questions about
@ -1519,7 +1551,7 @@ operations can influence I/O performance significantly. Reading data
in the same order in which it was written within each record will
minimize buffer flushes.
You should not expect netCDF classic or 64-bit offset format data
You should not expect netCDF classic format data
access to work with multiple writers having the same file open for
writing simultaneously.
@ -1547,12 +1579,12 @@ NetCDF uses the HDF5 parallel programming model for parallel I/O with
netCDF-4/HDF5 files. The HDF5 tutorial
(http://hdfgroup.org/HDF5//HDF5/Tutor) is a good reference.
For classic and 64-bit offset files, netCDF uses the parallel-netcdf
(formerly pnetcdf) library from Argonne National Labs/Nortwestern
University. For parallel access of classic and 64-bit offset files,
netCDF must be configured with the –with-pnetcdf option at build
time. See the parallel-netcdf site for more information
(http://www.mcs.anl.gov/parallel-netcdf).
For classic files, netCDF uses the PnetCDF library from Argonne National
Labs/Northwestern University. For parallel access of files in classic formats,
netCDF must be configured with the --with-pnetcdf option at build time. See the
PnetCDF site for more information (https://parallel-netcdf.github.io).
Addition information and example programs can be found in
(http://cucis.ece.northwestern.edu/projects/PnetCDF/#InteroperabilityWithNetCDF4)
\section interoperability_with_hdf5 Interoperability with HDF5
@ -2241,7 +2273,7 @@ possible floating-point values would requires '-p 9,17'.)
@par -k
Show \e kind of netCDF file, that is which format variant the file uses.
Other options are ignored if this option is specified. Output will be
one of 'classic'. '64-bit offset', 'netCDF-4', or 'netCDF-4 classic
one of 'classic'. '64-bit offset', '64-bit data', 'netCDF-4', or 'netCDF-4 classic
model'.
@par -s
@ -2350,7 +2382,7 @@ ncgen(1), netcdf(3)
\subsection ncdump_string_note NOTE ON STRING OUTPUT
For classic, 64-bit offset or netCDF-4 classic model data, \b ncdump
For classic, 64-bit offset, 64-bit data, or netCDF-4 classic model data, \b ncdump
generates line breaks after embedded newlines in displaying character
data. This is not done for netCDF-4 files, because netCDF-4 supports
arrays of real strings of varying length.
@ -2374,7 +2406,7 @@ The \b nccopy utility copies an input netCDF file in any supported
format variant to an output netCDF file, optionally converting the
output to any compatible netCDF format variant, compressing the data,
or rechunking the data. For example, if built with the netCDF-3
library, a netCDF classic file may be copied to a netCDF 64-bit offset
library, a classic CDF-1 file may be copied to a CDF-2 or CDF-5
file, permitting larger variables. If built with the netCDF-4
library, a netCDF classic file may be copied to a netCDF-4 file or to
a netCDF-4 classic model file as well, permitting data compression,
@ -2383,7 +2415,7 @@ netCDF-4 features.
If no output format is specified, with either \b -k \e kind_name
or \e -kind_code, then the output will use the same
format as the input, unless the input is classic or 64-bit offset
format as the input, unless the input is classic format
and either chunking or compression is specified, in which case the
output will be netCDF-4 classic model format. Attempting
some kinds of format conversion will result in an error, if the
@ -2409,7 +2441,8 @@ Use format name to specify the kind of file to be created
and, by inference, the data model (i.e. netcdf-3 (classic) or
netcdf-4 (enhanced)). The possible arguments are: \n
'nc3' or 'classic' => netCDF classic format \n
'nc6' or '64-bit offset' => netCDF 64-bit format \n
'nc6' or '64-bit offset' => netCDF 64-bit offset format \n
'cdf5' => netCDF 64-bit data format \n
'nc4' or 'netCDF-4' => netCDF-4 format (enhanced data model) \n
'nc7' or 'netCDF-4 classic model' => netCDF-4 classic model format \n
@ -2424,7 +2457,8 @@ Use format numeric code (instead of format name) to specify the kind of file to
and, by inference, the data model (i.e. netcdf-3 (classic) versus
netcdf-4 (enhanced)). The numeric codes are: \n
3 => netcdf classic format \n
6 => netCDF 64-bit format \n
6 => netCDF 64-bit offset format (CDF_2) \n
5 => netCDF 64-bit data format (CDF-5) \n
4 => netCDF-4 format (enhanced data model) \n
7 => netCDF-4 classic model format \n
@ -2440,7 +2474,7 @@ corresponds to no compression and 9 to maximum compression, with
higher levels of compression requiring marginally more time to
compress or uncompress than lower levels. Compression achieved may
also depend on output chunking parameters. If this option is
specified for a classic format or 64-bit offset format input file, it
specified for a classic format input file, it
is not necessary to also specify that the output should be netCDF-4
classic model, as that will be the default. If this option is not
specified and the input file has compressed variables, the compression
@ -2618,7 +2652,7 @@ but no general algorithm for computing the optimum number of chunk
cache elements has been implemented yet.
\par -r
Read netCDF classic or 64-bit offset input file into a diskless netCDF
Read netCDF classic input file into a diskless netCDF
file in memory before copying. Requires that input file be small
enough to fit into memory. For \b nccopy, this doesn't seem to provide
any significant speedup, so may not be a useful option.
@ -2665,9 +2699,9 @@ type, to the output file bar.nc:
\code
nccopy -d1 foo.nc bar.nc
\endcode
If foo.nc was a classic or 64-bit offset netCDF file, bar.nc will be a
netCDF-4 classic model netCDF file, because the classic and 64-bit
offset format variants don't support compression. If foo.nc was a
If foo.nc was a classic netCDF file, bar.nc will be a
netCDF-4 classic model netCDF file, because the classic
formats don't support compression. If foo.nc was a
netCDF-4 file with some variables compressed using various deflation
levels, the output will also be a netCDF-4 file of the same type, but
all the variables, including any uncompressed variables in the input,

29
docs/tutorial.dox
View File

@ -144,9 +144,9 @@ must add support for netCDF-4 advanced features.
In addition to the main netCDF-3 C API, there is an additional (older)
C API, the netCDF-2 API. This API produces exactly the same files as
the netCDF-3 API - only the API is different. (That is, users can
create either classic format files, the default, or 64-bit offset
files, or netCDF-4/HDF5 files.)
the netCDF-3 API - only the API is different. That is, users can
create either classic CDF-1 format files, the default, 64-bit offset
files (CDF-2), 64-bit data files (CDF-5), or netCDF-4/HDF5 files.
The version 2 API was the API before netCDF-3.0 came out. It is still
fully supported, however. Programs written to the version 2 API will
@ -262,7 +262,7 @@ opaque types.
</ul>
These features may only be used when working with a netCDF-4/HDF5
file. Files created in classic or 64-bit offset format cannot support
file. Files created in classic formats cannot support
groups or user-defined types (see \ref netcdf_format).
<p>\image html nc4-model.png
@ -797,8 +797,11 @@ with nc_get_vara()/nc_put_vara().
\page parallel_io Parallel I/O with NetCDF-4
NetCDF-4 provides access to HDF5 parallel I/O features for netCDF-4/HDF5 files. NetCDF classic and 64-bit offset format may not be opened or created for use with HDF5-based parallel I/O. (They may be opened and created, but parallel I/O is not available.) They may be opened or created for use with parallel-netcdf-based parallel I/O. A few functions have been added to the netCDF C API to handle parallel I/O. You must build netCDF-4 properly to take advantage of parallel
features (see \ref build_parallel).
NetCDF-4 provides parallel file access to both classic and netCDF-4/HDF5 files.
The parallel I/O to netCDF-4 files is achieved through the HDF5 library while
the parallel I/O to classic files is through PnetCDF. A few functions have been
added to the netCDF C API to handle parallel I/O. You must build netCDF-4
properly to take advantage of parallel features (see \ref build_parallel).
The nc_open_par() and nc_create_par() functions are used to
create/open a netCDF file with parallel access.
@ -807,9 +810,13 @@ create/open a netCDF file with parallel access.
\section collective_independent Collective/Independent Access
Parallel file access is either collective (all processors must
participate) or independent (any processor may access the data without
waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access().
Parallel file access is either collective (all processors must participate) or
independent (any processor may access the data without waiting for others). All
netCDF metadata writing operations are collective. That is, all creation of
groups, types, variables, dimensions, or attributes. Data reads and writes
(e.g. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent, the
default) or collective. To make writes to a variable collective, call
nc_var_par_access().
\page tutorial_ncids Numbering of NetCDF IDs
@ -846,7 +853,7 @@ may change the next time the file is opened.
To construct a netCDF file you need to:
- create the file - Specify the name, optionally the format: classic
(the default) or 64bit-offset.
(the default), 64bit-offset, or 64-bit data.
- define metadata - Specify the names and types of dimensions, data
variables, and attributes.
- write data - Write arrays of data from program variables to the
@ -914,7 +921,7 @@ attributes, and the unlimited dimid.
num attributes.
- nc_inq_dimid() Find dimension ID from its name.
- nc_inq_varid() Find variable ID from its name.
- nc_inq_format() Find file format: classic or 64-bit offset
- nc_inq_format() Find file format: classic CDF-1, 64-bit offset CDF-2, or 64-bit data CDF-5
- nc_inq_libvers() Find the netCDF library version.
\page accessing_subsets Reading and Writing Subsets of Data

16
docs/types.dox
View File

@ -12,8 +12,8 @@ a user-defined data type (see \ref user_defined_types).
The atomic external types supported by the netCDF interface are:
- ::NC_BYTE 8-bit signed integer
- ::NC_UBYTE 8-bit unsigned integer
- ::NC_CHAR 8-bit character byte
- ::NC_UBYTE 8-bit unsigned integer *
- ::NC_CHAR 8-bit character
- ::NC_SHORT 16-bit signed integer
- ::NC_USHORT 16-bit unsigned integer *
- ::NC_INT (or ::NC_LONG) 32-bit signed integer
@ -24,7 +24,7 @@ The atomic external types supported by the netCDF interface are:
- ::NC_DOUBLE 64-bit floating point
- ::NC_STRING variable length character string +
\remark * These types are available only for CDF5 (NC_CDF5) and netCDF-4 format (NC_NETCDF4) files. All the unsigned ints (except \ref NC_CHAR) and the 64-bit ints are for CDF5 or netCDF-4 files only.
\remark * These types are available only for CDF5 (NC_CDF5) and netCDF-4 format (NC_NETCDF4) files. All the unsigned ints and the 64-bit ints are for CDF5 or netCDF-4 files only.
\remark + These types are available only for netCDF-4 (NC_NETCDF4) files.
These types were chosen to provide a reasonably wide range of
@ -73,7 +73,7 @@ avoid such precision loss, check the external types of the variables
you access to make sure you use an internal type that has adequate
precision.
The names for the primitive external data types (byte, char, short,
The names for the primitive external data types (char, byte, ubyte, short,
ushort, int, uint, int64, uint64, float or real, double, string) are
reserved words in CDL, so the names of variables, dimensions, and
attributes must not be type names.
@ -85,16 +85,16 @@ converted into other numeric types, it is interpreted as signed.
For the correspondence between netCDF external data types and the data
types of a language see \ref variables.
\section classic_structures Data Structures in Classic and 64-bit Offset Files
\section classic_structures Data Structures in Classic Files
The only kind of data structure directly supported by the netCDF
classic (and 64-bit offset) abstraction is a collection of named
classic abstraction, i.e. CDF-1, 2, and 5 formats, is a collection of named
arrays with attached vector attributes. NetCDF is not particularly
well-suited for storing linked lists, trees, sparse matrices, ragged
arrays or other kinds of data structures requiring pointers.
It is possible to build other kinds of data structures in netCDF
classic or 64-bit offset formats, from sets of arrays by adopting
classic formats, from sets of arrays by adopting
various conventions regarding the use of data in one array as pointers
into another array. The netCDF library won't provide much help or
hindrance with constructing such data structures, but netCDF provides
@ -116,7 +116,7 @@ arrays. See below.)
\endcode
As another example, netCDF variables may be grouped within a netCDF
classic or 64-bit offset dataset by defining attributes that list the
classic dataset by defining attributes that list the
names of the variables in each group, separated by a conventional
delimiter such as a space or comma. Using a naming convention for
attribute names for such groupings permits any number of named groups

7
include/hdf5internal.h
View File

@ -12,12 +12,11 @@
#define _HDF5INTERNAL_
#include "config.h"
#include "nc4internal.h"
#include "ncdimscale.h"
#include "ncdispatch.h"
#include <nc4dispatch.h>
#include <hdf5.h>
#include <hdf5_hl.h>
#include "nc4internal.h"
#include "ncdimscale.h"
#include "nc4dispatch.h"
#define NC_MAX_HDF5_NAME (NC_MAX_NAME + 10)

6
include/nc3internal.h
View File

@ -169,7 +169,7 @@ typedef struct NC_var {
int *dimids; /* assoc->value */
NC_attrarray attrs;
nc_type type; /* the discriminant */
size_t len; /* the total length originally allocated */
long long len; /* the total length originally allocated */
off_t begin;
/* end xdr */
int no_fill; /* whether fill mode is ON or OFF */
@ -212,7 +212,7 @@ extern int
NC_findvar(const NC_vararray *ncap, const char *name, NC_var **varpp);
extern int
NC_check_vlen(NC_var *varp, unsigned long long vlen_max);
NC_check_vlen(NC_var *varp, long long vlen_max);
extern int
NC_lookupvar(NC3_INFO* ncp, int varid, NC_var **varp);
@ -363,7 +363,7 @@ nc_get_NC(NC3_INFO* ncp);
/* Begin defined in putget.c */
extern int
fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno);
fill_NC_var(NC3_INFO* ncp, const NC_var *varp, long long varsize, size_t recno);
extern int
nc_inq_rec(int ncid, size_t *nrecvars, int *recvarids, size_t *recsizes);

2
include/ncdispatch.h
View File

@ -108,7 +108,7 @@ typedef struct NC_MPI_INFO {
/* Define known dispatch tables and initializers */
/*Forward*/
typedef struct NC_Dispatch NC_Dispatch;
// typedef struct NC_Dispatch NC_Dispatch;
extern int NCDISPATCH_initialize(void);
extern int NCDISPATCH_finalize(void);

32
libdispatch/dfile.c
View File

@ -372,7 +372,7 @@ and attributes.
NC_NOCLOBBER (do not overwrite existing file),
NC_SHARE (limit write caching - netcdf classic files only),
NC_64BIT_OFFSET (create 64-bit offset file),
NC_64BIT_DATA (Alias NC_CDF5) (create CDF-5 file),
NC_64BIT_DATA (alias NC_CDF5) (create CDF-5 file),
NC_NETCDF4 (create netCDF-4/HDF5 file),
NC_CLASSIC_MODEL (enforce netCDF classic mode on netCDF-4/HDF5 files),
NC_DISKLESS (store data in memory),
@ -408,14 +408,10 @@ file, instead of a netCDF classic format file. The 64-bit offset
format imposes far fewer restrictions on very large (i.e. over 2 GB)
data files. See Large File Support.
Setting NC_64BIT_DATA (Alias NC_CDF5) causes netCDF to create a CDF-5
Setting NC_64BIT_DATA (alias NC_CDF5) causes netCDF to create a CDF-5
file format that supports large files (i.e. over 2GB) and large
variables (over 2B array elements.). See Large File Support.
Note that the flag NC_PNETCDF also exists as the combination of
NC_CDF5 or'd with NC_MPIIO to indicate that the pnetcdf library
should be used.
A zero value (defined for convenience as NC_CLOBBER) specifies the
default behavior: overwrite any existing dataset with the same file
name and buffer and cache accesses for efficiency. The dataset will be
@ -425,7 +421,7 @@ Setting NC_NETCDF4 causes netCDF to create a HDF5/NetCDF-4 file.
Setting NC_CLASSIC_MODEL causes netCDF to enforce the classic data
model in this file. (This only has effect for netCDF-4/HDF5 files, as
classic and 64-bit offset files always use the classic model.) When
CDF-1, 2 and 5 files always use the classic model.) When
used with NC_NETCDF4, this flag ensures that the resulting
netCDF-4/HDF5 file may never contain any new constructs from the
enhanced data model. That is, it cannot contain groups, user defined
@ -580,7 +576,7 @@ nc_create(const char *path, int cmode, int *ncidp)
* @param initialsz On some systems, and with custom I/O layers, it
* may be advantageous to set the size of the output file at creation
* time. This parameter sets the initial size of the file at creation
* time. This only applies to classic and 64-bit offset files. The
* time. This only applies to classic CDF-1, 2, and 5 files. The
* special value NC_SIZEHINT_DEFAULT (which is the value 0), lets the
* netcdf library choose a suitable initial size.
* @param chunksizehintp A pointer to the chunk size hint, which
@ -597,7 +593,7 @@ nc_create(const char *path, int cmode, int *ncidp)
* call to discover the system pagesize, we just set default bufrsize
* to 8192. The bufrsize is a property of a given open netcdf
* descriptor ncid, it is not a persistent property of the netcdf
* dataset. This only applies to classic and 64-bit offset files.
* dataset. This only applies to classic files.
* @param ncidp Pointer to location where returned netCDF ID is to be
* stored.
*
@ -704,7 +700,7 @@ nc_create_mem(const char* path, int mode, size_t initialsize, int* ncidp)
* ignored for other files.
* @param basepe Deprecated parameter from the Cray days.
* @param chunksizehintp A pointer to the chunk size hint. This only
* applies to classic and 64-bit offset files.
* applies to classic files.
* @param ncidp Pointer that gets ncid.
*
* @return ::NC_NOERR No error.
@ -723,7 +719,7 @@ nc__create_mp(const char *path, int cmode, size_t initialsz,
*
* This function opens an existing netCDF dataset for access. It
* determines the underlying file format automatically. Use the same
* call to open a netCDF classic, 64-bit offset, or netCDF-4 file.
* call to open a netCDF classic or netCDF-4 file.
*
* @param path File name for netCDF dataset to be opened. When DAP
* support is enabled, then the path may be an OPeNDAP URL rather than
@ -746,7 +742,7 @@ nc__create_mp(const char *path, int cmode, size_t initialsz,
* renaming dimensions, variables, and attributes, or deleting
* attributes.)
*
* The NC_SHARE flag is only used for netCDF classic and 64-bit offset
* The NC_SHARE flag is only used for netCDF classic
* files. It is appropriate when one process may be writing the
* dataset and one or more other processes reading the dataset
* concurrently; it means that dataset accesses are not buffered and
@ -847,7 +843,7 @@ file path.
access) and NC_SHARE as in nc_open().
\param chunksizehintp A size hint for the classic library. Only
applies to classic and 64-bit offset files. See below for more
applies to classic files. See below for more
information.
\param ncidp Pointer to location where returned netCDF ID is to be
@ -1033,7 +1029,7 @@ nc_open_memio(const char* path, int mode, NC_memio* params, int* ncidp)
* @param mode Open mode.
* @param basepe Deprecated parameter from the Cray days.
* @param chunksizehintp A pointer to the chunk size hint. This only
* applies to classic and 64-bit offset files.
* applies to classic files.
* @param ncidp Pointer to location where returned netCDF ID is to be
* stored.
*
@ -2005,11 +2001,11 @@ check_create_mode(int mode)
* @param path0 The file name of the new netCDF dataset.
* @param cmode The creation mode flag, the same as in nc_create().
* @param initialsz This parameter sets the initial size of the file
* at creation time. This only applies to classic and 64-bit offset
* at creation time. This only applies to classic
* files.
* @param basepe Deprecated parameter from the Cray days.
* @param chunksizehintp A pointer to the chunk size hint. This only
* applies to classic and 64-bit offset files.
* applies to classic files.
* @param useparallel Non-zero if parallel I/O is to be used on this
* file.
* @param parameters Pointer to MPI comm and info.
@ -2106,7 +2102,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
model = NC_FORMATX_NC4;
break;
#endif
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
case NC_FORMAT_CDF5:
xcmode |= NC_64BIT_DATA;
model = NC_FORMATX_NC3;
@ -2322,7 +2318,7 @@ NC_open(const char *path0, int cmode, int basepe, size_t *chunksizehintp,
hdf4built = 1;
#endif
#endif
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
cdf5built = 1;
#endif
if(!hdf5built && model == NC_FORMATX_NC4) {

5
libhdf5/hdf5attr.c
View File

@ -11,11 +11,8 @@
* @author Ed Hartnett
*/
#include "nc.h"
#include "nc4internal.h"
#include "config.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include "ncdispatch.h"
int nc4typelen(nc_type type);

3
libhdf5/hdf5dim.c
View File

@ -11,9 +11,8 @@
* @author Ed Hartnett
*/
#include "nc4internal.h"
#include "config.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**
* @internal Dimensions are defined in attributes attached to the

9
libhdf5/hdf5file.c
View File

@ -12,16 +12,7 @@
*/
#include "config.h"
#include <errno.h> /* netcdf functions sometimes return system errors */
#include "nc.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include "netcdf_mem.h"
#ifdef USE_HDF4
#include <mfhdf.h>
#endif
#include <hdf5_hl.h>
extern int nc4_vararray_add(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);

3
libhdf5/hdf5grp.c
View File

@ -11,9 +11,8 @@
* @author Ed Hartnett
*/
#include "nc4internal.h"
#include "config.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**
* @internal Create a group. Its ncid is returned in the new_ncid

6
libhdf5/hdf5internal.c
View File

@ -13,13 +13,9 @@
*
* @author Ed Hartnett
*/
#include "config.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc.h" /* from libsrc */
#include "ncdispatch.h" /* from libdispatch */
#include "ncutf8.h"
#include "H5DSpublic.h"
#undef DEBUGH5

4
libhdf5/hdf5type.c
View File

@ -11,9 +11,9 @@
*
* @author Ed Hartnett
*/
#include "nc4internal.h"
#include "config.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
/**
* @internal Determine if two types are equal.

3
libhdf5/nc4hdf.c
View File

@ -14,10 +14,7 @@
*/
#include "config.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#include "nc4dispatch.h"
#include <H5DSpublic.h>
#include <math.h>
#ifdef HAVE_INTTYPES_H

6
libhdf5/nc4info.c
View File

@ -6,12 +6,8 @@
* and redistribution conditions.
* @author Dennis Heimbigner
*/
#include "config.h"
#include <stdlib.h>
#include <string.h>
#include <hdf5.h>
#include "netcdf.h"
#include "nc4internal.h"
#include "hdf5internal.h"
#define HDF5_MAX_NAME 1024 /**< HDF5 max name. */

10
libsrc/nc3internal.c
View File

@ -100,7 +100,7 @@ err:
int
nc3_cktype(int mode, nc_type type)
{
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
if (mode & NC_CDF5) { /* CDF-5 format */
if (type >= NC_BYTE && type < NC_STRING) return NC_NOERR;
} else
@ -713,7 +713,7 @@ NC_check_vlens(NC3_INFO *ncp)
/* maximum permitted variable size (or size of one record's worth
of a record variable) in bytes. This is different for format 1
and format 2. */
unsigned long long vlen_max;
long long vlen_max;
size_t ii;
size_t large_vars_count;
size_t rec_vars_count;
@ -723,7 +723,7 @@ NC_check_vlens(NC3_INFO *ncp)
return NC_NOERR;
if (fIsSet(ncp->flags,NC_64BIT_DATA)) /* CDF-5 */
vlen_max = (size_t)X_INT64_MAX - 3; /* "- 3" handles rounded-up size */
vlen_max = X_INT64_MAX - 3; /* "- 3" handles rounded-up size */
else if (fIsSet(ncp->flags,NC_64BIT_OFFSET) && sizeof(off_t) > 4)
/* CDF2 format and LFS */
vlen_max = X_UINT_MAX - 3; /* "- 3" handles rounded-up size */
@ -1167,7 +1167,7 @@ nc_set_default_format(int format, int *old_formatp)
return NC_EINVAL;
#else
if (format != NC_FORMAT_CLASSIC && format != NC_FORMAT_64BIT_OFFSET
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
&& format != NC_FORMAT_CDF5
#endif
)
@ -1698,7 +1698,7 @@ NC3_inq_format(int ncid, int *formatp)
return NC_NOERR;
/* only need to check for netCDF-3 variants, since this is never called for netCDF-4 files */
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
if (fIsSet(nc3->flags, NC_64BIT_DATA))
*formatp = NC_FORMAT_CDF5;
else

4
libsrc/putget.m4
View File

@ -147,7 +147,7 @@ NCFILL(ulonglong, ulonglong, X_SIZEOF_ULONGLONG, NC_FILL_UINT64)
xdr_NC_fill()
*/
int
fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno)
fill_NC_var(NC3_INFO* ncp, const NC_var *varp, long long varsize, size_t recno)
{
char xfillp[NFILL * X_SIZEOF_DOUBLE];
const size_t step = varp->xsz;
@ -155,7 +155,7 @@ fill_NC_var(NC3_INFO* ncp, const NC_var *varp, size_t varsize, size_t recno)
const size_t xsz = varp->xsz * nelems;
NC_attr **attrpp = NULL;
off_t offset;
size_t remaining = varsize;
long long remaining = varsize;
void *xp;
int status = NC_NOERR;

4
libsrc/v1hpg.c
View File

@ -1050,7 +1050,9 @@ v1h_get_NC_var(v1hs *gsp, NC_var **varpp)
if(status != NC_NOERR)
goto unwind_alloc;
status = v1h_get_size_t(gsp, &varp->len);
size_t tmp;
status = v1h_get_size_t(gsp, &tmp);
varp->len = tmp;
if(status != NC_NOERR)
goto unwind_alloc;

10
libsrc/var.c
View File

@ -484,9 +484,6 @@ NC_var_shape(NC_var *varp, const NC_dimarray *dims)
out :
/* No variable size can be > X_INT64_MAX - 3 */
if (0 == NC_check_vlen(varp, (size_t)X_INT64_MAX-3)) return NC_EVARSIZE;
/*
* For CDF-1 and CDF-2 formats, the total number of array elements
* cannot exceed 2^32, unless this variable is the last fixed-size
@ -513,16 +510,15 @@ out :
* systems with LFS it should be 2^32 - 4.
*/
int
NC_check_vlen(NC_var *varp, unsigned long long vlen_max) {
unsigned long long prod=varp->xsz; /* product of xsz and dimensions so far */
NC_check_vlen(NC_var *varp, long long vlen_max) {
int ii;
long long prod=varp->xsz; /* product of xsz and dimensions so far */
assert(varp != NULL);
for(ii = IS_RECVAR(varp) ? 1 : 0; ii < varp->ndims; ii++) {
if(!varp->shape)
return 0; /* Shape is undefined/NULL. */
if (varp->shape[ii] > (size_t)(vlen_max / prod)) {
if ((long long)varp->shape[ii] > vlen_max / prod) {
return 0; /* size in bytes won't fit in a 32-bit int */
}
prod *= varp->shape[ii];

2
nc_test/Makefile.am
View File

@ -27,7 +27,7 @@ endif
if TEST_PARALLEL4
if USE_PNETCDF
if USE_CDF5
if ENABLE_CDF5
TESTPROGRAMS += tst_cdf5format
endif
endif

2
nc_test/nc_test.c
View File

@ -134,7 +134,7 @@ main(int argc, char *argv[])
break;
case NC_FORMAT_CDF5:
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
nc_set_default_format(NC_FORMAT_CDF5, NULL);
fprintf(stderr, "\n\nSwitching to 64-bit data format.\n");
strcpy(testfile, "nc_test_cdf5.nc");

2
nc_test/t_nc.c
View File

@ -361,7 +361,7 @@ main(int argc, char *argv[])
MPI_Init(&argc, &argv);
cmode |= (NC_PNETCDF);
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
cmode |= (NC_64BIT_DATA);
#endif
ret = nc_create_par(fname,cmode, MPI_COMM_WORLD, MPI_INFO_NULL, &id);

2
nc_test/test_write.m4
View File

@ -2424,7 +2424,7 @@ APIFunc(get_file_version)(char *path, int *version)
if (strncmp(magic, "CDF", MAGIC_NUM_LEN-1)==0) {
if (magic[MAGIC_NUM_LEN-1] == NC_FORMAT_CLASSIC ||
magic[MAGIC_NUM_LEN-1] == NC_FORMAT_64BIT_OFFSET
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
|| magic[MAGIC_NUM_LEN-1] == NC_FORMAT_CDF5
#endif
)

2
nc_test/tst_def_var_fill.c
View File

@ -54,7 +54,7 @@ int main(int argc, char** argv) {
buf = (int*) malloc(NY*NX * sizeof(int));
for (k=0; k<5; k++) {
#ifndef USE_CDF5
#ifndef ENABLE_CDF5
if (formats[k] == NC_FORMAT_CDF5) continue;
#endif
#ifndef USE_NETCDF4

4
nc_test/tst_global_fillval.c
View File

@ -37,7 +37,7 @@ main(int argc, char **argv)
num_formats += 2;
#endif
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
num_formats++;
#endif
@ -46,7 +46,7 @@ main(int argc, char **argv)
formats[n++] = 0;
formats[n++] = NC_64BIT_OFFSET;
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
formats[n++] = NC_64BIT_DATA;
#endif
#ifdef USE_NETCDF4

2
nc_test/tst_names.c
View File

@ -225,7 +225,7 @@ main(int argc, char **argv)
NC_FORMAT_CLASSIC
,
NC_FORMAT_64BIT_OFFSET
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
,
NC_FORMAT_CDF5
#endif

4
nc_test/tst_small.c
View File

@ -55,7 +55,7 @@ static int file_create(const char *filename, int cmode, int *ncid)
#ifdef USE_PNETCDF
if (default_format == NC_FORMAT_CLASSIC ||
default_format == NC_FORMAT_64BIT_OFFSET
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
|| default_format == NC_FORMAT_64BIT_DATA
#endif
)
@ -525,7 +525,7 @@ main(int argc, char **argv)
printf("Switching to 64-bit offset format.\n");
strcpy(testfile, "tst_small_64bit.nc");
break;
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
case NC_FORMAT_CDF5:
nc_set_default_format(NC_FORMAT_CDF5, NULL);
printf("Switching to 64-bit data format.\n");

8
nc_test4/tst_parallel5.c
View File

@ -178,10 +178,10 @@ main(int argc, char **argv)
/* Create a compound type. */
if (nc_def_compound(ncid, sizeof(struct crew), COMPOUND_NAME, &typeid)) ERR;
if (nc_insert_array_compound(ncid, typeid, "name", NC_COMPOUND_OFFSET(struct crew, name), NC_CHAR, 1, &dim_size));
if (nc_insert_array_compound(ncid, typeid, "description", NC_COMPOUND_OFFSET(struct crew, description), NC_CHAR, 1, &dim_size));
if (nc_insert_array_compound(ncid, typeid, "origin", NC_COMPOUND_OFFSET(struct crew, origin), NC_CHAR, 1, &dim_size));
if (nc_insert_compound(ncid, typeid, "age", NC_COMPOUND_OFFSET(struct crew, age), NC_INT));
if (nc_insert_array_compound(ncid, typeid, "name", NC_COMPOUND_OFFSET(struct crew, name), NC_CHAR, 1, &dim_size)) ERR;
if (nc_insert_array_compound(ncid, typeid, "description", NC_COMPOUND_OFFSET(struct crew, description), NC_CHAR, 1, &dim_size)) ERR;
if (nc_insert_array_compound(ncid, typeid, "origin", NC_COMPOUND_OFFSET(struct crew, origin), NC_CHAR, 1, &dim_size)) ERR;
if (nc_insert_compound(ncid, typeid, "age", NC_COMPOUND_OFFSET(struct crew, age), NC_INT)) ERR;
/* Create one var. */
if (nc_def_var(ncid, COMPOUND_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR;

2
ncdump/nccopy.c
View File

@ -1706,7 +1706,7 @@ copy(char* infile, char* outfile)
create_mode |= NC_64BIT_OFFSET;
break;
case NC_FORMAT_CDF5:
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
create_mode |= NC_64BIT_DATA;
break;
#else

4
ncdump/run_ncgen_tests.sh
View File

@ -11,7 +11,7 @@ set -e
# This shell script runs the ncdump tests.
# get some config.h parameters
if test -f ${top_builddir}/config.h ; then
if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
CDF5=1
else
CDF5=0
@ -51,7 +51,7 @@ echo "*** creating 64-bit offset file c0_64_run_ncgen_tests.nc from c0.cdl..."
validateNC c0 "c0_64" -k 64-bit-offset -b
if test "x$USE_CDF5" = x1 ; then
if test "x$ENABLE_CDF5" = x1 ; then
echo "*** creating 64-bit data file c5.nc from c5.cdl..."
${NCGEN} -k 64-bit-data -b -o tst_c5_run_ncgen_tests.nc $top_srcdir/ncgen/c5.cdl

2
ncdump/tst_64bit.sh
View File

@ -8,7 +8,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
# This shell script runs the ncdump tests.
# get some config.h parameters
if test -f ${top_builddir}/config.h ; then
if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
CDF5=1
else
CDF5=0

6
ncdump/tst_dimsizes.c
View File

@ -12,7 +12,7 @@
#define DIMMAXCLASSIC (NC_MAX_INT - 3)
#define DIMMAX64OFFSET (NC_MAX_UINT - 3)
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
#define DIMMAX64DATA (NC_MAX_UINT64 - 3)
#endif
@ -65,7 +65,7 @@ main(int argc, char **argv)
if(dimsize != DIMMAX64OFFSET) ERR;
if ((stat=nc_close(ncid))) ERRSTAT(stat);
#ifdef USE_CDF5
#ifdef ENABLE_CDF5
if(sizeof(size_t) == 8) {
printf("\n*** Writing Max Dimension Size (%llu) For NC_64BIT_DATA\n",DIMMAX64DATA);
if ((stat=nc_create(FILE64DATA, NC_CLOBBER | NC_64BIT_DATA, &ncid))) ERRSTAT(stat);
@ -80,7 +80,7 @@ main(int argc, char **argv)
if(dimsize != DIMMAX64DATA) ERR;
if ((stat=nc_close(ncid))) ERRSTAT(stat);
}
#endif /* USE_CDF5 */
#endif /* ENABLE_CDF5 */
SUMMARIZE_ERR;
FINAL_RESULTS;

10
ncdump/tst_formatx3.sh
View File

@ -6,10 +6,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
# This shell script runs the ncdump tests.
# get some config.h parameters
if test -f ${top_builddir}/config.h ; then
if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
USE_CDF5=1
if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
ENABLE_CDF5=1
else
USE_CDF5=0
ENABLE_CDF5=0
fi
else
echo "Cannot locate config.h"
@ -42,9 +42,9 @@ ECODE=1
fi
# Only do following test if USE_CDF5 is true.
# Only do following test if ENABLE_CDF5 is true.
if test "x$USE_CDF5" = x1 ; then
if test "x$ENABLE_CDF5" = x1 ; then
echo "Test extended format output for a 64-bit CDF-5 classic file"
rm -f tmp_tst_formatx3
${NCGEN} -k5 -b -o ./tst_formatx3.nc $srcdir/ref_tst_small.cdl

2
ncdump/tst_lengths.sh
View File

@ -5,7 +5,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
# get some config.h parameters
if test -f ${top_builddir}/config.h ; then
if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
HAVE_CDF5=1
else
HAVE_CDF5=0

2
ncdump/tst_nccopy3.sh
View File

@ -13,7 +13,7 @@ echo ""
# get some config.h parameters
if test -f ${top_builddir}/config.h ; then
if fgrep -e '#define USE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
if fgrep -e '#define ENABLE_CDF5 1' ${top_builddir}/config.h >/dev/null ; then
HAVE_CDF5=1
else
HAVE_CDF5=0

2
ncgen/main.c
View File

@ -512,7 +512,7 @@ main(
/* Compute the k_flag (1st pass) using rules in the man page (ncgen.1).*/
#ifndef USE_CDF5
#ifndef ENABLE_CDF5
if(k_flag == NC_FORMAT_CDF5) {
derror("Output format CDF5 requested, but netcdf was built without cdf5 support.");
return 0;