mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-01-18 15:55:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1201 from Unidata/v4.6.2-release-branch.wif

Browse Source 
V4.6.2 release branch.wif
This commit is contained in:
Ward Fisher 2018-11-15 14:12:40 -07:00 committed by GitHub
commit 572121d3dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
83 changed files with 4727 additions and 2488 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -273,7 +273,7 @@ SET(CMAKE_REQUIRED_INCLUDES ${CMAKE_SOURCE_DIR}/libsrc)
# Configuration for post-install RPath
# Adapted from http://www.cmake.org/Wiki/CMake_RPATH_handling
##
IF(NOT MSVC)
IF(NOT MSVC AND BUILD_SHARED_LIBS)
# use, i.e. don't skip the full RPATH for the build tree
SET(CMAKE_SKIP_BUILD_RPATH FALSE)

10
dap4_test/baselineremote/test_atomic_array.syn.dmp
View File

@ -1,8 +1,8 @@
netcdf test_atomic_array {
types:
byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2,
Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6,
Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10,
byte enum cloud_class_t {Clear = 0, Cumulonimbus = 1, Stratus = 2,
Stratocumulus = 3, Cumulus = 4, Altostratus = 5, Nimbostratus = 6,
Altocumulus = 7, Cirrostratus = 8, Cirrocumulus = 9, Cirrus = 10,
Missing = 127} ;
opaque(16) opaque16_t ;
dimensions:
@ -47,6 +47,8 @@ data:
vo =
0XA2177AA7287C04FA8BB57BCDF76EC80F, 0X34FA472AA9404DD543143CADED303A75 ;
primary_cloud = Stratus, Cirrocumulus, Nimbostratus, Cirrostratus,
primary_cloud = Stratus, Cirrocumulus, Nimbostratus, Cirrostratus,
Stratocumulus ;
}

2
docs/CMakeLists.txt
View File

@ -84,6 +84,6 @@ install-fortran.md all-error-codes.md credits.md auth.md
obsolete/fan_utils.html bestpractices.md filters.md indexing.md
inmemory.md DAP2.dox attribute_conventions.md FAQ.md
file_format_specifications.md known_problems.md
COPYRIGHT.dox user_defined_formats.md)
COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox)
ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

215
docs/DAP4.dox
View File

@ -1,23 +1,28 @@
/*! \page dap4 DAP4 Support
/*!
\page dap4 DAP4 Protocol Support
\section dap4_accessing The DAP4 Prototocol
<!-- Note that this file has the .dox extension, but is mostly markdown -->
<!-- Begin MarkDown -->
\tableofcontents
\subsection dap4_introduction DAP4 Introduction
# DAP4 Introduction {#dap4_introduction}
Beginning with netCDF version 4.2.2, optional support is provided for
accessing data through DAP4 servers using the DAP4 protocol.
Beginning with netCDF version 4.5.0, optional support is provided for
accessing data from servers supporting the DAP4 protocol.
DAP4 support is automatically enabled if a usable curl library can be
set using the LDFLAGS environment variable (similar to the way
that the HDF5 libraries are referenced).
DAP2 plus DAP4 support can forcibly be enabled or disabled using the --enable-dap
flag or the --disable-dap flag, respectively. If enabled,
then DAP4 support requires access to the curl library.
Refer to the installation manual for details
DAP4 support is enabled if the _--enable-dap_ option
is used with './configure'. If DAP4 support is enabled, then
a usable version of _libcurl_ must be specified
using the _LDFLAGS_ environment variable (similar to the way
that the _HDF5_ libraries are referenced).
Refer to the installation manual for details.
By default DAP4 support is enabled if _libcurl_ is found.
DAP4 support can be disabled using the _--disable-dap_.
DAP4 uses a data model that is similar to, but not identical with,
DAP4 uses a data model that is, by design, similar to,
but -- for historical reasons -- not identical with,
the netCDF-Enhanced (aka netcdf-4) model.
Generically, the DAP4 data model is encoded in XML document
called a DMR.
@ -25,31 +30,30 @@ For detailed information about the DAP4 DMR, refer to
the DAP4 specification Volume 1:
http://docs.opendap.org/index.php/DAP4:_Specification_Volume_1
\subsection dap4_access Accessing Data Using the DAP4 Prototocol
# Accessing Data Using the DAP4 Prototocol {#dap4_accessing_data}
In order to access a DAP4 data source through the netCDF API, the
file name normally used is replaced with a URL with a specific
format. The URL is composed of three parts.
- URL - this is a standard form URL with specific markers to indicate that
it refers to a DAP4 encoded dataset. The markers can be one of the following
examples.
+ [dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ [protocol=dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#dap4
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#protocol=dap4
+ dap4://remotetest.unidata.ucar.edu/d4ts/test.01
Note that the last case is converted internally to one of the other forms.
- URL - this is a standard form URL with specific markers to indicate that
it refers to a DAP4 encoded dataset. The markers are of the form
"dap4", "protocol=dap4", or "/thredds/dap4". The following
examples show how they are specified. Note that the "/thredds/dap4"
case will only work when accessing a Thredds-based server.
+ [dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ [protocol=dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#dap4
+ http://remotetest.unidata.ucar.edu/d4ts/test.01#protocol=dap4
+ http://thredds.ucar.edu/thredds/dap4/...
- Constraints - these are suffixed to the URL and take the form
“?dap4.ce=\<expression\>”. The form of the constraint expression
is somewhat complicated, and the specification should be consulted.
- Client parameters - these may be specified in either of
two ways. The older, deprecated form prefixes text to the
front of the url and is of the the general form [\<name>]
or [\<name>=value]. Examples include [show=fetch].
or [\<name>=value]. Examples include [show=fetch] or [noprefetch].
The newer, preferred form prefixes the
parameters to the end of the url using the semi-standard '#'
format: e.g. http://....#show=fetch&noprefetch.
@ -61,19 +65,19 @@ the translation using the "ncdump -h" command to see the
corresponding netCDF-4 representation.
For example, if a web browser is given the following (fictional) URL,
it will return the DMR for the specified dataset
it will return the DMR for the specified dataset (in XML format).
\code
http://remotetest.unidata.ucar.edu/d4ts/test.01.dmr#dap4
http://remotetest.unidata.ucar.edu/d4ts/test.01.dmr.xml#dap4
\endcode
By using the following ncdump command, it is possible to see the
equivalent netCDF-4 translation.
\code
ncdump -h http://remotetest.unidata.ucar.edu/d4ts/test.01#dap4
ncdump -h '[dap4]http://remotetest.unidata.ucar.edu/d4ts/test.01'
\endcode
\subsection dap4_defined_params Defined Client Parameters
# Defined Client Parameters {#dap4_defined_params}
Currently, a limited set of client parameters is
recognized. Parameters not listed here are
@ -87,12 +91,23 @@ Parameter Name Legal Values Semantics
of the complete url for every HTTP get request. If logging is
enabled, then this can be helpful in checking to see the access
behavior of the netCDF code.
- "translate=nc4" - This parameter causes the netCDF code to look
for specially named elements in the DMR XML in order to
achieve a better translation of the DAP4 meta-data to NetCDF enhanced
metadata.
- "opaquesize=<integer>" - This parameter causes the netCDF code to
convert DAP4 variable size OPAQUE objects to netcdf-4 fixed size
objects and forces all of them to be of the size specified.
- "fillmismatch" - Unfortunately, a number of servers sometimes
fail to make sure that the type of the "_FillValue" attribute of a variable
is the same as the type of the containing variable. Setting this tag
caused the netcdf translation to attempt to fix this mismatch. If not set,
then an error will occur.
\subsection dap4_debug Notes on Debugging DAP4 Access
# Notes on Debugging DAP4 Access {#dap4_debug}
The DAP4 support has a logging facility.
Note that this is currently separate from the
existing netCDF logging facility. Turning on this logging can
Turning on this logging can
sometimes give important information. Logging can be enabled by
using the client parameter "log" or "log=filename",
where the first case will send log output to standard error and the
@ -102,4 +117,138 @@ Users should also be aware that if one is
accessing data over an NFS mount, one may see some .nfsxxxxx files;
those can be ignored.
## HTTP Configuration. {#dap4_http2_config}
Limited support for configuring the http connection is provided via
parameters in the “.daprc” configuration file (aka ".dodsrc").
The relevant .daprc file is
located by first looking in the current working directory, and if not
found, then looking in the directory specified by the “$HOME”
environment variable.
Entries in the .daprc file are of the form:
````
['['<url>']']<key>=<value>
````
That is, it consists of a key name and value pair and optionally
preceded by a url enclosed in square brackets.
For given KEY and URL strings, the value chosen is as follows:
If URL is null, then look for the .daprc entry that has no url prefix
and whose key is same as the KEY for which we are looking.
If the URL is not null, then look for all the .daprc entries that
have a url, URL1, say, and for which URL1 has the same host and port
as URL. All parts of the url's except host and port are ignored.
For example, if URL = http//x.y/a, then it will match
entries of the form
_[http//x.y/a]KEY=VALUE_ or _[http//x.y/b]KEY=VALUE_.
It will not match an entry of the form _[http//x.y:8080]KEY=VALUE
because the second has a port number (8080) different than the URL.
Finally from the set so constructed, choose the first matching entry.
Currently, the supported set of keys (with descriptions) are as
follows.
-# HTTP.VERBOSE
Type: boolean ("1"/"0")
Description: Produce verbose output, especially using SSL.
Related CURL Flags: CURLOPT_VERBOSE
-# HTTP.DEFLATE
Type: boolean ("1"/"0")
Description: Allow use of compression by the server.
Related CURL Flags: CURLOPT_ENCODING
-# HTTP.COOKIEJAR
Type: String representing file path
Description: Specify the name of file into which to store cookies. Defaults to in-memory storage.
Related CURL Flags:CURLOPT_COOKIEJAR
-# HTTP.CREDENTIALS.USER
Type: String representing user name
Description: Specify the user name for Digest and Basic authentication.
Related CURL Flags:
-# HTTP.CREDENTIALS.PASSWORD
Type: String representing password
Type: boolean ("1"/"0")
Description: Specify the password for Digest and Basic authentication.
Related CURL Flags:
-# HTTP.SSL.CERTIFICATE
Type: String representing file path
Description: Path to a file containing a PEM cerficate.
Related CURL Flags: CURLOPT_CERT
-# HTTP.SSL.KEY
Type: String representing file path
Description: Same as HTTP.SSL.CERTIFICATE, and should usually have the same value.
Related CURL Flags: CURLOPT_SSLKEY
-# HTTP.SSL.KEYPASSWORD
Type: String representing password
Description: Password for accessing the HTTP.SSL.KEY/HTTP.SSL.CERTIFICATE
Related CURL Flags: CURLOPT_KEYPASSWORD
-# HTTP.SSL.CAPATH
Type: String representing directory
Description: Path to a directory containing trusted certificates for validating server certificates.
Related CURL Flags: CURLOPT_CAPATH
-# HTTP.SSL.VALIDATE
Type: boolean ("1"/"0")
Description: Cause the client to verify the server's presented certificate.
Related CURL Flags: CURLOPT_SSL_VERIFYPEER, CURLOPT_SSL_VERIFYHOST
-# HTTP.TIMEOUT
Type: String ("dddddd")
Description: Specify the maximum time in seconds that you allow the http transfer operation to take.
Related CURL Flags: CURLOPT_TIMEOUT, CURLOPT_NOSIGNAL
-# HTTP.PROXY_SERVER
Type: String representing url to access the proxy: (e.g.http://[username:password@]host[:port])
Description: Specify the needed information for accessing a proxy.
Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD
-# HTTP.READ.BUFFERSIZE
Type: String ("dddddd")
Description: Specify the the internal buffer size for curl reads.
Related CURL Flags: CURLOPT_BUFFERSIZE, CURL_MAX_WRITE_SIZE (16kB),
CURL_MAX_READ_SIZE (512kB).
-# HTTP.KEEPALIVE
Type: String ("on|n/m")
Description: Specify that TCP KEEPALIVE should be enabled and that the associated idle wait time is n and that the associated repeat interval is m. If the value is of the form is the string "on", then turn on keepalive, but do not set idle or interval.
Related CURL Flags: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE,
CURLOPT_TCP_KEEPINTVL.
The related curl flags line indicates the curl flags modified by this
key. See the libcurl documentation of the _curl_easy_setopt()_ function
for more detail (http://curl.haxx.se/libcurl/c/curl_easy_setopt.html).
For servers that require client authentication, as a rule, the following
.daprc entries should be specified.
````
HTTP.SSL.VALIDATE
HTTP.COOKIEJAR
HTTP.SSL.CERTIFICATE
HTTP.SSL.KEY
HTTP.SSL.CAPATH
````
Additionally, the _HTTP.SSL.CERTIFICATE_ and _HTTP.SSL.KEY_
entries should have same value, which is the file path for a
certificate. The HTTP.SSL.CAPATH entry should
be the path to a directory containing validation "certificates".
# Point of Contact {#dap4_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 6/5/2017<br>
__Last Revised__: 11/7/2018
*/

624
docs/DAP4.md Normal file
View File

@ -0,0 +1,624 @@
# DAP4 Introduction {#dap4_intro}
Beginning with netCDF version 4.5.0, optional support is provided for
accessing data through servers supporting the DAP4 protocol.
DAP4 support is enabled if the _--enable-dap__ option
is used with _./configure_. If DAP4 support is enabled, then
a usable version of _libcurl_ must be specified
using the _LDFLAGS_ environment variable (similar to the way
that the _HDF5_ libraries are referenced).
Refer to the installation manual for details.
By default DAP4 support is enabled if _libcurl_ is found.
DAP4 support can be disabled using the _--disable-dap_.
DAP4 uses a data model that is by design similar to the netCDF enhanced
(netCDF-4) data model.
Generically, the DAP4 meta-data is encoded textually in a _DMR_
object. For detailed information about DAP4, refer to the
DAP4 specification
https://docs.opendap.org/index.php/OPULS_Development#DAP4_Specification
# Accessing DAP4 Data {#dap4_accessing_data}
In order to access a DAP4 data source through the netCDF API, the
file name normally used is replaced with a URL with a specific
format. The URL is composed of three parts.
- URL - this is a standard form URL such as
http://remotetest.unidata.ucar.edu/d4ts/test.01
- Constraints - these are suffixed to the URL in the query part
of the url (“?\<constraint\>”. The structure of the constraint
is somewhat complicated; and DAP4 specification
should be consulted. The interaction of DAP4
constraints with netCDF is complex and at the moment requires an
understanding of how DAP4 is translated to netCDF.
- Client parameters - these may be specified in either of
two ways. The older, deprecated form prefixes text to the
front of the url and is of the the general form [\<name>]
or [\<name>=value]. Examples include [show=fetch] and
[noprefetch]. The newer, preferred form prefixes the
parameters to the end of the url using the semi-standard '#'
format: e.g. http://....#show=fetch&noprefetch.
It is possible to see what the translation does to a particular
DAP4 data source by examining the DMR source through a web
browser and then examining the translation using the _ncdump -h_
command to see the netCDF Classic translation.
For example, if a web browser is given the following, this URL
will return the DMR in XML format for the specified dataset.
````
http://149.165.169.123:8080/d4ts/testfiles/test_one_var.nc.dmr.xml
````
Then by using the following ncdump command, it is possible to see the
equivalent netCDF enhanced translation.
````
ncdump -h http://149.165.169.123:8080/d4ts/testfiles/test_one_var.nc#dap4
````
Note the use of the "dap4" fragment added at the end. This tells the netCDF
library to use the DAP4 protocol instead of the default DAP2 protocol.
The DMR output from the web server should look like this.
````
<Dataset name="test_one_var.nc" dapVersion="4.0" dmrVersion="1.0">
<Int32 name="t"/>
<Attribute name="_DAP4_Little_Endian" type="UInt8">
<Value value="1"/>
</Attribute>
</Dataset>
````
The output from ncdump should look like this.
````
netcdf test_one_var {
variables:
int t ;
// global attributes:
:_DAP4_Little_Endian = 1UB ;
}
````
# DAP4 to NetCDF Translation Rules {#dap4_to_netcdf}
The netCDF library DAP4 code translates the DAP4 data model
into the netCDF enhanced (netCDF-4) data model.
## netCDF-4 Translation Rules {#dap4_nc42_trans_rules}
For illustrative purposes, the following example DMR will be used.
````
<Dataset name="test_groups1.nc" dapVersion="4.0" dmrVersion="1.0">
<Dimension name="dim1" size="5"/>
<Attribute name="_DAP4_Little_Endian" type="UInt8"><Value value="1"/></Attribute>
<Group name="g">
<Dimension name="dim2" size="3"/>
<Group name="h">
<Dimension name="dim3" size="7"/>
<Int32 name="v1"><Dim name="/dim1"/></Int32>
<Float32 name="v2"><Dim name="/g/dim2"/></Float32>
</Group>
<Group name="i">
<Dimension name="dim3" size="7"/>
<Int32 name="v1"><Dim name="/dim1"/></Int32>
<Float32 name="v3"><Dim name="/g/i/dim3"/></Float32>
</Group>
</Group>
</Dataset>
````
This will translate (via ncdump) into this.
````
netcdf test_groups1 {
dimensions:
dim1 = 5 ;
// global attributes:
:_DAP4_Little_Endian = 1UB ;
group: g {
dimensions:
dim2 = 3 ;
group: h {
dimensions:
dim3 = 7 ;
variables:
int v1(dim1) ;
float v2(dim2) ;
} // group h
group: i {
dimensions:
dim3 = 7 ;
variables:
int v1(dim1) ;
float v3(dim3) ;
} // group i
} // group g
}
````
## Variable Definition {#dap4_var2_def}
The set of netCDF variables is derived from the fields with primitive
base types as they occur in Sequences, Grids, and Structures. The
field names are modified to be fully qualified initially. For the
above, the set of variables are as follows. The coordinate variables
within grids are left out in order to mimic the behavior of libnc-dap4.
````
f1
S1.f11
S1.FS2.f1
S1.FS2.f2
S2.G1.temp
S2.G2.G2
lat
lon
````
## DAP4 Reserved Keywords {#dap4_reserved_keywords}
In the OPeNDAP DAP4 protocol, there are a number of reserved keywords. These keywords are case insensitive and if you use one as a netCDF variable name, you may encounter odd behavior such as case changes (depending on the client DDS/DAS parser). The list of reserved keywords as used by the netCDF-C library parser are as follows:
- alias
- array
- attributes
- byte
- dataset
- error
- float32
- float64
- grid
- int16
- int32
- maps
- sequence
- string
- structure
- uint16
- uint32
- url
- code
- message
- program_type
- program
## Variable Dimension Translation {#dap4_var_dim_trans}
A variable's rank is determined from three sources.
- The variable has the dimensions associated with the field it
represents (e.g. S1.FS2.f1[3] in the above example).
- The variable inherits the dimensions associated with any containing
structure that has a rank greater than zero. These dimensions precede
those of case 1. Thus, we have in our example, f1[2][3], where the
first dimension comes from the containing Structure FS2[2].
- The variable's set of dimensions are altered if any of its
containers is a DAP4 DDS Sequence. This is discussed more fully below.
If the type of the netCDF variable is char, then an extra string
dimension is added as the last dimension.
## Dimension translation {#dap4_dim2_trans}
For dimensions, the rules are as follows.
Fields in dimensioned structures inherit the dimension of the
structure; thus the above list would have the following dimensioned
variables.
````
S1.FS2.f1 -> S1.FS2.f1[2][3]
S1.FS2.f2 -> S1.FS2.f2[2]
S2.G1.temp -> S2.G1.temp[lat=2][lon=2]
S2.G1.lat -> S2.G1.lat[lat=2]
S2.G1.lon -> S2.G1.lon[lon=2]
S2.G2.G2 -> S2.G2.lon[lat=2][lon=2]
S2.G2.lat -> S2.G2.lat[lat=2]
S2.G2.lon -> S2.G2.lon[lon=2]
lat -> lat[lat=2]
lon -> lon[lon=2]
````
Collect all of the dimension specifications from the DDS, both named
and anonymous (unnamed) For each unique anonymous dimension with value
NN create a netCDF dimension of the form "XX_\<i\>=NN", where XX is the
fully qualified name of the variable and i is the i'th (inherited)
dimension of the array where the anonymous dimension occurs. For our
example, this would create the following dimensions.
````
S1.FS2.f1_0 = 2 ;
S1.FS2.f1_1 = 3 ;
S1.FS2.f2_0 = 2 ;
S2.G2.lat_0 = 2 ;
S2.G2.lon_0 = 2 ;
````
If however, the anonymous dimension is the single dimension of a MAP
vector in a Grid then the dimension is given the same name as the map
vector This leads to the following.
````
S2.G2.lat_0 -> S2.G2.lat
S2.G2.lon_0 -> S2.G2.lon
````
For each unique named dimension "<name>=NN", create a netCDF dimension
of the form "<name>=NN", where name has the qualifications removed. If
this leads to duplicates (i.e. same name and same value), then the
duplicates are ignored. This produces the following.
````
S2.G2.lat -> lat
S2.G2.lon -> lon
````
Note that this produces duplicates that will be ignored later.
At this point the only dimensions left to process should be named
dimensions with the same name as some dimension from step number 3,
but with a different value. For those dimensions create a dimension of
the form "<name>M=NN" where M is a counter starting at 1. The example
has no instances of this.
Finally and if needed, define a single UNLIMITED dimension named
"unlimited" with value zero. Unlimited will be used to handle certain
kinds of DAP4 sequences (see below).
This leads to the following set of dimensions.
````
dimensions:
unlimited = UNLIMITED;
lat = 2 ;
lon = 2 ;
S1.FS2.f1_0 = 2 ;
S1.FS2.f1_1 = 3 ;
S1.FS2.f2_0 = 2 ;
````
## Variable Name Translation {#dap4_var_name_trans}
The steps for variable name translation are as follows.
Take the set of variables captured above. Thus for the above DDS, the
following fields would be collected.
````
f1
S1.f11
S1.FS2.f1
S1.FS2.f2
S2.G1.temp
S2.G2.G2
lat
lon
````
All grid array variables are renamed to be the same as the containing
grid and the grid prefix is removed. In the above DDS, this results in
the following changes.
````
G1.temp -> G1
G2.G2 -> G2
````
It is important to note that this process could produce duplicate
variables (i.e. with the same name); in that case they are all assumed
to have the same content and the duplicates are ignored. If it turns
out that the duplicates have different content, then the translation
will not detect this. YOU HAVE BEEN WARNED.
The final netCDF-3 schema (minus attributes) is then as follows.
````
netcdf t {
dimensions:
unlimited = UNLIMITED ;
lat = 2 ;
lon = 2 ;
S1.FS2.f1_0 = 2 ;
S1.FS2.f1_1 = 3 ;
S1.FS2.f2_0 = 2 ;
variables:
int f1 ;
int lat(lat) ;
int lon(lon) ;
int S1.f11 ;
int S1.FS2.f1(S1.FS2.f1_0, S1.FS2.f1_1) ;
int S1.FS2.f2(S1_FS2_f2_0) ;
float S2.G1(lat, lon) ;
float G2(lat, lon) ;
}
````
In practice, the unlimited dimension is dropped because it is unused.
There are differences with the original libnc-dap4 here because
libnc-dap4 technically was incorrect. The original would have said
this, for example.
````
int S1.FS2.f1(lat, lat) ;
````
Note that this is incorrect because it dimensions S1.FS2.f1(2,2)
rather than S1.FS2.f1(2,3).
## Translating DAP4 DDS Sequences {#dap4_translation}
Any variable (as determined above) that is contained directly or
indirectly by a Sequence is subject to revision of its rank using the
following rules.
Let the variable be contained in Sequence Q1, where Q1 is the
innermost containing sequence. If Q1 is itself contained (directly or
indirectly) in a sequence, or Q1 is contained (again directly or
indirectly) in a structure that has rank greater than 0, then the
variable will have an initial UNLIMITED dimension. Further, all
dimensions coming from "above" and including (in the containment
sense) the innermost Sequence, Q1, will be removed and replaced by
that single UNLIMITED dimension. The size associated with that
UNLIMITED is zero, which means that its contents are inaccessible
through the netCDF-3 API. Again, this differs from libnc-dap4, which
leaves out such variables. Again, however, this difference is backward
compatible.
If the variable is contained in a single Sequence (i.e. not nested)
and all containing structures have rank 0, then the variable will have
an initial dimension whose size is the record count for that
Sequence. The name of the new dimension will be the name of the
Sequence.
Consider this example.
````
Dataset {
Structure {
Sequence {
Int32 f1[3];
Int32 f2;
} SQ1;
} S1[2];
Sequence {
Structure {
Int32 x1[7];
} S2[5];
} Q2;
} D;
````
The corresponding netCDF-3 translation is pretty much as follows (the
value for dimension Q2 may differ).
````
dimensions:
unlimited = UNLIMITED ; // (0 currently)
S1.SQ1.f1_0 = 2 ;
S1.SQ1.f1_1 = 3 ;
S1.SQ1.f2_0 = 2 ;
Q2.S2.x1_0 = 5 ;
Q2.S2.x1_1 = 7 ;
Q2 = 5 ;
variables:
int S1.SQ1.f1(unlimited, S1.SQ1.f1_1) ;
int S1.SQ1.f2(unlimited) ;
int Q2.S2.x1(Q2, Q2.S2.x1_0, Q2.S2.x1_1) ;
````
Note that for example S1.SQ1.f1_0 is not actually used because it has
been folded into the unlimited dimension.
Note that for sequences without a leading unlimited dimension, there
is a performance cost because the translation code has to walk the
data to determine how many records are associated with the
sequence. Since libnc-dap4 did essentially the same thing, it can be
assumed that the cost is not prohibitive.
# Caching {#dap4_dap4_caching}
In an effort to provide better performance for some access patterns,
client-side caching of data is available. The default is no caching,
but it may be enabled by prefixing the URL with the parameter "cache".
Caching operates basically as follows.
When a URL is first accessed using _nc_open()_, netCDF automatically
does a pre-fetch of selected variables. These include all variables
smaller than a specified (and user definable) size. This allows, for
example, quick access to coordinate variables. This can be suppressed
with the parameter "noprefetch".
Whenever a request is made using some variant of the _nc_get_var()_ API
procedures, the complete variable is fetched and stored in the cache
as a new cache entry. Subsequence requests for any part of that
variable will access the cache entry to obtain the data.
The cache may become too full, either because there are too many
entries or because it is taking up too much disk space. In this case
cache entries are purged until the cache size limits are reached. The
cache purge algorithm is LRU (least recently used) so that variables
that are repeatedly referenced will tend to stay in the cache.
The cache is completely purged when _nc_close()_ is invoked.
In order to decide if you should enable caching, you will need to have
some understanding of the access patterns of your program.
The ncdump program always dumps one or more whole variables so it
turns on caching.
If your program accesses only parts of a number of variables, then
caching should probably not be used since fetching whole variables
will probably slow down your program for no purpose.
Unfortunately, caching is currently an all or nothing proposition, so
for more complex access patterns, the decision to cache or not may not
have an obvious answer. Probably a good rule of thumb is to avoid
caching initially and later turn it on to see its effect on
performance.
# Defined Client Parameters {#dap4_dap4_defined_params}
Currently, a limited set of client parameters is
recognized. Parameters not listed here are ignored, but no error is
signalled. All names are case insensitive.
Parameter Name Legal Values Semantics
- "log" | "log=<file>" - Turn on logging and send the log output to
the specified file. If no file is specified, then log output is sent
to standard error.
- "show=... das|dds|url" - This causes information to appear as
specific global attributes. The currently recognized tags are "dds"
to display the underlying DDS, "das" similarly, and "url" to display
the url used to retrieve the data. This parameter may be specified
multiple times (e.g. “show=dds&show=url”).
- "show=fetch" - This parameter causes the netCDF code to log a copy
of the complete url for every HTTP get request. If logging is
enabled, then this can be helpful in checking to see the access
behavior of the netCDF code.
- "stringlength=NN" - Specify the default string length to use for
string dimensions. The default is 64. The name "maxstrlen" is an
alias for "stringlength".
- "stringlength_\<var\>=NN" - Specify the default string length to use
for a string dimension for the specified variable. The default is
64. The name "maxstrlen_\<var\>" is an alias for "stringlength_\<var\>".
- "cache" - This enables caching.
- "nocache" - This disbles caching.
- "cachelimit=NN" - Specify the maximum amount of space allowed for
the cache.
- "cachecount=NN" - Specify the maximum number of entries in the
cache.
- "prefetch" - This enables prefetch of small variables (default).
- "noprefetch" - This disables prefetch of small variables.
- "fillmismatch" - This enables _FillValue/Variable type mismatch.
- "nofillmismatch" - This disables _FillValue/Variable type mismatch (default).
# Notes on Debugging OPeNDAP Access {#dap4_dap4_debug}
The OPeNDAP support makes use of the logging facility of the
underlying oc system (see http://www.OPeNDAP.org/oc).
Note that this is currently separate from the
existing netCDF logging facility. Turning on this logging can
sometimes give important information. Logging can be enabled by
using the client parameter "log" or "log=filename",
where the first case will send log output to standard error and the
second will send log output to the specified file.
Users should also be aware that if one is
accessing data over an NFS mount, one may see some .nfsxxxxx files;
those can be ignored.
## HTTP Configuration. {#dap4_http2_config}
Limited support for configuring the http connection is provided via
parameters in the “.dodsrc” configuration file. The relevant .dodsrc file is
located by first looking in the current working directory, and if not
found, then looking in the directory specified by the “$HOME”
environment variable.
Entries in the .dodsrc file are of the form:
````
['['<url>']']<key>=<value>
````
That is, it consists of a key name and value pair and optionally
preceded by a url enclosed in square brackets.
For given KEY and URL strings, the value chosen is as follows:
If URL is null, then look for the .dodsrc entry that has no url prefix
and whose key is same as the KEY for which we are looking.
If the URL is not null, then look for all the .dodsrc entries that
have a url, URL1, say, and for which URL1 has the same host and port
as URL. All parts of the url's except host and port are ignored.
For example, if URL = http//x.y/a, then it will match
entries of the form
_[http//x.y/a]KEY=VALUE_ or _[http//x.y/b]KEY=VALUE_.
It will not match an entry of the form _[http//x.y:8080]KEY=VALUE
because the second has a port number (8080) different than the URL.
Finally from the set so constructed, choose the first matching entry.
Currently, the supported set of keys (with descriptions) are as
follows.
1. HTTP.VERBOSE
Type: boolean ("1"/"0")
Description: Produce verbose output, especially using SSL.
Related CURL Flags: CURLOPT_VERBOSE
1. HTTP.DEFLATE
Type: boolean ("1"/"0")
Description: Allow use of compression by the server.
Related CURL Flags: CURLOPT_ENCODING
1. HTTP.COOKIEJAR
Type: String representing file path
Description: Specify the name of file into which to store cookies. Defaults to in-memory storage.
Related CURL Flags:CURLOPT_COOKIEJAR
1. HTTP.CREDENTIALS.USER
Type: String representing user name
Description: Specify the user name for Digest and Basic authentication.
Related CURL Flags:
1. HTTP.CREDENTIALS.PASSWORD
Type: String representing password
Type: boolean ("1"/"0")
Description: Specify the password for Digest and Basic authentication.
Related CURL Flags:
1. HTTP.SSL.CERTIFICATE
Type: String representing file path
Description: Path to a file containing a PEM cerficate.
Related CURL Flags: CURLOPT_CERT
1. HTTP.SSL.KEY
Type: String representing file path
Description: Same as HTTP.SSL.CERTIFICATE, and should usually have the same value.
Related CURL Flags: CURLOPT_SSLKEY
1. HTTP.SSL.KEYPASSWORD
Type: String representing password
Description: Password for accessing the HTTP.SSL.KEY/HTTP.SSL.CERTIFICATE
Related CURL Flags: CURLOPT_KEYPASSWORD
1. HTTP.SSL.CAPATH
Type: String representing directory
Description: Path to a directory containing trusted certificates for validating server certificates.
Related CURL Flags: CURLOPT_CAPATH
1. HTTP.SSL.VALIDATE
Type: boolean ("1"/"0")
Description: Cause the client to verify the server's presented certificate.
Related CURL Flags: CURLOPT_SSL_VERIFYPEER, CURLOPT_SSL_VERIFYHOST
1. HTTP.TIMEOUT
Type: String ("dddddd")
Description: Specify the maximum time in seconds that you allow the http transfer operation to take.
Related CURL Flags: CURLOPT_TIMEOUT, CURLOPT_NOSIGNAL
1. HTTP.PROXY_SERVER
Type: String representing url to access the proxy: (e.g.http://[username:password@]host[:port])
Description: Specify the needed information for accessing a proxy.
Related CURL Flags: CURLOPT_PROXY, CURLOPT_PROXYHOST, CURLOPT_PROXYUSERPWD
1. HTTP.READ.BUFFERSIZE
Type: String ("dddddd")
Description: Specify the the internal buffer size for curl reads.
Related CURL Flags: CURLOPT_BUFFERSIZE, CURL_MAX_WRITE_SIZE (16kB),
CURL_MAX_READ_SIZE (512kB).
1. HTTP.KEEPALIVE
Type: String ("on|n/m")
Description: Specify that TCP KEEPALIVE should be enabled and that the associated idle wait time is n and that the associated repeat interval is m. If the value is of the form is the string "on", then turn on keepalive, but do not set idle or interval.
Related CURL Flags: CURLOPT_TCP_KEEPALIVE, CURLOPT_TCP_KEEPIDLE,
CURLOPT_TCP_KEEPINTVL.
The related curl flags line indicates the curl flags modified by this
key. See the libcurl documentation of the _curl_easy_setopt()_ function
for more detail (http://curl.haxx.se/libcurl/c/curl_easy_setopt.html).
For ESG client side key support, the following entries must be specified:
````
HTTP.SSL.VALIDATE
HTTP.COOKIEJAR
HTTP.SSL.CERTIFICATE
HTTP.SSL.KEY
HTTP.SSL.CAPATH
````
Additionally, for ESG, the _HTTP.SSL.CERTIFICATE_ and _HTTP.SSL.KEY_
entries should have same value, which is the file path for the
certificate produced by MyProxyLogon. The HTTP.SSL.CAPATH entry should
be the path to the "certificates" directory produced by MyProxyLogon.
# Point of Contact {#dap4_poc}
__Author__: Dennis Heimbigner<br>
__Email__: dmh at ucar dot edu<br>
__Initial Version__: 6/5/2017<br>
__Last Revised__: 9/25/2018

2
docs/Makefile.am
View File

@ -11,7 +11,7 @@ install-fortran.md all-error-codes.md credits.md auth.md \
obsolete/fan_utils.html bestpractices.md filters.md indexing.dox \
inmemory.md DAP2.dox attribute_conventions.md FAQ.md \
file_format_specifications.md known_problems.md COPYRIGHT.dox \
user_defined_formats.md inmeminternal.dox
user_defined_formats.md inmeminternal.dox DAP4.md DAP4.dox
# Turn off parallel builds in this directory.
.NOTPARALLEL:

4
docs/guide.dox
View File

@ -8,12 +8,12 @@
- \subpage netcdf_data_set_components
- \subpage netcdf_perf_chunking
- \subpage netcdf_utilities_guide
- \subpage dap2
- \subpage dap4
- \subpage compress
- \subpage BestPractices
- \subpage user_defined_formats
- \subpage users_guide_appendices
- \subpage dap2
- \subpage dap4
<H2>The Purpose of NetCDF</H2>

6
h5_test/tst_h_files.c
View File

@ -244,7 +244,7 @@ main()
char file_name[STR_LEN + 1];
char dimscale_wo_var[STR_LEN];
void *bufr;
void *fillp;
void *fillp = NULL;
sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME);
@ -305,7 +305,7 @@ main()
if (!(fillp = malloc(1))) ERR;
#define FILL_BYTE 255
*(signed char *)fillp = FILL_BYTE;
if (H5Pset_fill_value(plistid, H5T_NATIVE_SCHAR, fillp) < 0) ERR;
if(H5Pset_fill_value(plistid, H5T_NATIVE_SCHAR, fillp) < 0) ERR;
if (!(chunksize = malloc(NDIMS2 * sizeof(hsize_t)))) ERR;
chunksize[0] = 1024;
chunksize[1] = 1048576;
@ -319,7 +319,7 @@ main()
if ((spaceid = H5Screate_simple(NDIMS2, dimsize, maxdimsize)) < 0) ERR;
if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
H5P_CRT_ORDER_INDEXED) < 0) ERR;
if ((datasetid = H5Dcreate(grpid, VAR_NAME2, H5T_NATIVE_SCHAR, spaceid, plistid)) < 0)
if ((datasetid = H5Dcreate(grpid, VAR_NAME2, H5T_NATIVE_SCHAR, spaceid, plistid)) < 0) ERR;
free(fillp);
free(chunksize);

25
include/ncaux.h
View File

@ -1,25 +0,0 @@
/*********************************************************************
* Copyright 2010, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
* $Id$
* $Header$
*********************************************************************/
#ifndef NCAUX_H
#define NCAUX_H
#define NCAUX_ALIGN_C 0
#define NCAUX_ALIGN_UNIFORM 1
extern int ncaux_begin_compound(int ncid, const char *name, int alignmode,
void** tag);
extern int ncaux_end_compound(void* tag, nc_type* typeid);
extern int ncaux_abort_compound(void* tag);
extern int ncaux_add_field(void* tag, const char *name, nc_type field_type,
int ndims, const int* dimsizes);
#endif /*NCAUX_H*/

82
include/ncoffsets.h
View File

@ -6,51 +6,53 @@
#ifndef NCOFFSETS_H
#define NCOFFSETS_H 1
typedef struct Alignment {
char* type_name;
unsigned int alignment;
} Alignment;
/* Define indices for every primitive C type */
/* NAT => NOT-A-TYPE*/
#define NATINDEX 0
#define CHARINDEX 1
#define UCHARINDEX 2
#define SHORTINDEX 3
#define USHORTINDEX 4
#define INTINDEX 5
#define UINTINDEX 6
#define LONGINDEX 7
#define ULONGINDEX 8
#define LONGLONGINDEX 9
#define ULONGLONGINDEX 10
#define FLOATINDEX 11
#define DOUBLEINDEX 12
#define PTRINDEX 13
#define NCVLENINDEX 14
#define NC_NATINDEX 0
#define NC_CHARINDEX 1
#define NC_UCHARINDEX 2
#define NC_SHORTINDEX 3
#define NC_USHORTINDEX 4
#define NC_INTINDEX 5
#define NC_UINTINDEX 6
#define NC_LONGINDEX 7
#define NC_ULONGINDEX 8
#define NC_LONGLONGINDEX 9
#define NC_ULONGLONGINDEX 10
#define NC_FLOATINDEX 11
#define NC_DOUBLEINDEX 12
#define NC_PTRINDEX 13
#define NC_NCVLENINDEX 14
#define NCTYPES 15
#define NC_NCTYPES 15
typedef struct NCalignment {
char* type_name;
size_t alignment;
} NCalignment;
typedef NCalignment NCtypealignvec;
/* Capture in struct and in a vector*/
typedef struct Typealignset {
Alignment charalign; /* char*/
Alignment ucharalign; /* unsigned char*/
Alignment shortalign; /* short*/
Alignment ushortalign; /* unsigned short*/
Alignment intalign; /* int*/
Alignment uintalign; /* unsigned int*/
Alignment longalign; /* long*/
Alignment ulongalign; /* unsigned long*/
Alignment longlongalign; /* long long*/
Alignment ulonglongalign; /* unsigned long long*/
Alignment floatalign; /* float*/
Alignment doublealign; /* double*/
Alignment ptralign; /* void**/
Alignment ncvlenalign; /* nc_vlen_t*/
} Typealignset;
typedef struct NCtypealignset {
NCalignment charalign; /* char*/
NCalignment ucharalign; /* unsigned char*/
NCalignment shortalign; /* short*/
NCalignment ushortalign; /* unsigned short*/
NCalignment intalign; /* int*/
NCalignment uintalign; /* unsigned int*/
NCalignment longalign; /* long*/
NCalignment ulongalign; /* unsigned long*/
NCalignment longlongalign; /* long long*/
NCalignment ulonglongalign; /* unsigned long long*/
NCalignment floatalign; /* float*/
NCalignment doublealign; /* double*/
NCalignment ptralign; /* void**/
NCalignment ncvlenalign; /* nc_vlen_t*/
} NCtypealignset;
typedef Alignment Typealignvec;
EXTERNL unsigned int nctypealignment(nc_type nctype);
size_t NC_class_alignment(int ncclass);
void NC_compute_alignments(void);
int NC_alignments_computed;
#endif /*NCOFFSETS_H*/

11
include/netcdf_aux.h
View File

@ -29,11 +29,10 @@ we do not know how it was allocated.
Should work for any netcdf format.
*/
EXTERNL int ncaux_reclaim_data(int ncid, int xtype, const void* memory, size_t count);
EXTERNL int ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count);
EXTERNL int ncaux_begin_compound(int ncid, const char *name, int alignmode,
void** tag);
EXTERNL int ncaux_begin_compound(int ncid, const char *name, int alignmode, void** tag);
EXTERNL int ncaux_end_compound(void* tag, nc_type* typeid);
@ -42,6 +41,12 @@ EXTERNL int ncaux_abort_compound(void* tag);
EXTERNL int ncaux_add_field(void* tag, const char *name, nc_type field_type,
int ndims, const int* dimsizes);
/* Takes any type */
EXTERNL size_t ncaux_type_alignment(int xtype, int ncid);
/* Takes type classes only */
EXTERNL size_t ncaux_class_alignment(int ncclass);
#if defined(__cplusplus)
}
#endif

6
libdap2/dapdump.c
View File

@ -485,14 +485,14 @@ dumpnode(CDFnode* node)
}
char*
dumpalign(NCalignment* ncalign)
dumpalign(NCD2alignment* ncalign)
{
char* result;
char tmp[1024];
if(ncalign == NULL)
result = nulldup("NCalignment{size=-- alignment=-- offset=--}");
result = nulldup("NCD2alignment{size=-- alignment=-- offset=--}");
else {
snprintf(tmp,sizeof(tmp),"NCalignment{size=%lu alignment=%lu offset=%lu}",
snprintf(tmp,sizeof(tmp),"NCD2alignment{size=%lu alignment=%lu offset=%lu}",
ncalign->size,ncalign->alignment,ncalign->offset);
result = nulldup(tmp);
}

2
libdap2/dapdump.h
View File

@ -46,7 +46,7 @@ extern char* dumptree(CDFnode* root);
extern char* dumpvisible(CDFnode* root);
extern char* dumpnode(CDFnode* node);
extern char* dumpalign(struct NCalignment*);
extern char* dumpalign(struct NCD2alignment*);
extern char* dumpcachenode(NCcachenode* node);
extern char* dumpcache(NCcache* cache);

8
libdap2/nccommon.h
View File

@ -199,16 +199,16 @@ typedef struct NCDODS {
char* dimname;
} NCDODS;
typedef struct NCalignment {
typedef struct NCD2alignment {
unsigned long size; /* size of single instance of this type*/
unsigned long alignment; /* alignment of this field */
unsigned long offset; /* offset of this field in parent */
} NCalignment;
} NCD2alignment;
typedef struct NCtypesize {
BOOL aligned; /* have instance and field been defined? */
NCalignment instance; /* Alignment, etc for instance data */
NCalignment field; /* Alignment, etc WRT to parent */
NCD2alignment instance; /* Alignment, etc for instance data */
NCD2alignment field; /* Alignment, etc WRT to parent */
} NCtypesize;
/* Closely mimics struct OCnode*/

6
libdap4/d4data.c
View File

@ -264,7 +264,11 @@ fillstring(NCD4meta* meta, void** offsetp, void** dstp, NClist* blobs)
*dstp = dst;
offset = INCR(offset,count);
*offsetp = offset;
q = NULL; /*nclistpush(blobs,q);*/
#if 0
nclistpush(blobs,q);
#else
q = NULL;
#endif
done:
return THROW(ret);
}

11
libdap4/d4debug.c
View File

@ -8,6 +8,7 @@
#include "d4includes.h"
#include "ncdispatch.h"
#include "netcdf_aux.h"
int ncdap4debug = 0;
@ -110,8 +111,10 @@ NCD4_debugcopy(NCD4INFO* info)
int varid = var->meta.id;
d4size_t varsize;
void* memory = NULL;
size_t dimprod = NCD4_dimproduct(var);
int ncid = info->substrate.nc4id;
varsize = type->meta.memsize * NCD4_dimproduct(var);
varsize = type->meta.memsize * dimprod;
memory = d4alloc(varsize);
if(memory == NULL)
{ret = NC_ENOMEM; goto done;}
@ -138,8 +141,14 @@ NCD4_debugcopy(NCD4INFO* info)
if((ret=nc_put_vara(grpid,varid,nc_sizevector0,edges,memory)))
goto done;
}
if((ret=ncaux_reclaim_data(ncid,type->meta.id,memory,dimprod)))
goto done;
free(memory);
memory = NULL;
}
done:
if(topvars)
nclistfree(topvars);
if(ret != NC_NOERR) {
fprintf(stderr,"debugcopy: %d %s\n",ret,nc_strerror(ret));
}

406
libdap4/d4mem.c Normal file
View File

@ -0,0 +1,406 @@
/*********************************************************************
* Copyright 2009, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include <stdlib.h>
#include "netcdf.h"
#include "ncoffsets.h"
/* It is helpful to have a structure that contains memory and an offset */
typedef struct Position {char* memory; ptrdiff_t offset;} Position;
#ifndef emalloc
#define emalloc(n) malloc(n)
#define ecalloc(n) calloc(1,n)
#define erealloc(p,n) realloc(p,n)
#define efree(p) free(p)
#endif
/*Forward*/
static int d4reclaim_datar(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4reclaim_usertype(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4reclaim_vlen(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4reclaim_enum(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4reclaim_opaque(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4reclaim_compound(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_datar(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_usertype(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_vlen(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_enum(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_opaque(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static int d4clone_compound(NC_TYPE_INFO_T* tsym, Position* reclaimer);
static ptrdiff_t read_align(ptrdiff_t offset, unsigned long alignment);
static size_t hdf5typealignment(NC_TYPE_INFO_T* type);
/**
Reclaim a chunk of memory. Do not reclaim top level
because we do not know how it was allocated.
@param tsym - the root type node of the data to be reclaimed
@param memory - top-level memory whose content is to be reclaimed
@param count - number of instances of tsym in memory
*/
int
d4reclaim_data(NCD4node* tsym, void* memory, size_t count)
{
int stat = NC_NOERR;
size_t i;
Position position;
if(tsym == NULL
|| (memory == NULL && count > 0))
{stat = NC_EINVAL; goto done;}
if(memory == NULL || count == 0)
goto done; /* ok, do nothing */
if(tsym->meta.isfixedsize)
goto done; /* Should be nothing hanging off of the memory chunk */
position.offset = 0;
position.memory = memory;
for(i=0;i<count;i++) {
if((stat=d4reclaim_datar(tsym,&position))) /* reclaim one instance */
break;
}
done:
return stat;
}
/* Recursive type walker: reclaim a single instance */
static int
d4reclaim_datar(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
switch (tsym->subsort) {
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
case NC_SHORT: case NC_USHORT:
case NC_INT: case NC_UINT: case NC_FLOAT:
case NC_INT64: case NC_UINT64: case NC_DOUBLE:
position->offset += tsym->meta.memsize;
break;
case NC_STRING: {
char** sp = (char**)(position->memory+position->offset);
/* Need to reclaim string */
if(*sp != NULL) efree(*sp);
position->offset += tsym->meta.memsize;
} break;
default:
/* reclaim a user type */
stat = d4reclaim_usertype(tsym,position);
break;
}
return stat;
}
static int
d4reclaim_usertype(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
/* Get info about the xtype */
switch (tsym->nc_type_class) {
case NC_OPAQUE: stat = d4reclaim_opaque(tsym,position); break;
case NC_ENUM: stat = d4reclaim_enum(tsym,position); break;
case NC_VLEN: stat = d4reclaim_vlen(tsym,position); break;
case NC_COMPOUND: stat = d4reclaim_compound(tsym,position); break;
default:
stat = NC_EINVAL;
break;
}
return stat;
}
static int
d4reclaim_vlen(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
size_t i;
NCD4node* basetype;
nc_vlen_t* vl = (nc_vlen_t*)(position->memory+position->offset);
if((stat=nc4_find_type(tsym->container->nc4_info,tsym->u.v.base_nc_typeid,&basetype)))
goto done;
/* Free up each entry in the vlen list */
if(vl->p != NULL) {
Reclaim vposition;
vposition.memory = vl->p;
vposition.offset = 0;
for(i=0;i<vl->len;i++) {
size_t alignment = hdf5typealignment(basetype);
vposition.offset = read_align(vposition.offset,alignment);
if((stat = d4reclaim_datar(basetype,&vposition))) goto done;
vposition.offset += basetype->size;
}
position->offset += tsym->size;
efree(vl->p);
}
done:
return stat;
}
static int
d4reclaim_enum(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
NCD4node* basetype;
if((stat=nc4_find_type(tsym->container->nc4_info,tsym->u.v.base_nc_typeid,&basetype)))
return stat;
return d4reclaim_datar(basetype,position);
}
static int
d4reclaim_opaque(NCD4node* tsym, Position* position)
{
/* basically a fixed size sequence of bytes */
position->offset += tsym->size;
return NC_NOERR;
}
static int
d4reclaim_compound(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
int nfields;
size_t fid, i, arraycount;
ptrdiff_t saveoffset;
size_t cmpdalign = hdf5typealignment(tsym);
position->offset = read_align(position->offset,cmpdalign);
saveoffset = position->offset;
/* Get info about each field in turn and reclaim it */
nfields = nclistlength(tsym->u.c.field);
for(fid=0;fid<nfields;fid++) {
NC_FIELD_INFO_T* field = nclistget(tsym->u.c.field,fid);
NCD4node* fieldtype;
int ndims = field->ndims;
size_t fieldalign;
if((stat=nc4_find_type(tsym->container->nc4_info,field->nc_typeid,&fieldtype)))
goto done;
fieldalign = hdf5typealignment(fieldtype);
/* compute the total number of elements in the field array */
for(i=0;i<ndims;i++) arraycount *= field->dim_size[i];
position->offset = read_align(position->offset,fieldalign);
for(i=0;i<arraycount;i++) {
if((stat = d4reclaim_datar(fieldtype, position))) goto done;
}
}
position->offset = saveoffset;
position->offset += tsym->size;
done:
return stat;
}
/**************************************************/
int
d4clone_data(NCD4node* tsym, void* src, void** dstp, size_t count)
{
int stat = NC_NOERR;
size_t i;
Position srcpos;
Position dstpos = {NULL,0};
if(tsym == NULL)
{stat = NC_EINVAL; goto done;}
if(src == NULL && count > 0)
{stat = NC_EINVAL; goto done;}
if(src == NULL || count == 0)
goto done; /* ok, do nothing */
srcpos.offset = 0;
srcpos.memory = src;
for(i=0;i<count;i++) {
if((stat=d4clone_datar(tsym,&srcpos,&dstpos))) /* clone one instance */
break;
}
done:
return stat;
}
/* Recursive type walker: reclaim a single instance */
static int
d4clone_datar(NCD4node* tsym, Position* src, Position* dst)
{
int stat = NC_NOERR;
switch (tsym->nc_type_class) {
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
case NC_SHORT: case NC_USHORT:
case NC_INT: case NC_UINT: case NC_FLOAT:
case NC_INT64: case NC_UINT64: case NC_DOUBLE:
position->offset += tsym->size;
break;
case NC_STRING: {
char** sp = (char**)(position->memory+position->offset);
/* Need to reclaim string */
if(*sp != NULL) efree(*sp);
position->offset += tsym->size;
} break;
default:
/* reclaim a user type */
stat = d4clone_usertype(tsym,position);
break;
}
return stat;
}
static int
d4clone_usertype(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
/* Get info about the xtype */
switch (tsym->nc_type_class) {
case NC_OPAQUE: stat = d4clone_opaque(tsym,position); break;
case NC_ENUM: stat = d4clone_enum(tsym,position); break;
case NC_VLEN: stat = d4clone_vlen(tsym,position); break;
case NC_COMPOUND: stat = d4clone_compound(tsym,position); break;
default:
stat = NC_EINVAL;
break;
}
return stat;
}
static int
d4clone_vlen(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
size_t i;
NCD4node* basetype;
nc_vlen_t* vl = (nc_vlen_t*)(position->memory+position->offset);
if((stat=nc4_find_type(tsym->container->nc4_info,tsym->u.v.base_nc_typeid,&basetype)))
goto done;
/* Free up each entry in the vlen list */
if(vl->p != NULL) {
Reclaim vposition;
vposition.memory = vl->p;
vposition.offset = 0;
for(i=0;i<vl->len;i++) {
size_t alignment = hdf5typealignment(basetype);
vposition.offset = read_align(vposition.offset,alignment);
if((stat = d4clone_datar(basetype,&vposition))) goto done;
vposition.offset += basetype->size;
}
position->offset += tsym->size;
efree(vl->p);
}
done:
return stat;
}
static int
d4clone_enum(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
NCD4node* basetype;
if((stat=nc4_find_type(tsym->container->nc4_info,tsym->u.v.base_nc_typeid,&basetype)))
return stat;
return d4clone_datar(basetype,position);
}
static int
d4clone_opaque(NCD4node* tsym, Position* position)
{
/* basically a fixed size sequence of bytes */
position->offset += tsym->size;
return NC_NOERR;
}
static int
d4clone_compound(NCD4node* tsym, Position* position)
{
int stat = NC_NOERR;
int nfields;
size_t fid, i, arraycount;
ptrdiff_t saveoffset;
size_t cmpdalign = hdf5typealignment(tsym);
position->offset = read_align(position->offset,cmpdalign);
saveoffset = position->offset;
/* Get info about each field in turn and reclaim it */
nfields = nclistlength(tsym->u.c.field);
for(fid=0;fid<nfields;fid++) {
NC_FIELD_INFO_T* field = nclistget(tsym->u.c.field,fid);
NCD4node* fieldtype;
int ndims = field->ndims;
size_t fieldalign;
if((stat=nc4_find_type(tsym->container->nc4_info,field->nc_typeid,&fieldtype)))
goto done;
fieldalign = hdf5typealignment(fieldtype);
/* compute the total number of elements in the field array */
for(i=0;i<ndims;i++) arraycount *= field->dim_size[i];
position->offset = read_align(position->offset,fieldalign);
for(i=0;i<arraycount;i++) {
if((stat = d4clone_datar(fieldtype, position))) goto done;
}
}
position->offset = saveoffset;
position->offset += tsym->size;
done:
return stat;
}
/**************************************************/
/* Alignment code */
static ptrdiff_t
read_align(ptrdiff_t offset, size_t alignment)
{
size_t delta = (offset % alignment);
if(delta == 0) return offset;
return offset + (alignment - delta);
}
/*
The heart of this is the following macro,
which computes the offset of a field x
when preceded by a char field.
The assumptions appear to be as follows:
1. the offset produced in this situation indicates
the alignment for x relative in such a way that it
depends only on the types that precede it in the struct.
2. the compiler does not reorder fields.
3. arrays are tightly packed.
4. nested structs are alignd according to their first member
(this actually follows from C language requirement that
a struct can legally be cast to an instance of its first member).
Given the alignments for the various common primitive types,
it is assumed that one can use them anywhere to construct
the layout of a struct of such types.
It seems to work for HDF5 for a wide variety of machines.
*/
static size_t
hdf5typealignment(NCD4node* typ)
{
if(!nc_alignments_computed) {
nc_compute_alignments();
nc_alignments_computed = 1;
}
if(typ->hdr.id <= NC_MAX_ATOMIC_TYPE)
return nctypealignment(typ->hdr.id);
else {/* Presumably a user type */
switch(typ->nc_type_class) {
case NC_VLEN: return nctypealignment(typ->hdr.id);
case NC_OPAQUE: return nctypealignment(typ->hdr.id);
case NC_COMPOUND: {/* get alignment of the first field of the compound */
NC_FIELD_INFO_T* field0 = nclistget(typ->u.c.field,0);
NCD4node* fieldtype = NULL;
if(nc4_find_type(typ->container->nc4_info,field0->nc_typeid,&fieldtype))
return 0;
return hdf5typealignment(fieldtype); /* may recurse repeatedly */
} break;
default: break;
}
}
return 0; /* fail */
}

10
libdap4/d4util.c
View File

@ -423,16 +423,6 @@ NCD4_userpwd(NCURI* uri, char* space, size_t len)
}
}
#ifdef BLOB
void
NCD4_saveblob(NCD4meta* meta, void* mem)
{
if(meta->blobs == NULL)
meta->blobs = nclistnew();
nclistpush(meta->blobs,mem);
}
#endif
/**************************************************/
/* Error reporting */

406
libdispatch/daux.c
View File

@ -24,6 +24,8 @@ See LICENSE.txt for license information.
#include "config.h"
#include "netcdf.h"
#include "netcdf_aux.h"
#include "ncoffsets.h"
#include "nclog.h"
struct NCAUX_FIELD {
char* name;
@ -47,21 +49,23 @@ struct NCAUX_CMPD {
};
/* It is helpful to have a structure that contains memory and an offset */
typedef struct Position{char* memory; ptrdiff_t offset;} Position;
/* Forward */
static int reclaim_datar(int ncid, int xtype, void* memory);
#ifdef USE_NETCDF4
static int reclaim_usertype(int ncid, int xtype, void* memory);
static int reclaim_compound(int ncid, int xtype, size_t size, size_t nfields, void* memory);
static int reclaim_vlen(int ncid, int xtype, int basetype, void* memory);
static int reclaim_enum(int ncid, int xtype, int basetype, void* memory);
static int reclaim_opaque(int ncid, int xtype, size_t size, void* memory);
#endif
static int reclaim_datar(int ncid, int xtype, size_t typesize, Position*);
static int ncaux_initialized = 0;
#ifdef USE_NETCDF4
static int ncaux_initialized = 0;
static void compute_alignments(void);
static int reclaim_usertype(int ncid, int xtype, Position* offset);
static int reclaim_compound(int ncid, int xtype, size_t size, size_t nfields, Position* offset);
static int reclaim_vlen(int ncid, int xtype, int basetype, Position* offset);
static int reclaim_enum(int ncid, int xtype, int basetype, size_t, Position* offset);
static int reclaim_opaque(int ncid, int xtype, size_t size, Position* offset);
static int computefieldinfo(struct NCAUX_CMPD* cmpd);
#endif
#endif /* USE_NETCDF4 */
/**************************************************/
@ -78,15 +82,20 @@ we do not know how it was allocated.
Should work for any netcdf format.
@param ncid file ncid
@param xtype type id
@param memory to reclaim
@param count number of instances of the type in memory
@return error code
*/
int
ncaux_reclaim_data(int ncid, int xtype, const void* memory, size_t count)
EXTERNL int
ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count)
{
int stat = NC_NOERR;
size_t typesize = 0;
char* p;
size_t i;
Position offset;
if(ncid < 0 || xtype < 0
|| (memory == NULL && count > 0)
@ -95,9 +104,10 @@ ncaux_reclaim_data(int ncid, int xtype, const void* memory, size_t count)
if(memory == NULL || count == 0)
goto done; /* ok, do nothing */
if((stat=nc_inq_type(ncid,xtype,NULL,&typesize))) goto done;
p = (char*)memory; /* use char* so we can do pointer arithmetic */
for(i=0;i<count;i++,p+=typesize) {
if((stat=reclaim_datar(ncid,xtype,p))) /* reclaim one instance */
offset.memory = (char*)memory; /* use char* so we can do pointer arithmetic */
offset.offset = 0;
for(i=0;i<count;i++) {
if((stat=reclaim_datar(ncid,xtype,typesize,&offset))) /* reclaim one instance */
break;
}
@ -107,24 +117,28 @@ done:
/* Recursive type walker: reclaim a single instance */
static int
reclaim_datar(int ncid, int xtype, void* memory)
reclaim_datar(int ncid, int xtype, size_t typesize, Position* offset)
{
int stat = NC_NOERR;
switch (xtype) {
case NC_CHAR: case NC_BYTE: case NC_UBYTE: break;
case NC_SHORT: case NC_USHORT: break;
case NC_INT: case NC_UINT: case NC_FLOAT: break;
case NC_INT64: case NC_UINT64: case NC_DOUBLE: break;
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
case NC_SHORT: case NC_USHORT:
case NC_INT: case NC_UINT: case NC_FLOAT:
case NC_INT64: case NC_UINT64: case NC_DOUBLE:
offset->offset += typesize;
break;
#ifdef USE_NETCDF4
case NC_STRING: {
char** sp = (char**)memory;
char** sp = (char**)(offset->memory + offset->offset);
/* Need to reclaim string */
if(*sp != NULL) free(*sp);
offset->offset += typesize;
} break;
default:
/* reclaim a user type */
stat = reclaim_usertype(ncid,xtype,memory);
stat = reclaim_usertype(ncid,xtype,offset);
#else
default:
stat = NC_ENOTNC4;
@ -135,8 +149,17 @@ reclaim_datar(int ncid, int xtype, void* memory)
}
#ifdef USE_NETCDF4
static ptrdiff_t
read_align(ptrdiff_t offset, size_t alignment)
{
size_t delta = (offset % alignment);
if(delta == 0) return offset;
return offset + (alignment - delta);
}
static int
reclaim_usertype(int ncid, int xtype, void* memory)
reclaim_usertype(int ncid, int xtype, Position* offset)
{
int stat = NC_NOERR;
size_t size;
@ -144,15 +167,13 @@ reclaim_usertype(int ncid, int xtype, void* memory)
size_t nfields;
int klass;
if(memory == NULL) return NC_EINVAL;
/* Get info about the xtype */
stat = nc_inq_user_type(ncid, xtype, NULL, &size, &basetype, &nfields, &klass);
switch (klass) {
case NC_OPAQUE: stat = reclaim_opaque(ncid,xtype,size,memory); break;
case NC_ENUM: stat = reclaim_enum(ncid,xtype,basetype,memory); break;
case NC_COMPOUND: stat = reclaim_compound(ncid,xtype,size,nfields,memory); break;
case NC_VLEN: stat = reclaim_vlen(ncid,xtype,basetype,memory); break;
case NC_OPAQUE: stat = reclaim_opaque(ncid,xtype,size,offset); break;
case NC_ENUM: stat = reclaim_enum(ncid,xtype,basetype,size,offset); break;
case NC_COMPOUND: stat = reclaim_compound(ncid,xtype,size,nfields,offset); break;
case NC_VLEN: stat = reclaim_vlen(ncid,xtype,basetype,offset); break;
default:
stat = NC_EINVAL;
break;
@ -161,47 +182,25 @@ reclaim_usertype(int ncid, int xtype, void* memory)
}
static int
reclaim_compound(int ncid, int xtype, size_t size, size_t nfields, void* memory)
reclaim_vlen(int ncid, int xtype, int basetype, Position* offset)
{
int stat = NC_NOERR;
size_t fid, offset, i, fieldsize, arraycount;
int dimsizes[NC_MAX_VAR_DIMS];
int ndims;
nc_type fieldtype;
char* p;
/* Get info about each field in turn and reclaim it */
for(fid=0;fid<nfields;fid++) {
if((stat = nc_inq_compound_field(ncid,xtype,fid,NULL,&offset, &fieldtype, &ndims, dimsizes))) goto done;
if((stat = nc_inq_type(ncid,fieldtype,NULL,&fieldsize))) goto done;
if(ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* compute the total number of elements in the field array */
arraycount = 1;
for(i=0;i<ndims;i++) arraycount *= dimsizes[i];
for(i=0;i<arraycount;i++) {
p = ((char*)memory) + offset + (i*fieldsize);
if((stat = reclaim_datar(ncid, fieldtype, p))) goto done;
}
}
done:
return stat;
}
static int
reclaim_vlen(int ncid, int xtype, int basetype, void* memory)
{
int stat = NC_NOERR;
nc_vlen_t* vl = (nc_vlen_t*)memory;
size_t i, size;
size_t i, basesize;
nc_vlen_t* vl = (nc_vlen_t*)(offset->memory+offset->offset);
/* Get size of the basetype */
if((stat=nc_inq_type(ncid,basetype,NULL,&size))) goto done;
if((stat=nc_inq_type(ncid,basetype,NULL,&basesize))) goto done;
/* Free up each entry in the vlen list */
if(vl->p != NULL) {
char* p = vl->p;
for(i=0;i<vl->len;i++,p+=size)
if((stat = reclaim_datar(ncid,basetype,p))) goto done;
Position voffset;
unsigned int alignment = ncaux_type_alignment(basetype,ncid);
voffset.memory = vl->p;
voffset.offset = 0;
for(i=0;i<vl->len;i++) {
voffset.offset = read_align(voffset.offset,alignment);
if((stat = reclaim_datar(ncid,basetype,basesize,&voffset))) goto done;
}
offset->offset += sizeof(nc_vlen_t);
free(vl->p);
}
@ -210,21 +209,60 @@ done:
}
static int
reclaim_enum(int ncid, int xtype, int basetype, void* memory)
reclaim_enum(int ncid, int xtype, int basetype, size_t basesize, Position* offset)
{
/* basically same as an instance of the enum's integer basetype */
return reclaim_datar(ncid,basetype,basesize,offset);
}
static int
reclaim_opaque(int ncid, int xtype, size_t opsize, Position* offset)
{
/* basically a fixed size sequence of bytes */
offset->offset += opsize;
return NC_NOERR;
}
static int
reclaim_opaque(int ncid, int xtype, size_t size, void* memory)
reclaim_compound(int ncid, int xtype, size_t cmpdsize, size_t nfields, Position* offset)
{
/* basically a fixed size sequence of bytes */
return NC_NOERR;
int stat = NC_NOERR;
size_t fid, fieldoffset, i, fieldsize, arraycount;
int dimsizes[NC_MAX_VAR_DIMS];
int ndims;
nc_type fieldtype;
ptrdiff_t saveoffset;
saveoffset = offset->offset;
/* Get info about each field in turn and reclaim it */
for(fid=0;fid<nfields;fid++) {
unsigned int fieldalignment;
/* Get all relevant info about the field */
if((stat = nc_inq_compound_field(ncid,xtype,fid,NULL,&fieldoffset, &fieldtype, &ndims, dimsizes))) goto done;
fieldalignment = ncaux_type_alignment(fieldtype,ncid);
if((stat = nc_inq_type(ncid,fieldtype,NULL,&fieldsize))) goto done;
if(ndims == 0) {ndims=1; dimsizes[0]=1;} /* fake the scalar case */
/* Align to this field */
offset->offset = read_align(offset->offset,fieldalignment);
/* compute the total number of elements in the field array */
arraycount = 1;
for(i=0;i<ndims;i++) arraycount *= dimsizes[i];
for(i=0;i<arraycount;i++) {
if((stat = reclaim_datar(ncid, fieldtype, fieldsize, offset))) goto done;
}
}
/* Return to beginning of the compound and move |compound| */
offset->offset = saveoffset;
offset->offset += cmpdsize;
done:
return stat;
}
#endif /*USE_NETCDF4*/
/**************************************************/
/*
@ -232,18 +270,13 @@ This code is a variant of the H5detect.c code from HDF5.
Author: D. Heimbigner 10/7/2008
*/
int
EXTERNL int
ncaux_begin_compound(int ncid, const char *name, int alignmode, void** tagp)
{
#ifdef USE_NETCDF4
int status = NC_NOERR;
struct NCAUX_CMPD* cmpd = NULL;
if(!ncaux_initialized) {
compute_alignments();
ncaux_initialized = 1;
}
if(tagp) *tagp = NULL;
cmpd = (struct NCAUX_CMPD*)calloc(1,sizeof(struct NCAUX_CMPD));
@ -269,7 +302,7 @@ fail:
#endif
}
int
EXTERNL int
ncaux_abort_compound(void* tag)
{
#ifdef USE_NETCDF4
@ -291,7 +324,7 @@ done:
#endif
}
int
EXTERNL int
ncaux_add_field(void* tag, const char *name, nc_type field_type,
int ndims, const int* dimsizes)
{
@ -331,7 +364,7 @@ done:
#endif
}
int
EXTERNL int
ncaux_end_compound(void* tag, nc_type* idp)
{
#ifdef USE_NETCDF4
@ -369,164 +402,55 @@ done:
}
/**************************************************/
#ifdef USE_NETCDF4
static size_t
dimproduct(size_t ndims, int* dimsizes)
{
int i;
size_t product = 1;
for(i=0;i<ndims;i++) product *= (size_t)dimsizes[i];
return product;
}
/*
The heart of this is the following macro,
which computes the offset of a field x
when preceded by a char field.
The assumptions appear to be as follows:
1. the offset produced in this situation indicates
the alignment for x relative in such a way that it
depends only on the types that precede it in the struct.
2. the compiler does not reorder fields.
3. arrays are tightly packed.
4. nested structs are alignd according to their first member
(this actually follows from C language requirement that
a struct can legally be cast to an instance of its first member).
Given the alignments for the various common primitive types,
it is assumed that one can use them anywhere to construct
the layout of a struct of such types.
It seems to work for HDF5 for a wide variety of machines.
/**
@param ncclass - type class for which alignment is requested; excludes ENUM|COMPOUND
*/
#define COMP_ALIGNMENT(DST,TYPE) {\
struct {char f1; TYPE x;} tmp; \
DST.typename = #TYPE ; \
DST.alignment = (size_t)((char*)(&(tmp.x)) - (char*)(&tmp));}
/* Define indices for every primitive C type */
/* NAT => NOT-A-TYPE*/
#define NATINDEX 0
#define CHARINDEX 1
#define UCHARINDEX 2
#define SHORTINDEX 3
#define USHORTINDEX 4
#define INTINDEX 5
#define UINTINDEX 6
#define LONGINDEX 7
#define ULONGINDEX 8
#define LONGLONGINDEX 9
#define ULONGLONGINDEX 10
#define FLOATINDEX 11
#define DOUBLEINDEX 12
#define PTRINDEX 13
#define NCVLENINDEX 14
#define NCTYPES 15
typedef struct Alignment {
char* typename;
size_t alignment;
} Alignment;
typedef Alignment Typealignvec;
/* Capture in struct and in a vector*/
typedef struct Typealignset {
Alignment charalign; /* char*/
Alignment ucharalign; /* unsigned char*/
Alignment shortalign; /* short*/
Alignment ushortalign; /* unsigned short*/
Alignment intalign; /* int*/
Alignment uintalign; /* unsigned int*/
Alignment longalign; /* long*/
Alignment ulongalign; /* unsigned long*/
Alignment longlongalign; /* long long*/
Alignment ulonglongalign; /* unsigned long long*/
Alignment floatalign; /* float*/
Alignment doublealign; /* double*/
Alignment ptralign; /* void**/
Alignment ncvlenalign; /* nc_vlen_t*/
} Typealignset;
static Typealignvec vec[NCTYPES];
static Typealignset set;
static void
compute_alignments(void)
EXTERNL size_t
ncaux_class_alignment(int ncclass)
{
/* Compute the alignments for all the common C data types*/
/* First for the struct*/
/* initialize*/
memset((void*)&set,0,sizeof(set));
memset((void*)vec,0,sizeof(vec));
COMP_ALIGNMENT(set.charalign,char);
COMP_ALIGNMENT(set.ucharalign,unsigned char);
COMP_ALIGNMENT(set.shortalign,short);
COMP_ALIGNMENT(set.ushortalign,unsigned short);
COMP_ALIGNMENT(set.intalign,int);
COMP_ALIGNMENT(set.uintalign,unsigned int);
COMP_ALIGNMENT(set.longalign,long);
COMP_ALIGNMENT(set.ulongalign,unsigned long);
COMP_ALIGNMENT(set.longlongalign,long long);
COMP_ALIGNMENT(set.ulonglongalign,unsigned long long);
COMP_ALIGNMENT(set.floatalign,float);
COMP_ALIGNMENT(set.doublealign,double);
COMP_ALIGNMENT(set.ptralign,void*);
COMP_ALIGNMENT(set.ncvlenalign,nc_vlen_t);
/* Then the vector*/
COMP_ALIGNMENT(vec[CHARINDEX],char);
COMP_ALIGNMENT(vec[UCHARINDEX],unsigned char);
COMP_ALIGNMENT(vec[SHORTINDEX],short);
COMP_ALIGNMENT(vec[USHORTINDEX],unsigned short);
COMP_ALIGNMENT(vec[INTINDEX],int);
COMP_ALIGNMENT(vec[UINTINDEX],unsigned int);
COMP_ALIGNMENT(vec[LONGINDEX],long);
COMP_ALIGNMENT(vec[ULONGINDEX],unsigned long);
COMP_ALIGNMENT(vec[LONGLONGINDEX],long long);
COMP_ALIGNMENT(vec[ULONGLONGINDEX],unsigned long long);
COMP_ALIGNMENT(vec[FLOATINDEX],float);
COMP_ALIGNMENT(vec[DOUBLEINDEX],double);
COMP_ALIGNMENT(vec[PTRINDEX],void*);
COMP_ALIGNMENT(vec[NCVLENINDEX],nc_vlen_t);
if(ncclass <= NC_MAX_ATOMIC_TYPE || ncclass == NC_VLEN || ncclass == NC_OPAQUE)
return NC_class_alignment(ncclass);
nclog(NCLOGERR,"ncaux_class_alignment: class %d; alignment cannot be determermined",ncclass);
return 0;
}
static size_t
nctypealignment(nc_type nctype)
/**
@param ncid - only needed for a compound type
@param xtype - type for which alignment is requested
*/
EXTERNL size_t
ncaux_type_alignment(int xtype, int ncid)
{
Alignment* align = NULL;
int index = 0;
switch (nctype) {
case NC_BYTE: index = UCHARINDEX; break;
case NC_CHAR: index = CHARINDEX; break;
case NC_SHORT: index = SHORTINDEX; break;
case NC_INT: index = INTINDEX; break;
case NC_FLOAT: index = FLOATINDEX; break;
case NC_DOUBLE: index = DOUBLEINDEX; break;
case NC_UBYTE: index = UCHARINDEX; break;
case NC_USHORT: index = USHORTINDEX; break;
case NC_UINT: index = UINTINDEX; break;
case NC_INT64: index = LONGLONGINDEX; break;
case NC_UINT64: index = ULONGLONGINDEX; break;
case NC_STRING: index = PTRINDEX; break;
case NC_VLEN: index = NCVLENINDEX; break;
case NC_OPAQUE: index = UCHARINDEX; break;
default: assert(0);
if(!NC_alignments_computed) {
NC_compute_alignments();
ncaux_initialized = 1;
}
align = &vec[index];
return align->alignment;
}
static size_t
getpadding(size_t offset, size_t alignment)
{
size_t rem = (alignment==0?0:(offset % alignment));
size_t pad = (rem==0?0:(alignment - rem));
return pad;
if(xtype <= NC_MAX_ATOMIC_TYPE)
return NC_class_alignment(xtype); /* type == class */
#ifdef USE_NETCDF4
else {/* Presumably a user type */
int klass = NC_NAT;
int stat = nc_inq_user_type(ncid,xtype,NULL,NULL,NULL,NULL,&klass);
if(stat) goto done;
switch(klass) {
case NC_VLEN: return NC_class_alignment(klass);
case NC_OPAQUE: return NC_class_alignment(klass);
case NC_COMPOUND: {/* get alignment of the first field of the compound */
int fieldtype = NC_NAT;
if((stat=nc_inq_compound_fieldtype(ncid,xtype,0,&fieldtype))) goto done;
return ncaux_type_alignment(fieldtype,ncid); /* may recurse repeatedly */
} break;
default: break;
}
}
done:
#endif /*USE_NETCDF4 */
return 0; /* fail */
}
#ifdef USE_NETCDF4
/* Find first primitive field of a possibly nested sequence of compounds */
static nc_type
findfirstfield(int ncid, nc_type xtype)
@ -543,6 +467,23 @@ done:
return (status == NC_NOERR?fieldtype:NC_NAT);
}
static size_t
getpadding(size_t offset, size_t alignment)
{
size_t rem = (alignment==0?0:(offset % alignment));
size_t pad = (rem==0?0:(alignment - rem));
return pad;
}
static size_t
dimproduct(size_t ndims, int* dimsizes)
{
int i;
size_t product = 1;
for(i=0;i<ndims;i++) product *= (size_t)dimsizes[i];
return product;
}
static int
computefieldinfo(struct NCAUX_CMPD* cmpd)
{
@ -571,15 +512,16 @@ computefieldinfo(struct NCAUX_CMPD* cmpd)
field->alignment = 1;
break;
case NC_ENUM:
field->alignment = nctypealignment(firsttype);
field->alignment = ncaux_type_alignment(firsttype,cmpd->ncid);
break;
case NC_VLEN: /*fall thru*/
case NC_COMPOUND:
field->alignment = nctypealignment(firsttype);
field->alignment = ncaux_type_alignment(firsttype,cmpd->ncid);
break;
default:
field->alignment = nctypealignment(field->fieldtype);
field->alignment = ncaux_type_alignment(field->fieldtype,cmpd->ncid);
break;
}
offset += getpadding(offset,alignment);
field->offset = offset;
@ -591,7 +533,7 @@ computefieldinfo(struct NCAUX_CMPD* cmpd)
done:
return status;
}
#endif /*USE_NETCDF4*/

4
libdispatch/dfile.c
View File

@ -279,9 +279,9 @@ done:
*
* @param path File name.
* @param flags
* @param use_parallel
* @param parameters
* @param model Pointer that gets the model to use for the dispatch
* table.
* @param model Pointer that gets the model to use for the dispatch table.
* @param version Pointer that gets version of the file.
*
* @return ::NC_NOERR No error.

100
libdispatch/doffsets.c
View File

@ -29,6 +29,7 @@ Author: D. Heimbigner 10/7/2008
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "nclog.h"
#ifdef OFFSETTEST
@ -102,49 +103,51 @@ char* ctypenames[NCTYPES] = {
};
#endif
static Typealignvec vec[NCTYPES];
static Typealignset set;
static int computed = 0;
static NCtypealignvec vec[NC_NCTYPES];
static NCtypealignset set;
int NC_alignments_computed = 0;
/*Forward*/
static void compute_alignments(void);
unsigned int
nctypealignment(nc_type nctype)
/* Argument is a netcdf type class, except compound|ENUM */
size_t
NC_class_alignment(int ncclass)
{
Alignment* align = NULL;
NCalignment* align = NULL;
int index = 0;
if(!computed) {
compute_alignments();
computed = 1;
if(!NC_alignments_computed) {
NC_compute_alignments();
NC_alignments_computed = 1;
}
switch (nctype) {
case NC_BYTE: index = UCHARINDEX; break;
case NC_CHAR: index = CHARINDEX; break;
case NC_SHORT: index = SHORTINDEX; break;
case NC_INT: index = INTINDEX; break;
case NC_FLOAT: index = FLOATINDEX; break;
case NC_DOUBLE: index = DOUBLEINDEX; break;
case NC_UBYTE: index = UCHARINDEX; break;
case NC_USHORT: index = USHORTINDEX; break;
case NC_UINT: index = UINTINDEX; break;
case NC_INT64: index = LONGLONGINDEX; break;
case NC_UINT64: index = ULONGLONGINDEX; break;
case NC_STRING: index = PTRINDEX; break;
case NC_VLEN: index = NCVLENINDEX; break;
case NC_OPAQUE: index = UCHARINDEX; break;
switch (ncclass) {
case NC_BYTE: index = NC_UCHARINDEX; break;
case NC_CHAR: index = NC_CHARINDEX; break;
case NC_SHORT: index = NC_SHORTINDEX; break;
case NC_INT: index = NC_INTINDEX; break;
case NC_FLOAT: index = NC_FLOATINDEX; break;
case NC_DOUBLE: index = NC_DOUBLEINDEX; break;
case NC_UBYTE: index = NC_UCHARINDEX; break;
case NC_USHORT: index = NC_USHORTINDEX; break;
case NC_UINT: index = NC_UINTINDEX; break;
case NC_INT64: index = NC_LONGLONGINDEX; break;
case NC_UINT64: index = NC_ULONGLONGINDEX; break;
case NC_STRING: index = NC_PTRINDEX; break;
/* Here class matters */
case NC_VLEN: index = NC_NCVLENINDEX; break;
case NC_OPAQUE: index = NC_UCHARINDEX; break;
case NC_ENUM: /* fall thru */
case NC_COMPOUND: /* fall thru */
default:
fprintf(stderr,"nctypealignment: bad type code: %d",nctype);
exit(1);
nclog(NCLOGERR,"nc_class_alignment: class code %d cannot be aligned",ncclass);
return 0;
}
align = &vec[index];
return align->alignment;
}
static void
compute_alignments(void)
void
NC_compute_alignments(void)
{
if(NC_alignments_computed) return;
/* Compute the alignments for all the common C data types*/
/* First for the struct*/
/* initialize*/
@ -165,18 +168,19 @@ compute_alignments(void)
COMP_ALIGNMENT(set.ncvlenalign,nc_vlen_t);
/* Then the vector*/
COMP_ALIGNMENT(vec[CHARINDEX],char);
COMP_ALIGNMENT(vec[UCHARINDEX],unsigned char);
COMP_ALIGNMENT(vec[SHORTINDEX],short);
COMP_ALIGNMENT(vec[USHORTINDEX],unsigned short);
COMP_ALIGNMENT(vec[INTINDEX],int);
COMP_ALIGNMENT(vec[UINTINDEX],unsigned int);
COMP_ALIGNMENT(vec[LONGLONGINDEX],long long);
COMP_ALIGNMENT(vec[ULONGLONGINDEX],unsigned long long);
COMP_ALIGNMENT(vec[FLOATINDEX],float);
COMP_ALIGNMENT(vec[DOUBLEINDEX],double);
COMP_ALIGNMENT(vec[PTRINDEX],void*);
COMP_ALIGNMENT(vec[NCVLENINDEX],nc_vlen_t);
COMP_ALIGNMENT(vec[NC_CHARINDEX],char);
COMP_ALIGNMENT(vec[NC_UCHARINDEX],unsigned char);
COMP_ALIGNMENT(vec[NC_SHORTINDEX],short);
COMP_ALIGNMENT(vec[NC_USHORTINDEX],unsigned short);
COMP_ALIGNMENT(vec[NC_INTINDEX],int);
COMP_ALIGNMENT(vec[NC_UINTINDEX],unsigned int);
COMP_ALIGNMENT(vec[NC_LONGLONGINDEX],long long);
COMP_ALIGNMENT(vec[NC_ULONGLONGINDEX],unsigned long long);
COMP_ALIGNMENT(vec[NC_FLOATINDEX],float);
COMP_ALIGNMENT(vec[NC_DOUBLEINDEX],double);
COMP_ALIGNMENT(vec[NC_PTRINDEX],void*);
COMP_ALIGNMENT(vec[NC_NCVLENINDEX],nc_vlen_t);
NC_alignments_computed = 1;
}
#ifdef OFFSETTEST
@ -219,17 +223,17 @@ padname(char* name)
}
static void
verify(Typealignvec* vec)
verify(NCtypealignvec* vec)
{
int i,j;
Typealignvec* vec16;
Typealignvec* vec32;
NCtypealignvec* vec16;
NCtypealignvec* vec32;
int* sizes8;
int* sizes16;
int* sizes32;
vec16 = (Typealignvec*)emalloc(sizeof(Typealignvec)*NCTYPES);
vec32 = (Typealignvec*)emalloc(sizeof(Typealignvec)*NCTYPES);
vec16 = (NCtypealignvec*)emalloc(sizeof(NCtypealignvec)*NCTYPES);
vec32 = (NCtypealignvec*)emalloc(sizeof(NCtypealignvec)*NCTYPES);
sizes8 = (int*)emalloc(sizeof(int)*NCTYPES);
sizes16 = (int*)emalloc(sizeof(int)*NCTYPES);
sizes32 = (int*)emalloc(sizeof(int)*NCTYPES);

20
libdispatch/dvlen.c
View File

@ -23,6 +23,16 @@ will free the vlen memory.
The function nc_free_vlens() is more useful than this function,
because it can free an array of VLEN objects.
WARNING: this code is incorrect because it will only
work if the basetype of the vlen is
- atomic
- + enum
- + opaque
- excluding string basetype,
The reason is that to operate properly, it needs to recurse when
the basetype is a complex object such as another vlen or compound.
\param vl pointer to the vlen object.
\returns ::NC_NOERR No error.
@ -43,6 +53,16 @@ space for the data. This storage space must be freed, so pass the
pointer back to this function, when you're done with the data, and it
will free the vlen memory.
WARNING: this code is incorrect because it will only
work if the basetype of the vlen is
- atomic
- + enum
- + opaque
- excluding string basetype,
The reason is that to operate properly, it needs to recurse when
the basetype is a complex object such as another vlen or compound.
\param len number of elements in the array.
\param vlens pointer to the vlen object.

20
libhdf5/hdf5attr.c
View File

@ -475,7 +475,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
if (att->vldata)
{
for (i = 0; i < att->len; i++)
nc_free_vlen(&att->vldata[i]);
nc_free_vlen(&att->vldata[i]); /* FIX: see warning of nc_free_vlen */
free(att->vldata);
att->vldata = NULL;
}
@ -539,11 +539,18 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
if (var->type_info->nc_type_class == NC_VLEN)
{
nc_vlen_t *in_vlen = (nc_vlen_t *)data, *fv_vlen = (nc_vlen_t *)(var->fill_value);
NC_TYPE_INFO_T* basetype;
size_t basetypesize = 0;
/* get the basetype and its size */
basetype = var->type_info;
if ((retval = nc4_get_typelen_mem(grp->nc4_info, basetype->hdr.id, &basetypesize)))
return retval;
/* shallow clone the content of the vlen; shallow because it has only a temporary existence */
fv_vlen->len = in_vlen->len;
if (!(fv_vlen->p = malloc(size * in_vlen->len)))
if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
return NC_ENOMEM;
memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * size);
memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
}
else if (var->type_info->nc_type_class == NC_STRING)
{
@ -579,15 +586,15 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
if (type_class == NC_VLEN)
{
const hvl_t *vldata1;
NC_TYPE_INFO_T *type;
NC_TYPE_INFO_T *vltype;
size_t base_typelen;
/* Get the type object for the attribute's type */
if ((retval = nc4_find_type(h5, file_type, &type)))
if ((retval = nc4_find_type(h5, file_type, &vltype)))
BAIL(retval);
/* Retrieve the size of the base type */
if ((retval = nc4_get_typelen_mem(h5, type->u.v.base_nc_typeid, &base_typelen)))
if ((retval = nc4_get_typelen_mem(h5, vltype->u.v.base_nc_typeid, &base_typelen)))
BAIL(retval);
vldata1 = data;
@ -596,6 +603,7 @@ nc4_put_att(NC_GRP_INFO_T* grp, int varid, const char *name, nc_type file_type,
for (i = 0; i < att->len; i++)
{
att->vldata[i].len = vldata1[i].len;
/* Warning, this only works for cases described for nc_free_vlen() */
if (!(att->vldata[i].p = malloc(base_typelen * att->vldata[i].len)))
BAIL(NC_ENOMEM);
memcpy(att->vldata[i].p, vldata1[i].p, base_typelen * att->vldata[i].len);

8
libhdf5/nc4hdf.c
View File

@ -315,15 +315,19 @@ nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
if (var->type_info->nc_type_class == NC_VLEN)
{
nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
size_t basetypesize = 0;
if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
return retval;
fv_vlen->len = in_vlen->len;
if (!(fv_vlen->p = malloc(size * in_vlen->len)))
if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
{
free(*fillp);
*fillp = NULL;
return NC_ENOMEM;
}
memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * size);
memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
}
else if (var->type_info->nc_type_class == NC_STRING)
{

2
libhdf5/nc4info.c
View File

@ -493,7 +493,7 @@ int
NC4_free_provenance(struct NCPROVENANCE* prov)
{
if(prov == NULL) return NC_NOERR;
if(prov->propattr.properties != NULL);
if(prov->propattr.properties != NULL)
nclistfreeall(prov->propattr.properties);
prov->propattr.properties = NULL;
free(prov);

7
nc_test/run_diskless2.sh
View File

@ -14,8 +14,9 @@ OS=`uname`
# Test diskless on a reasonably large file size
# Try a large in-memory file
SIZE=1000000000
# Try a large in-memory file; two instances at once may be needed when doing realloc
#SIZE=1000000000
SIZE=500000000
FILE4=tst_diskless4.nc
@ -27,7 +28,7 @@ rm -fr ref_tst_diskless4.cdl
cat >ref_tst_diskless4.cdl <<EOF
netcdf tst_diskless4 {
dimensions:
dim = 1000000000 ;
dim = 500000000 ;
variables:
byte var0(dim) ;
}

2
nc_test/tst_diskless4.c
View File

@ -20,7 +20,7 @@
#define FILE_NAME "tst_diskless4.nc"
#define CHUNKSIZE 4096
#define DATASIZE (CHUNKSIZE/sizeof(int))
#define DIMMAX 1000000000L
#define DIMMAX 500000000L
typedef enum Tag {Create,CreateDiskless,Open,OpenDiskless} Tag;

4
nc_test4/Makefile.am
View File

@ -11,8 +11,8 @@
include $(top_srcdir)/lib_flags.am
# Un comment to use a more verbose test driver
SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
TEST_EXTENSIONS = .sh
# Note which tests depend on other tests. necessary for make -j check

3
nc_test4/tst_udf.c
View File

@ -25,8 +25,7 @@ NC4_show_metadata(int ncid)
int
tst_open(const char *path, int mode, int basepe, size_t *chunksizehintp,
int use_parallel, void *parameters, NC_Dispatch *dispatch,
NC *nc_file)
void *parameters, NC_Dispatch *dispatch, NC *nc_file)
{
return NC_NOERR;
}

9
ncdap_test/testurl.sh
View File

@ -19,12 +19,19 @@
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
# Figure our dst server; if none, then just stop
DTS=`${execdir}/findtestserver dap2 dts`
if test "x$DTS" = "x" ; then
echo "WARNING: Cannot locate test server for dts"
exit 1
fi
OCLOGFILE=stderr
if test "x$DBG" = x1 ; then
SHOW=1
fi
URL="http://remotetest.unidata.ucar.edu/dts/test.03"
URL="$DTS/test.03"
PREFIX="[log][show=fetch]"
SUFFIX="log&show=fetch"

1
ncdump/CMakeLists.txt
View File

@ -184,7 +184,6 @@ ENDIF(MSVC)
build_bin_test_no_prefix(tst_compress)
build_bin_test_no_prefix(tst_chunking)
###
# This test fails on Visual Studio builds with bash.
# It passes, but technically fails because the scientific

2
ncdump/Makefile.am
View File

@ -78,6 +78,8 @@ tst_group_data tst_enum_data tst_opaque_data tst_string_data \
tst_vlen_data tst_comp tst_comp2 tst_nans tst_special_atts \
tst_unicode tst_fillbug tst_compress tst_chunking tst_h_scalar
check_PROGRAMS += tst_vlen_demo
# Tests for netCDF-4 behavior.
TESTS += tst_fileinfo.sh tst_hdf5_offset.sh tst_inttags4.sh \
tst_netcdf4.sh tst_fillbug.sh tst_netcdf4_4.sh tst_nccopy4.sh \

27
ncdump/chunkspec.c
View File

@ -86,22 +86,24 @@ dimchunkspec_parse(int igrp, const char *spec)
const char *np; /* beginning of current dimension name */
size_t ndims = 0;
int idim;
int ret;
int ret = NC_NOERR;
int comma_seen = 0;
dimchunkspecs.ndims = 0;
dimchunkspecs.omit = false;
if (!spec || *spec == '\0') /* default chunking */
return NC_NOERR;
if (spec[0] == '/' && spec[1] == '\0') { /* no chunking */
goto done;
/* Special rule: // is treated as equivalent to / */
if ((spec[0] == '/' && spec[1] == '\0')
|| (spec[0] == '/' && spec[1] == '/' && spec[2] == '\0')) { /* no chunking */
dimchunkspecs.omit = true;
return NC_NOERR;
goto done;
}
/* Count unescaped commas, handle consecutive unescaped commas as error */
for(cp = spec; *cp; cp++) {
if(*cp == ',' && *pp != '\\') {
if(comma_seen) { /* consecutive commas detected */
return(NC_EINVAL);
{ret = NC_EINVAL; goto done;}
}
comma_seen = 1;
ndims++;
@ -129,7 +131,8 @@ dimchunkspec_parse(int igrp, const char *spec)
continue;
}
if(*pp != '/') { /* no '/' found, no chunksize specified for dimension */
return(NC_EINVAL);
ret = NC_EINVAL;
goto done;
}
/* extract dimension name */
dimname = (char *) emalloc(pp - np + 1);
@ -141,7 +144,7 @@ dimchunkspec_parse(int igrp, const char *spec)
/* look up dimension id from dimension pathname */
ret = nc_inq_dimid2(igrp, dimname, &dimid);
if(ret != NC_NOERR)
break;
{if(dimname) free(dimname); goto done;}
dimchunkspecs.idimids[idim] = dimid;
/* parse and assign corresponding chunksize */
pp++; /* now points to first digit of chunksize, ',', or '\0' */
@ -149,7 +152,7 @@ dimchunkspec_parse(int igrp, const char *spec)
size_t dimlen;
ret = nc_inq_dimlen(igrp, dimid, &dimlen);
if(ret != NC_NOERR)
return(ret);
{if(dimname) free(dimname); goto done;}
chunksize = dimlen;
} else { /* convert nnn string to long long integer */
char *ep;
@ -159,12 +162,13 @@ dimchunkspec_parse(int igrp, const char *spec)
long long val = strtol(pp, &ep, 0);
#endif
if(ep == pp || errno == ERANGE || val < 1) /* allow chunksize bigger than dimlen */
return (NC_EINVAL);
{if(dimname) free(dimname); ret = NC_EINVAL; goto done;}
chunksize = (size_t)val;
}
dimchunkspecs.chunksizes[idim] = chunksize;
idim++;
free(dimname);
if(dimname) free(dimname);
dimname = NULL;
if(*cp == '\0')
break;
/* set np to point to first char after comma */
@ -172,7 +176,8 @@ dimchunkspec_parse(int igrp, const char *spec)
}
pp = cp;
};
return NC_NOERR;
done:
return ret;
}
/* Return size in chunkspec string specified for dimension corresponding to dimid, 0 if not found */

5
ncdump/ctests.sh
View File

@ -135,7 +135,6 @@ ref_tst_comp \
ref_tst_comp2 \
ref_tst_group_data \
ref_tst_opaque_data \
ref_tst_vlen_data \
ref_tst_solar_1 \
ref_tst_solar_2 \
ref_tst_enum_data \
@ -143,6 +142,10 @@ ref_tst_nans \
ref_tst_special_atts \
ref_const_test"
if test "x$NOVLEN" = x ; then
TESTS4="$TESTS4 ref_tst_vlen_data"
fi
SPECIALTESTS="ref_tst_special_atts"
XFAILTESTS="ref_tst_special_atts"

4
ncdump/dumplib.c
View File

@ -920,7 +920,7 @@ int ncstring_typ_tostring(const nctype_t *typ, safebuf_t *sfbf, const void *valp
int
ncenum_typ_tostring(const nctype_t *typ, safebuf_t *sfbf, const void *valp) {
char symbol[NC_MAX_NAME + 1];
long long val;
long long val = 0;
switch (typ->base_tid) {
case NC_BYTE:
@ -1215,7 +1215,7 @@ ncdouble_val_tostring(const ncvar_t *varp, safebuf_t *sfbf, const void *valp) {
* lose precision for values of type NC_INT64 or NC_UINT64 */
static
double to_double(const ncvar_t *varp, const void *valp) {
double dd;
double dd = 0.0;
switch (varp->type) {
case NC_BYTE:
dd = *(signed char *)valp;

21
ncdump/ncdump.c
View File

@ -48,6 +48,7 @@ int optind;
#include "cdl.h"
#include "nclog.h"
#include "ncwinpath.h"
#include "netcdf_aux.h"
#ifdef USE_NETCDF4
#include "nc4internal.h" /* to get name of the special properties file */
@ -836,7 +837,7 @@ pr_att(
size_t type_size, nfields;
nc_type base_nc_type;
int class, i;
void *data;
void *data = NULL;
NC_CHECK( nc_inq_user_type(ncid, att.type, type_name, &type_size,
&base_nc_type, &nfields, &class));
@ -1494,9 +1495,13 @@ get_fill_info(int ncid, int varid, ncvar_t *vp)
case NC_STRING: {
char* s;
size_t len = strlen(NC_FILL_STRING);
#if 0
/* In order to avoid mem leak, allocate this string as part of fillvalp */
fillvalp = erealloc(fillvalp, vp->tinfo->size + 1 + len + 1);
s = ((char*)fillvalp) + vp->tinfo->size + 1;
#else
s = malloc(len+1);
#endif
memcpy(s,NC_FILL_STRING,len);
s[len] = '\0';
*((char **)fillvalp) = s;
@ -1689,10 +1694,10 @@ do_ncdump_rec(int ncid, const char *path)
* to know how to print strings with embedded newlines. */
NC_CHECK( nc_inq_format(ncid, &kind) );
/* For each var, get and print out info. */
memset((void*)&var,0,sizeof(var));
/* For each var, get and print out info. */
for (varid = 0; varid < nvars; varid++) {
NC_CHECK( nc_inq_varndims(ncid, varid, &var.ndims) );
if(var.dims != NULL) free(var.dims);
@ -1848,7 +1853,6 @@ do_ncdump_rec(int ncid, const char *path)
vdims = 0;
continue;
}
if(var.fillvalp != NULL) free(var.fillvalp);
get_fill_info(ncid, varid, &var); /* sets has_fillval, fillvalp mmbrs */
if(var.timeinfo != NULL) {
if(var.timeinfo->units) free(var.timeinfo->units);
@ -1863,6 +1867,8 @@ do_ncdump_rec(int ncid, const char *path)
error("can't output data for variable %s", var.name);
goto done;
}
if(var.fillvalp != NULL)
{ncaux_reclaim_data(ncid,var.tinfo->tid,var.fillvalp,1); free(var.fillvalp); var.fillvalp = NULL;}
}
if (vdims) {
free(vdims);
@ -1913,7 +1919,12 @@ do_ncdump_rec(int ncid, const char *path)
done:
if(var.dims != NULL) free(var.dims);
if(var.fillvalp != NULL) free(var.fillvalp);
if(var.fillvalp != NULL) {
/* Release any data hanging off of fillvalp */
ncaux_reclaim_data(ncid,var.tinfo->tid,var.fillvalp,1);
free(var.fillvalp);
var.fillvalp = NULL;
}
if(var.timeinfo != NULL) {
if(var.timeinfo->units) free(var.timeinfo->units);
free(var.timeinfo);

2
ncdump/run_ncgen_nc4_tests.sh
View File

@ -46,9 +46,11 @@ ${NCGEN} -lc $top_srcdir/ncgen/ref_camrun.cdl > camrun.c
echo "*** test for jira NCF-199 bug"
validateNC "ncf199" "ncf199" -k nc4
if test "x$NC_VLEN_NOTEST" = x ; then
echo "*** creating binary files for github issue 323..."
echo "*** github issue 323 test 1"
validateNC "compound_datasize_test" "compound_datasize_test" -k nc4
fi
echo "*** github issue 323 test 2"
validateNC "compound_datasize_test2" "compound_datasize_test2" -k nc4

7
ncdump/tst_nccopy4.sh
View File

@ -16,8 +16,11 @@ echo ""
# ref_tst_compounds2 ref_tst_compounds3 ref_tst_compounds4
TESTFILES='tst_comp tst_comp2 tst_enum_data tst_fillbug
tst_group_data tst_nans tst_opaque_data tst_solar_1 tst_solar_2
tst_solar_cmp tst_special_atts tst_string_data tst_unicode
tst_vlen_data'
tst_solar_cmp tst_special_atts tst_string_data tst_unicode'
if test "x$NC_VLEN_NOTEST" = x ; then
TESFILES="$TESTFILES tst_vlen_data"
fi
echo "*** Testing netCDF-4 features of nccopy on ncdump/*.nc files"
for i in $TESTFILES ; do

7
ncdump/tst_ncgen_shared.sh
View File

@ -94,14 +94,15 @@ ref_tst_special_atts \
ref_tst_nans \
ref_solar \
unlimtest2 \
ref_tst_vlen_data \
ref_tst_vlen_data \
ref_tst_vlen_data2 \
ref_niltest \
ref_tst_h_scalar \
ref_tst_nul4 \
"
if test "x$NC_VLEN_NOTEST" = x ; then
TESTS4="$TESTS4 ref_tst_vlen_data ref_tst_vlen_data2"
fi
SPECIALTESTS3="ref_tst_special_atts3"
SPECIALTESTS="${SPECIALTESTS3} ref_tst_special_atts"

2
ncdump/tst_netcdf4.sh
View File

@ -71,10 +71,12 @@ ${execdir}/tst_opaque_data ; ERR
${NCDUMP} tst_opaque_data.nc | sed 's/e+0/e+/g' > tst_opaque_data.cdl ; ERR
diff -b tst_opaque_data.cdl $srcdir/ref_tst_opaque_data.cdl ; ERR
if test "x$NC_VLEN_NOTEST" = x ; then
echo "*** Running tst_vlen_data.c to create test files."
${execdir}/tst_vlen_data ; ERR
${NCDUMP} tst_vlen_data.nc | sed 's/e+0/e+/g' > tst_vlen_data.cdl ; ERR
diff -b tst_vlen_data.cdl $srcdir/ref_tst_vlen_data.cdl ; ERR
fi
echo "*** Running tst_comp.c to create test files."
${execdir}/tst_comp ; ERR

26
ncdump/tst_vlen_demo.c Normal file
View File

@ -0,0 +1,26 @@
#include <stdlib.h>
#include "netcdf.h"
int
main(void)
{
int ncid, typeid, varid;
float missing_value = -999.0;
nc_vlen_t missing_val;
if(nc_create("tst_vlen_data.nc", NC_CLOBBER | NC_NETCDF4, &ncid)) abort();
/* Create a vlen type. */
if (nc_def_vlen(ncid, "row_of_floats", NC_FLOAT, &typeid)) abort();;
/* Declare a scalar variable of the vlen type */
if (nc_def_var(ncid, "ragged_array", typeid, 0, NULL, &varid)) abort();;
/* Create and write a variable attribute of the vlen type */
missing_val.p = &missing_value;
missing_val.len = 1;
if (nc_put_att(ncid, varid, "_FillValue", typeid, 1, (void *) &missing_val)) abort();;
if (nc_close(ncid)) abort();;
exit(0);
}

12
ncdump/vardata.c
View File

@ -18,6 +18,7 @@
#include "ncdump.h"
#include "indent.h"
#include "vardata.h"
#include "netcdf_aux.h"
/* maximum len of string needed for one value of a primitive type */
#define MAX_OUTPUT_LEN 100
@ -493,14 +494,9 @@ print_rows(
}
print_any_val(sb, vp, (void *)valp);
}
/* If this is an array of strings, then reclaim the space */
if(vp->type == NC_STRING) {
int i; /* compute number of elements specified by cor and edg */
size_t total = 1;
for(i=0;i<rank;i++)
total *= edg[i];
nc_free_string(total,(char**)vals);
}
/* In case vals has memory hanging off e.g. vlen or string, make sure to reclaim it */
(void)ncaux_reclaim_data(ncid,vp->type,vals,ncols);
/* determine if this is the last row */
lastrow = true;
for(j = 0; j < rank - 1; j++) {

419
ncgen/bindata.c
View File

@ -5,10 +5,12 @@
#include "includes.h"
#include "nc_iter.h"
#include "nclog.h"
#ifdef ENABLE_BINARY
static void alignto(int alignment, Bytebuffer* buf, int base);
/* Forward */
static void alignto(int alignment, Bytebuffer* buf, ptrdiff_t base);
static int bin_uid = 0;
@ -18,7 +20,7 @@ bin_charconstant(Generator* generator, Symbol* sym, Bytebuffer* buf, ...)
/* Just transfer charbuf to codebuf */
Bytebuffer* charbuf;
va_list ap;
vastart(ap,buf);
va_start(ap,buf);
charbuf = va_arg(ap, Bytebuffer*);
va_end(ap);
bbNull(charbuf);
@ -39,7 +41,7 @@ bin_constant(Generator* generator, Symbol* sym, NCConstant* con, Bytebuffer* buf
/* Assume the opaque string has been normalized */
bytes=makebytestring(con->value.opaquev.stringv,&len);
bbAppendn(buf,(void*)bytes,len);
free(bytes);
efree(bytes);
} break;
case NC_CHAR:
bbAppendn(buf,&con->value.charv,sizeof(con->value.charv));
@ -80,16 +82,16 @@ bin_constant(Generator* generator, Symbol* sym, NCConstant* con, Bytebuffer* buf
} break;
case NC_NIL:
case NC_STRING: {
char* ptr;
int len = (size_t)con->value.stringv.len;
if(len == 0 && con->value.stringv.stringv == NULL) {
char* nil = NULL;
bbAppendn(buf,(void*)&nil,sizeof(nil));
} else {
ptr = (char*)ecalloc(len+1);
char* ptr = (char*)ecalloc(len+1);
memcpy(ptr,con->value.stringv.stringv,len);
ptr[len] = '\0';
bbAppendn(buf,(void*)&ptr,sizeof(ptr));
ptr = NULL;
}
} break;
@ -136,11 +138,11 @@ bin_vlendecl(Generator* generator, Symbol* tsym, Bytebuffer* buf, int uid, size_
va_list ap;
Bytebuffer* vlenmem;
nc_vlen_t ptr;
vastart(ap,count);
va_start(ap,count);
vlenmem = va_arg(ap, Bytebuffer*);
va_end(ap);
ptr.len = count;
ptr.p = bbDup(vlenmem);
ptr.p = bbExtract(vlenmem);
bbAppendn(buf,(char*)&ptr,sizeof(ptr));
return 1;
}
@ -152,7 +154,7 @@ bin_vlenstring(Generator* generator, Symbol* sym, Bytebuffer* codebuf, int* uidp
nc_vlen_t ptr;
va_list ap;
if(uidp) *uidp = ++bin_uid;
vastart(ap,sizep);
va_start(ap,sizep);
vlenmem = va_arg(ap, Bytebuffer*);
va_end(ap);
ptr.len = bbLength(vlenmem);
@ -165,10 +167,10 @@ static const char zeros[] =
"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
static void
alignto(int alignment, Bytebuffer* buf, int base)
alignto(int alignment, Bytebuffer* buf, ptrdiff_t base)
{
int pad = 0;
int offset = bbLength(buf);
ptrdiff_t offset = bbLength(buf);
offset -= base; /* Need to actually align wrt to the base */
pad = getpadding(offset,alignment);
if(pad > 0) {
@ -189,5 +191,402 @@ static Generator bin_generator_singleton = {
};
Generator* bin_generator = &bin_generator_singleton;
/**************************************************/
static int bin_generate_data_r(NCConstant* instance, Symbol* tsym, Datalist* fillvalue, Bytebuffer* databuf);
static void
write_alignment(int alignment, Bytebuffer* buf)
{
int pad = 0;
ptrdiff_t offset = bbLength(buf);
pad = getpadding(offset,alignment);
if(pad > 0) {
bbAppendn(buf,(void*)zeros,pad);
}
}
/**
Alternate binary data generator.
Inputs:
Datalist* data - to use to generate the binary data
Symbol* tsym - the top-level type for which instances
are to be generated
Datalist* fillvalue - the fillvalue for the toplevel type
Bytebuffer* databuf - the buffer into which instances are to be stored
*/
int
binary_generate_data(Datalist* data, Symbol* tsym, Datalist* fillvalue, Bytebuffer* databuf)
{
int stat = NC_NOERR;
size_t count = data->length;
size_t i;
bbClear(databuf);
for(i=0;i<count;i++) {
NCConstant* instance = datalistith(data,i);
if((stat = bin_generate_data_r(instance, tsym, fillvalue, databuf))) goto done;
}
done:
return stat;
}
/* Recursive helper that does the bulk of the work */
static int
bin_generate_data_r(NCConstant* instance, Symbol* tsym, Datalist* fillvalue, Bytebuffer* databuf)
{
int stat = NC_NOERR;
if(instance->nctype == NC_FILLVALUE) {
/* replace with fillvalue for the type */
Datalist* filllist = (fillvalue == NULL ? getfiller(tsym) : fillvalue);
ASSERT(datalistlen(filllist)==1)
instance = datalistith(filllist,0);
}
switch (tsym->subclass) {
case NC_PRIM: {
switch (tsym->nc_id) {
case NC_CHAR: {
char* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_CHAR;
convert1(instance,tmp);
p = &tmp->value.charv;;
bbAppendn(databuf,p,sizeof(char));
reclaimconstant(tmp);
} break;
case NC_BYTE: {
signed char* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_BYTE;
convert1(instance,tmp);
p = &tmp->value.int8v;
bbAppendn(databuf,p,sizeof(signed char));
reclaimconstant(tmp);
} break;
case NC_UBYTE: {
unsigned char* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_UBYTE;
convert1(instance,tmp);
p = &tmp->value.uint8v;
bbAppendn(databuf,p,sizeof(unsigned char));
reclaimconstant(tmp);
} break;
case NC_SHORT: {
short* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_SHORT;
convert1(instance,tmp);
p = &tmp->value.int16v;
bbAppendn(databuf,p,sizeof(short));
reclaimconstant(tmp);
} break;
case NC_USHORT: {
unsigned short* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_USHORT;
convert1(instance,tmp);
p = &tmp->value.uint16v;
bbAppendn(databuf,p,sizeof(unsigned short));
reclaimconstant(tmp);
} break;
case NC_INT: {
int* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_INT;
convert1(instance,tmp);
p = &tmp->value.int32v;
bbAppendn(databuf,p,sizeof(int));
reclaimconstant(tmp);
} break;
case NC_UINT: {
unsigned int* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_UINT;
convert1(instance,tmp);
p = &tmp->value.uint32v;
bbAppendn(databuf,p,sizeof(unsigned int));
reclaimconstant(tmp);
} break;
case NC_INT64: {
long long* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_INT64;
convert1(instance,tmp);
p = &tmp->value.int64v;
bbAppendn(databuf,p,sizeof(long long));
reclaimconstant(tmp);
} break;
case NC_UINT64: {
unsigned long long* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_UINT64;
convert1(instance,tmp);
p = &tmp->value.uint64v;
bbAppendn(databuf,p,sizeof(unsigned long long));
reclaimconstant(tmp);
} break;
case NC_FLOAT: {
float* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_FLOAT;
convert1(instance,tmp);
p = &tmp->value.floatv;
bbAppendn(databuf,p,sizeof(float));
reclaimconstant(tmp);
} break;
case NC_DOUBLE: {
double* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_DOUBLE;
convert1(instance,tmp);
p = &tmp->value.doublev;
bbAppendn(databuf,p,sizeof(double));
reclaimconstant(tmp);
} break;
case NC_STRING: {
char* p = NULL;
NCConstant* tmp = nullconst();
tmp->nctype = NC_STRING;
convert1(instance,tmp);
p = emalloc(tmp->value.stringv.len+1);
memcpy(p,tmp->value.stringv.stringv,tmp->value.stringv.len);
p[tmp->value.stringv.len] = '\0';
bbAppendn(databuf,&p,sizeof(char*));
reclaimconstant(tmp);
} break;
default: stat = NC_EINTERNAL; goto done; /* Should never happen */
} break; /*switch*/
} break; /*NC_PRIM*/
case NC_ENUM: {
Symbol* basetype = tsym->typ.basetype;
/* Pretend */
stat = bin_generate_data_r(instance,basetype,fillvalue,databuf);
} break;
case NC_OPAQUE: {
unsigned char* bytes = NULL;
size_t len = 0;
if(instance->nctype != NC_OPAQUE)
{stat = NC_EBADTYPE; goto done;}
/* Assume the opaque string has been normalized */
bytes=makebytestring(instance->value.opaquev.stringv,&len);
if(bytes == NULL) {stat = NC_ENOMEM; goto done;}
bbAppendn(databuf,(void*)bytes,len);
free(bytes);
} break;
case NC_VLEN: {
Datalist* sublist = NULL;
Bytebuffer* vlendata = NULL;
nc_vlen_t p;
if(instance->nctype != NC_COMPOUND) {
nclog(NCLOGERR,"Translating vlen: expected sublist");
stat = NC_EBADTYPE; goto done;
}
sublist = instance->value.compoundv;
vlendata = bbNew();
if((stat = binary_generate_data(sublist,tsym->typ.basetype,NULL,vlendata))) goto done;
p.len = datalistlen(sublist);
p.p = bbContents(vlendata);
bbAppendn(databuf,(char*)&p,sizeof(nc_vlen_t));
} break;
case NC_COMPOUND: { /* The really hard one */
size_t nfields, fid, i;
Datalist* cmpd = instance->value.compoundv;
write_alignment(tsym->typ.cmpdalign,databuf);
/* Get info about each field in turn and build it*/
nfields = listlength(tsym->subnodes);
for(fid=0;fid<nfields;fid++) {
Symbol* field = listget(tsym->subnodes,fid);
NCConstant* fieldinstance = datalistith(cmpd,fid);
int ndims = field->typ.dimset.ndims;
size_t arraycount;
if(ndims == 0) {
ndims=1; /* fake the scalar case */
}
/* compute the total number of elements in the field array */
arraycount = 1;
for(i=0;i<ndims;i++) arraycount *= field->typ.dimset.dimsyms[i]->dim.declsize;
write_alignment(field->typ.alignment,databuf);
/* Write the instances */
for(i=0;i<arraycount;i++) {
if((stat = bin_generate_data_r(fieldinstance, field->typ.basetype, NULL, databuf))) goto done;
}
}
} break;
default: stat = NC_EINTERNAL; goto done; /* Should never happen */
}
done:
return stat;
}
/**
Internal equivalent of ncaux_reclaim_data.
*/
/* It is helpful to have a structure that contains memory and an offset */
typedef struct Reclaim {char* memory; ptrdiff_t offset;} Reclaim;
static ptrdiff_t read_alignment(ptrdiff_t offset, unsigned long alignment);
static int bin_reclaim_datar(Symbol* tsym, Reclaim* reclaim);
static int bin_reclaim_usertype(Symbol* tsym, Reclaim* reclaim);
static int bin_reclaim_compound(Symbol* tsym, Reclaim* reclaim);
static int bin_reclaim_vlen(Symbol* tsym, Reclaim* reclaim);
static int bin_reclaim_enum(Symbol* tsym, Reclaim* reclaim);
static int bin_reclaim_opaque(Symbol* tsym, Reclaim* reclaim);
int
binary_reclaim_data(Symbol* tsym, void* memory, size_t count)
{
int stat = NC_NOERR;
size_t i;
Reclaim reclaimer;
if(tsym == NULL
|| (memory == NULL && count > 0))
{stat = NC_EINVAL; goto done;}
if(memory == NULL || count == 0)
goto done; /* ok, do nothing */
reclaimer.offset = 0;
reclaimer.memory = memory;
for(i=0;i<count;i++) {
if((stat=bin_reclaim_datar(tsym,&reclaimer))) /* reclaim one instance */
break;
}
done:
return stat;
}
/* Recursive type walker: reclaim a single instance */
static int
bin_reclaim_datar(Symbol* tsym, Reclaim* reclaimer)
{
int stat = NC_NOERR;
switch (tsym->subclass) {
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
case NC_SHORT: case NC_USHORT:
case NC_INT: case NC_UINT: case NC_FLOAT:
case NC_INT64: case NC_UINT64: case NC_DOUBLE:
reclaimer->offset += tsym->typ.size;
break;
#ifdef USE_NETCDF4
case NC_STRING: {
char** sp = (char**)(reclaimer->memory+reclaimer->offset);
/* Need to reclaim string */
if(*sp != NULL) efree(*sp);
reclaimer->offset += tsym->typ.size;
} break;
default:
/* reclaim a user type */
stat = bin_reclaim_usertype(tsym,reclaimer);
#else
default:
stat = NC_ENOTNC4;
#endif
break;
}
return stat;
}
static int
bin_reclaim_usertype(Symbol* tsym, Reclaim* reclaimer)
{
int stat = NC_NOERR;
/* Get info about the xtype */
switch (tsym->subclass) {
case NC_OPAQUE: stat = bin_reclaim_opaque(tsym,reclaimer); break;
case NC_ENUM: stat = bin_reclaim_enum(tsym,reclaimer); break;
case NC_VLEN: stat = bin_reclaim_vlen(tsym,reclaimer); break;
case NC_COMPOUND: stat = bin_reclaim_compound(tsym,reclaimer); break;
default:
stat = NC_EINVAL;
break;
}
return stat;
}
static ptrdiff_t
read_alignment(ptrdiff_t offset, unsigned long alignment)
{
size_t delta = (offset % alignment);
if(delta == 0) return offset;
return offset + (alignment - delta);
}
static int
bin_reclaim_vlen(Symbol* tsym, Reclaim* reclaimer)
{
int stat = NC_NOERR;
size_t i;
Symbol* basetype = tsym->typ.basetype;
nc_vlen_t* vl = (nc_vlen_t*)(reclaimer->memory+reclaimer->offset);
/* Free up each entry in the vlen list */
if(vl->p != NULL) {
Reclaim vreclaimer;
vreclaimer.memory = vl->p;
vreclaimer.offset = 0;
for(i=0;i<vl->len;i++) {
vreclaimer.offset = read_alignment(vreclaimer.offset,basetype->typ.alignment);
if((stat = bin_reclaim_datar(basetype,&vreclaimer))) goto done;
vreclaimer.offset += basetype->typ.size;
}
reclaimer->offset += tsym->typ.size;
efree(vl->p);
}
done:
return stat;
}
static int
bin_reclaim_enum(Symbol* tsym, Reclaim* reclaimer)
{
return bin_reclaim_datar(tsym->typ.basetype,reclaimer);
}
static int
bin_reclaim_opaque(Symbol* tsym, Reclaim* reclaimer)
{
/* basically a fixed size sequence of bytes */
reclaimer->offset += tsym->typ.size;
return NC_NOERR;
}
static int
bin_reclaim_compound(Symbol* tsym, Reclaim* reclaimer)
{
int stat = NC_NOERR;
int nfields;
size_t fid, i, arraycount;
ptrdiff_t saveoffset;
reclaimer->offset = read_alignment(reclaimer->offset,tsym->typ.cmpdalign);
saveoffset = reclaimer->offset;
/* Get info about each field in turn and reclaim it */
nfields = listlength(tsym->subnodes);
for(fid=0;fid<nfields;fid++) {
Symbol* field = listget(tsym->subnodes,fid);
int ndims = field->typ.dimset.ndims;
/* compute the total number of elements in the field array */
for(i=0;i<ndims;i++) arraycount *= field->typ.dimset.dimsyms[i]->dim.declsize;
reclaimer->offset = read_alignment(reclaimer->offset,field->typ.alignment);
for(i=0;i<arraycount;i++) {
if((stat = bin_reclaim_datar(field->typ.basetype, reclaimer))) goto done;
}
}
reclaimer->offset = saveoffset;
reclaimer->offset += tsym->typ.size;
done:
return stat;
}
#endif /*ENABLE_BINARY*/

20
ncgen/bytebuffer.c
View File

@ -35,7 +35,7 @@ bbFail(void)
Bytebuffer*
bbNew(void)
{
Bytebuffer* bb = (Bytebuffer*)malloc(sizeof(Bytebuffer));
Bytebuffer* bb = (Bytebuffer*)emalloc(sizeof(Bytebuffer));
if(bb == NULL) return (Bytebuffer*)bbFail();
bb->alloc=0;
bb->length=0;
@ -53,7 +53,7 @@ bbSetalloc(Bytebuffer* bb, const unsigned int sz0)
if(sz <= 0) {sz = (bb->alloc?2*bb->alloc:DEFAULTALLOC);}
else if(bb->alloc >= sz) return TRUE;
else if(bb->nonextendible) return bbFail();
newcontent=(char*)calloc(sz,sizeof(char));
newcontent=(char*)ecalloc(sz*sizeof(char));
if(bb->alloc > 0 && bb->length > 0 && bb->content != NULL) {
memcpy((void*)newcontent,(void*)bb->content,sizeof(char)*bb->length);
}
@ -250,7 +250,7 @@ bbTailpeek(Bytebuffer* bb, char* pelem)
char*
bbDup(const Bytebuffer* bb)
{
char* result = (char*)malloc(bb->length+1);
char* result = (char*)emalloc(bb->length+1);
memcpy((void*)result,(const void*)bb->content,bb->length);
result[bb->length] = '\0'; /* just in case it is a string*/
return result;
@ -277,3 +277,17 @@ bbNull(Bytebuffer* bb)
bb->length--;
return 1;
}
/* Extract the content and leave content null */
char*
bbExtract(Bytebuffer* bb)
{
char* x = NULL;
if(bb == NULL || bb->content == NULL)
return NULL;
x = bb->content;
bb->content = NULL;
bb->length = 0;
bb->alloc = 0;
return x;
}

1
ncgen/bytebuffer.h
View File

@ -43,6 +43,7 @@ EXTERNC int bbCat(Bytebuffer*,const char*);
EXTERNC int bbCatbuf(Bytebuffer*,const Bytebuffer*);
EXTERNC int bbSetcontents(Bytebuffer*, char*, const unsigned int);
EXTERNC int bbNull(Bytebuffer*);
EXTERNC char* bbExtract(Bytebuffer*);
/* Following are always "in-lined"*/
#define bbLength(bb) ((bb)?(bb)->length:0U)

4
ncgen/cdata.c
View File

@ -19,7 +19,7 @@ c_charconstant(Generator* generator, Symbol* sym, Bytebuffer* codebuf, ...)
/* Just transfer charbuf to codebuf */
Bytebuffer* charbuf;
va_list ap;
vastart(ap,codebuf);
va_start(ap,codebuf);
charbuf = va_arg(ap, Bytebuffer*);
va_end(ap);
bbNull(charbuf);
@ -178,7 +178,7 @@ c_vlendecl(Generator* generator, Symbol* tsym, Bytebuffer* codebuf, int uid, siz
Bytebuffer* decl = bbNew();
Bytebuffer* vlenbuf;
va_list ap;
vastart(ap,count);
va_start(ap,count);
vlenbuf = va_arg(ap, Bytebuffer*);
va_end(ap);
bbprintf0(decl,"static const %s vlen_%u[] = {",

4
ncgen/cdfdata.c
View File

@ -340,8 +340,8 @@ genbin_vlenconstants(List* vlenconstants)
bbClear(memory);
count = 0;
if(chartype) {
/* Collect the char vlen in a separate buffer */
gen_charvlen(vlensrc,memory);
/* Collect the char sequence in a separate buffer */
gen_charseq(vlensrc,memory);
count = bbLength(memory);
} else {
while(srcmore(vlensrc)) {

8
ncgen/cvt.c
View File

@ -21,6 +21,9 @@ convert1(NCConstant* src, NCConstant* dst)
#ifdef _MSC_VER
int byteval;
#endif
memset(&tmp,0,sizeof(tmp));
dst->lineno = src->lineno;
/* Need to translate all possible sources to all possible sinks.*/
@ -30,7 +33,7 @@ convert1(NCConstant* src, NCConstant* dst)
/* special case for src being NC_FILLVALUE*/
if(src->nctype == NC_FILLVALUE) {
if(dst->nctype != NC_FILLVALUE) {
nc_getfill(dst);
nc_getfill(dst,NULL);
}
return;
}
@ -42,7 +45,7 @@ convert1(NCConstant* src, NCConstant* dst)
} else {
Symbol* econst;
econst = src->value.enumv;
convert1(&econst->typ.econst,dst);
convert1(econst->typ.econst,dst);
}
return;
} else if(dst->nctype == NC_ECONST) {
@ -655,4 +658,3 @@ convertFilterID(const char* id)
return nid;
return 0; /* Not a recognizable id */
}

699
ncgen/data.c
View File

@ -7,8 +7,11 @@
#include "includes.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
#include "dump.h"
#undef VERIFY
#define XVSNPRINTF vsnprintf
/*
#define XVSNPRINTF lvsnprintf
@ -18,7 +21,7 @@ extern int lvsnprintf(char*, size_t, const char*, va_list);
#define DATALISTINIT 32
/* Track all known datalist*/
Datalist* alldatalists = NULL;
List* alldatalists = NULL;
NCConstant nullconstant;
NCConstant fillconstant;
@ -31,32 +34,31 @@ Bytebuffer* stmt;
/* Forward */
static void setconstlist(NCConstant* con, Datalist* dl);
/**************************************************/
/**************************************************/
/* return 1 if the next element in the datasrc is compound*/
int
issublist(Datasrc* datasrc) {return istype(datasrc,NC_COMPOUND);}
/* return 1 if the next element in the datasrc is a string*/
int
isstring(Datasrc* datasrc) {return istype(datasrc,NC_STRING);}
/* return 1 if the next element in the datasrc is a fill value*/
int
isfillvalue(Datasrc* datasrc)
#ifdef VERIFY
/* index of match */
static int
verify(List* all, Datalist* dl)
{
return srcpeek(datasrc) == NULL || istype(datasrc,NC_FILLVALUE);
int i;
for(i=0;i<listlength(all);i++) {
void* pi = listget(all,i);
if(pi == dl)
return i;
}
return -1;
}
#endif
/* return 1 if the next element in the datasrc is nc_type*/
int
istype(Datasrc* datasrc , nc_type nctype)
/**************************************************/
/**************************************************/
NCConstant*
nullconst(void)
{
NCConstant* ci = srcpeek(datasrc);
if(ci != NULL && ci->nctype == nctype) return 1;
return 0;
NCConstant* n = ecalloc(sizeof(NCConstant));
return n;
}
int
@ -72,61 +74,15 @@ isstringable(nc_type nctype)
return 0;
}
/**************************************************/
void
freedatasrc(Datasrc* src)
{
efree(src);
}
Datasrc*
allocdatasrc(void)
{
Datasrc* src;
src = ecalloc(sizeof(Datasrc));
src->data = NULL;
src->index = 0;
src->length = 0;
src->prev = NULL;
return src;
}
Datasrc*
datalist2src(Datalist* list)
{
Datasrc* src;
ASSERT(list != NULL);
src = allocdatasrc();
src->data = list->data;
src->index = 0;
src->length = list->length;
DUMPSRC(src,"#");
return src;
}
Datasrc*
const2src(NCConstant* con)
{
Datasrc* src;
ASSERT(con != NULL);
src = allocdatasrc();
src->data = con;
src->index = 0;
src->length = 1;
DUMPSRC(src,"#");
return src;
}
NCConstant
NCConstant*
list2const(Datalist* list)
{
NCConstant con;
NCConstant* con = nullconst();
ASSERT(list != NULL);
con.nctype = NC_COMPOUND;
con.lineno = list->data[0].lineno;
con.value.compoundv = list;
con.filled = 0;
con->nctype = NC_COMPOUND;
con->lineno = list->data[0]->lineno;
setconstlist(con,list);
con->filled = 0;
return con;
}
@ -142,109 +98,6 @@ const2list(NCConstant* con)
return list;
}
NCConstant*
srcpeek(Datasrc* ds)
{
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return &ds->data[ds->index];
if(ds->spliced)
return srcpeek(ds->prev);
return NULL;
}
void
srcreset(Datasrc* ds)
{
ds->index = 0;
}
NCConstant*
srcnext(Datasrc* ds)
{
DUMPSRC(ds,"!");
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return &ds->data[ds->index++];
if(ds->spliced) {
srcpop(ds);
return srcnext(ds);
}
return NULL;
}
int
srcmore(Datasrc* ds)
{
if(ds == NULL) return 0;
if(ds->index < ds->length) return 1;
if(ds->spliced) return srcmore(ds->prev);
return 0;
}
int
srcline(Datasrc* ds)
{
int index = ds->index;
int len = ds->length;
/* pick closest available entry*/
if(len == 0) return 0;
if(index >= len) index = len-1;
return ds->data[index].lineno;
}
void
srcpush(Datasrc* src)
{
NCConstant* con;
ASSERT(src != NULL);
con = srcnext(src);
ASSERT(con->nctype == NC_COMPOUND);
srcpushlist(src,con->value.compoundv);
}
void
srcpushlist(Datasrc* src, Datalist* dl)
{
Datasrc* newsrc;
ASSERT(src != NULL && dl != NULL);
newsrc = allocdatasrc();
*newsrc = *src;
src->prev = newsrc;
src->index = 0;
src->data = dl->data;
src->length = dl->length;
DUMPSRC(src,">!");
}
void
srcpop(Datasrc* src)
{
if(src != NULL) {
Datasrc* prev = src->prev;
*src = *prev;
freedatasrc(prev);
}
DUMPSRC(src,"<");
}
void
srcsplice(Datasrc* ds, Datalist* list)
{
srcpushlist(ds,list);
ds->spliced = 1;
}
void
srcsetfill(Datasrc* ds, Datalist* list)
{
if(ds->index >= ds->length) PANIC("srcsetfill: no space");
if(ds->data[ds->index].nctype != NC_FILLVALUE) PANIC("srcsetfill: not fill");
ds->data[ds->index].nctype = NC_COMPOUND;
ds->data[ds->index].value.compoundv = list;
}
/**************************************************/
#ifdef GENDEBUG
void
@ -258,73 +111,114 @@ fflush(stderr);
bbFree(buf);
}
void
report0(char* lead, Datasrc* src, int index)
{
}
#endif
/**************************************************/
/* Shallow constant cloning*/
NCConstant
static void
setconstlist(NCConstant* con, Datalist* dl)
{
#ifdef VERIFY
int pos = verify(alldatalists,dl);
if(pos >= 0) {
dumpdatalist(listget(alldatalists,pos),"XXX");
}
#endif
con->value.compoundv = dl;
}
/* Deep constant cloning; return struct not pointer to struct*/
NCConstant*
cloneconstant(NCConstant* con)
{
NCConstant newcon = *con;
char* s;
switch (newcon.nctype) {
NCConstant* newcon = NULL;
Datalist* newdl = NULL;
char* s = NULL;
newcon = nullconst();
if(newcon == NULL) return newcon;
*newcon = *con;
switch (newcon->nctype) {
case NC_STRING:
s = (char*)ecalloc(newcon.value.stringv.len+1);
memcpy(s,newcon.value.stringv.stringv,newcon.value.stringv.len);
s[newcon.value.stringv.len] = '\0';
newcon.value.stringv.stringv = s;
if(newcon->value.stringv.len == 0)
s = NULL;
else {
s = (char*)ecalloc(newcon->value.stringv.len+1);
if(newcon->value.stringv.len > 0)
memcpy(s,newcon->value.stringv.stringv,newcon->value.stringv.len);
s[newcon->value.stringv.len] = '\0';
}
newcon->value.stringv.stringv = s;
break;
case NC_OPAQUE:
s = (char*)ecalloc(newcon.value.opaquev.len+1);
memcpy(s,newcon.value.opaquev.stringv,newcon.value.opaquev.len);
s[newcon.value.opaquev.len] = '\0';
newcon.value.opaquev.stringv = s;
s = (char*)ecalloc(newcon->value.opaquev.len+1);
if(newcon->value.opaquev.len > 0)
memcpy(s,newcon->value.opaquev.stringv,newcon->value.opaquev.len);
s[newcon->value.opaquev.len] = '\0';
newcon->value.opaquev.stringv = s;
break;
case NC_COMPOUND:
newdl = clonedatalist(con->value.compoundv);
setconstlist(newcon,newdl);
break;
default: break;
}
return newcon;
}
/* Deep constant clear*/
void
clearconstant(NCConstant* con)
{
if(con == NULL) return;
switch (con->nctype) {
case NC_STRING:
if(con->value.stringv.stringv != NULL)
efree(con->value.stringv.stringv);
break;
case NC_OPAQUE:
if(con->value.opaquev.stringv != NULL)
efree(con->value.opaquev.stringv);
break;
case NC_COMPOUND:
con->value.compoundv = NULL;
break;
default: break;
}
memset((void*)con,0,sizeof(NCConstant));
}
void
freeconstant(NCConstant* con, int shallow)
{
if(!shallow) clearconstant(con);
nullfree(con);
}
/**************************************************/
#if 0
Datalist*
datalistclone(Datalist* dl)
{
int i;
Datalist* clone = builddatalist(dl->length);
for(i=0;i<dl->length;i++) {
clone->data[i] = cloneconstant(dl->data+i);
clone->data[i] = cloneconstant(dl->data[i]);
}
return clone;
}
Datalist*
datalistconcat(Datalist* dl1, Datalist* dl2)
{
NCConstant* vector;
ASSERT(dl1 != NULL);
if(dl2 == NULL) return dl1;
vector = (NCConstant*)erealloc(dl1->data,sizeof(NCConstant)*(dl1->length+dl2->length));
if(vector == NULL) return NULL;
memcpy((void*)(vector+dl1->length),dl2->data,sizeof(NCConstant)*(dl2->length));
dl1->data = vector;
return dl1;
}
Datalist*
datalistappend(Datalist* dl, NCConstant* con)
{
NCConstant* vector;
NCConstant** vector;
ASSERT(dl != NULL);
if(con == NULL) return dl;
vector = (NCConstant*)erealloc(dl->data,sizeof(NCConstant)*(dl->length+1));
vector = (NCConstant**)erealloc(dl->data,sizeof(NCConstant*)*(dl->length+1));
if(vector == NULL) return NULL;
vector[dl->length] = *con;
vector[dl->length] = cloneconstant(con);
dl->length++;
dl->data = vector;
return dl;
@ -336,15 +230,16 @@ datalistreplace(Datalist* dl, unsigned int index, NCConstant* con)
ASSERT(dl != NULL);
ASSERT(index < dl->length);
ASSERT(con != NULL);
dl->data[index] = *con;
dl->data[index] = cloneconstant(con);
return dl;
}
#endif
int
datalistline(Datalist* ds)
{
if(ds == NULL || ds->length == 0) return 0;
return ds->data[0].lineno;
return ds->data[0]->lineno;
}
@ -441,19 +336,22 @@ wordstring(char* p, Bytebuffer* buf, int quote)
static const char zeros[] =
"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
void
alignbuffer(NCConstant* prim, Bytebuffer* buf)
{
int alignment,pad,offset;
ASSERT(prim->nctype != NC_COMPOUND);
if(prim->nctype == NC_ECONST)
alignment = nctypealignment(prim->value.enumv->typ.typecode);
alignment = ncaux_class_alignment(prim->value.enumv->typ.typecode);
else if(usingclassic && prim->nctype == NC_STRING)
alignment = nctypealignment(NC_CHAR);
alignment = ncaux_class_alignment(NC_CHAR);
else if(prim->nctype == NC_CHAR)
alignment = nctypealignment(NC_CHAR);
alignment = ncaux_class_alignment(NC_CHAR);
else
alignment = nctypealignment(prim->nctype);
alignment = ncaux_class_alignment(prim->nctype);
offset = bbLength(buf);
pad = getpadding(offset,alignment);
if(pad > 0) {
@ -553,10 +451,10 @@ retry: switch ((c=*p++)) {
case 'u':
if(hcount == 2) {
snprintf(tmp,sizeof(tmp),"%hhu",
(unsigned int)va_arg(argv,unsigned int));
(unsigned char)va_arg(argv,unsigned int));
} else if(hcount == 1) {
snprintf(tmp,sizeof(tmp),"%hu",
(unsigned int)va_arg(argv,unsigned int));
(unsigned short)va_arg(argv,unsigned int));
} else if(lcount == 2) {
snprintf(tmp,sizeof(tmp),"%llu",
(unsigned long long)va_arg(argv,unsigned long long));
@ -572,10 +470,10 @@ retry: switch ((c=*p++)) {
case 'd':
if(hcount == 2) {
snprintf(tmp,sizeof(tmp),"%hhd",
(signed int)va_arg(argv,signed int));
(signed char)va_arg(argv,signed int));
} else if(hcount == 1) {
snprintf(tmp,sizeof(tmp),"%hd",
(signed int)va_arg(argv,signed int));
(signed short)va_arg(argv,signed int));
} else if(lcount == 2) {
snprintf(tmp,sizeof(tmp),"%lld",
(signed long long)va_arg(argv,signed long long));
@ -645,19 +543,21 @@ codeprintf(const char *fmt, ...)
}
NCConstant*
emptycompoundconst(int lineno, NCConstant* c)
emptycompoundconst(int lineno)
{
ASSERT(c != NULL);
NCConstant* c = nullconst();
c->lineno = lineno;
c->nctype = NC_COMPOUND;
c->value.compoundv = builddatalist(0);
setconstlist(c,builddatalist(0));
c->filled = 0;
return c;
}
/* Make an empty string constant*/
NCConstant*
emptystringconst(int lineno, NCConstant* c)
emptystringconst(int lineno)
{
NCConstant* c = nullconst();
ASSERT(c != NULL);
c->lineno = lineno;
c->nctype = NC_STRING;
@ -688,17 +588,24 @@ void
dlextend(Datalist* dl)
{
size_t newalloc;
NCConstant* newdata = NULL;
NCConstant** newdata = NULL;
newalloc = (dl->alloc > 0?2*dl->alloc:2);
newdata = (NCConstant*)ecalloc(newalloc*sizeof(NCConstant));
newdata = (NCConstant**)ecalloc(newalloc*sizeof(NCConstant*));
if(dl->length > 0)
memcpy(newdata,dl->data,sizeof(NCConstant)*dl->length);
memcpy(newdata,dl->data,sizeof(NCConstant*)*dl->length);
dl->alloc = newalloc;
nullfree(dl->data);
dl->data = newdata;
}
void
capture(Datalist* dl)
{
if(alldatalists == NULL) alldatalists = listnew();
listpush(alldatalists,dl);
}
Datalist*
builddatalist(int initial)
{
@ -706,9 +613,8 @@ builddatalist(int initial)
if(initial <= 0) initial = DATALISTINIT;
initial++; /* for header*/
ci = (Datalist*)ecalloc(sizeof(Datalist));
memset((void*)ci,0,sizeof(Datalist)); /* only clear the hdr*/
ci->data = (NCConstant*)ecalloc(sizeof(NCConstant)*initial);
memset((void*)ci->data,0,sizeof(NCConstant)*initial);
if(ci == NULL) semerror(0,"out of memory\n");
ci->data = (NCConstant**)ecalloc(sizeof(NCConstant*)*initial);
ci->alloc = initial;
ci->length = 0;
return ci;
@ -719,44 +625,319 @@ dlappend(Datalist* dl, NCConstant* constant)
{
if(dl->length >= dl->alloc)
dlextend(dl);
if(constant == NULL) constant = &nullconstant;
dl->data[dl->length++] = *constant;
dl->data[dl->length++] = (constant);
}
NCConstant
void
dlset(Datalist* dl, size_t pos, NCConstant* constant)
{
ASSERT(pos < dl->length);
dl->data[pos] = (constant);
}
/* Convert a datalist to a compound constant */
NCConstant*
builddatasublist(Datalist* dl)
{
NCConstant d;
d.nctype = NC_COMPOUND;
d.lineno = (dl->length > 0?dl->data[0].lineno:0);
d.value.compoundv = dl;
d.filled = 0;
NCConstant* d = nullconst();
d->nctype = NC_COMPOUND;
d->lineno = (dl->length > 0?dl->data[0]->lineno:0);
setconstlist(d,dl);
d->filled = 0;
return d;
}
/*! Function to free an allocated datalist.
/* Deep copy */
Datalist*
clonedatalist(Datalist* dl)
{
int i;
size_t len;
Datalist* newdl;
This function is used to free an individual datalist
object. It is possible, hypothetically, that a
datalist will appear in the middle of a set of datalists,
in which case we'll need to determine that and shuffle around
'next' pointers. For the time being, this assumes that
we are freeing a datalist which was allocated locally
and must be discarded.
Using this function instead of an inline 'free' just in
case we ever want to extend it, we won't have to go back
and re-write a bunch of stuff. I hope.
@param dlist Pointer to datalist object being freed.
*/
void dlfree(Datalist **dlist) {
if(*dlist) free(*dlist);
dlist = NULL;
if(dl == NULL) return NULL;
len = datalistlen(dl);
newdl = builddatalist(len);
/* initialize */
for(i=0;i<len;i++) {
NCConstant* con = datalistith(dl,i);
con = cloneconstant(con);
dlappend(newdl,con);
}
#if 0
newdl->vlen = dl->vlen;
#endif
newdl->readonly = dl->readonly;
return newdl;
}
/* recursive helpers */
#if 0
static int
isdup(Datalist* dl)
{
int i;
size_t limit = listlength(alldatalists);
for(i=0;i<limit;i++) {
Datalist* di = listget(alldatalists,i);
if(di == dl) return 1;
}
return 0;
}
#endif
void
reclaimconstant(NCConstant* con)
{
if(con == NULL) return;
switch (con->nctype) {
case NC_STRING:
if(con->value.stringv.stringv != NULL)
efree(con->value.stringv.stringv);
break;
case NC_OPAQUE:
if(con->value.opaquev.stringv != NULL)
efree(con->value.opaquev.stringv);
break;
case NC_COMPOUND:
#ifdef VERIFY
{int pos;
if((pos=verify(alldatalists,con->value.compoundv)) >= 0) {
dumpdatalist(listget(alldatalists,pos),"XXX");
abort();
}
}
#endif
reclaimdatalist(con->value.compoundv);
con->value.compoundv = NULL;
break;
default: break;
}
efree(con);
}
void
reclaimdatalist(Datalist* list)
{
int i;
if(list == NULL) return;
if(list->data != NULL) {
for(i=0;i<list->length;i++) {
NCConstant* con = list->data[i];
if(con != NULL) reclaimconstant(con);
}
efree(list->data);
list->data = NULL;
}
efree(list);
}
void
reclaimalldatalists(void)
{
int i;
#if 0
int j;
/* Remove duplicates */
for(i=0;i<listlength(alldatalists);i++) {
Datalist* di = listget(alldatalists,i);
if(di == NULL) continue;
for(j=i;j<listlength(alldatalists);j++) {
Datalist* dj = listget(alldatalists,j);
if(dj == di) {
listset(alldatalists,j,NULL);
fprintf(stderr,"XXX\n");
}
}
}
#endif
for(i=0;i<listlength(alldatalists);i++) {
Datalist* di = listget(alldatalists,i);
if(di != NULL)
reclaimdatalist(di);
}
efree(alldatalists);
alldatalists = NULL;
}
/* Obsolete */
#if 0
/* return 1 if the next element in the datasrc is compound*/
int
issublist(Datasrc* datasrc) {return istype(datasrc,NC_COMPOUND);}
/* return 1 if the next element in the datasrc is a string*/
int
isstring(Datasrc* datasrc) {return istype(datasrc,NC_STRING);}
/* return 1 if the next element in the datasrc is a fill value*/
int
isfillvalue(Datasrc* datasrc)
{
return srcpeek(datasrc) == NULL || istype(datasrc,NC_FILLVALUE);
}
/* return 1 if the next element in the datasrc is nc_type*/
int
istype(Datasrc* datasrc , nc_type nctype)
{
NCConstant* ci = srcpeek(datasrc);
if(ci != NULL && ci->nctype == nctype) return 1;
return 0;
}
/**************************************************/
void
freedatasrc(Datasrc* src)
{
efree(src);
}
Datasrc*
allocdatasrc(void)
{
Datasrc* src;
src = ecalloc(sizeof(Datasrc));
src->data = NULL;
src->index = 0;
src->length = 0;
src->prev = NULL;
return src;
}
Datasrc*
datalist2src(Datalist* list)
{
Datasrc* src;
ASSERT(list != NULL);
src = allocdatasrc();
src->data = list->data;
src->index = 0;
src->length = list->length;
DUMPSRC(src,"#");
return src;
}
Datasrc*
const2src(NCConstant* con)
{
Datasrc* src;
ASSERT(con != NULL);
src = allocdatasrc();
src->data = emalloc(sizeof(NCConstant*));
src->data[0] = con;
src->index = 0;
src->length = 1;
DUMPSRC(src,"#");
return src;
}
NCConstant*
srcpeek(Datasrc* ds)
{
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return ds->data[ds->index];
if(ds->spliced)
return srcpeek(ds->prev);
return NULL;
}
void
srcreset(Datasrc* ds)
{
ds->index = 0;
}
NCConstant*
srcnext(Datasrc* ds)
{
DUMPSRC(ds,"!");
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return ds->data[ds->index++];
if(ds->spliced) {
srcpop(ds);
return srcnext(ds);
}
return NULL;
}
int
srcmore(Datasrc* ds)
{
if(ds == NULL) return 0;
if(ds->index < ds->length) return 1;
if(ds->spliced) return srcmore(ds->prev);
return 0;
}
int
srcline(Datasrc* ds)
{
int index = ds->index;
int len = ds->length;
/* pick closest available entry*/
if(len == 0) return 0;
if(index >= len) index = len-1;
return ds->data[index]->lineno;
}
void
srcpush(Datasrc* src)
{
NCConstant* con;
ASSERT(src != NULL);
con = srcnext(src);
ASSERT(con->nctype == NC_COMPOUND);
srcpushlist(src,con->value.compoundv);
}
void
srcpushlist(Datasrc* src, Datalist* dl)
{
Datasrc* newsrc;
ASSERT(src != NULL && dl != NULL);
newsrc = allocdatasrc();
*newsrc = *src;
src->prev = newsrc;
src->index = 0;
src->data = dl->data;
src->length = dl->length;
DUMPSRC(src,">!");
}
void
srcpop(Datasrc* src)
{
if(src != NULL) {
Datasrc* prev = src->prev;
*src = *prev;
freedatasrc(prev);
}
DUMPSRC(src,"<");
}
void
srcsplice(Datasrc* ds, Datalist* list)
{
srcpushlist(ds,list);
ds->spliced = 1;
}
void
srcsetfill(Datasrc* ds, Datalist* list)
{
if(ds->index >= ds->length) PANIC("srcsetfill: no space");
if(ds->data[ds->index]->nctype != NC_FILLVALUE) PANIC("srcsetfill: not fill");
ds->data[ds->index]->nctype = NC_COMPOUND;
setconstlist(ds->data[ds->index],list);
}
#endif /*0*/

87
ncgen/data.h
View File

@ -12,7 +12,7 @@
# include <varargs.h>
#endif
/* nmemonic*/
/* nmemonics*/
#define TOPLEVEL 1
/* Forward types */
@ -56,18 +56,19 @@ typedef struct NCConstant {
} NCConstant;
typedef struct Datalist {
struct Datalist* next; /* chain of all known datalists*/
int readonly; /* data field is shared with another Datalist*/
size_t length; /* |data| */
size_t alloc; /* track total allocated space for data field*/
NCConstant* data; /* actual list of constants constituting the datalist*/
NCConstant** data; /* actual list of constants constituting the datalist*/
/* Track various values associated with the datalist*/
/* (used to be in Constvalue.compoundv)*/
#if 0
struct Vlen {
struct Symbol* schema; /* type/var that defines structure of this*/
unsigned int count; /* # of vlen basetype instances*/
unsigned int uid; /* unique id for NC_VLEN*/
} vlen;
#endif
} Datalist;
/* Define a structure to track
@ -75,56 +76,46 @@ typedef struct Datalist {
In effect, we are parsing the data sequence.
Push and pop of data sources is supported (see srcpush() below).*/
typedef struct Datasrc {
NCConstant* data; /* duplicate pointer; so do not free.*/
NCConstant** data; /* duplicate pointer; so do not free.*/
int index;
int length;
int spliced; /* Was this list spliced into our parent ? */
struct Datasrc* prev; /* linked list for debugging */
} Datasrc;
#if 0
/* Define a holder for passing a start/count array */
struct Vlendata {
char* data;
unsigned long count;
};
extern struct Vlendata* vlendata;
extern Datalist* alldatalists;
#endif
extern List* alldatalists;
/* from: data.c */
extern Datalist* builddatalist(int initialize);
extern void dlfree(Datalist **dlist);
extern void capture(Datalist* dl);
extern void dlappend(Datalist*, NCConstant*);
extern NCConstant builddatasublist(Datalist* dl);
extern void dlset(Datalist*, size_t, NCConstant*);
extern NCConstant* builddatasublist(Datalist* dl);
extern void dlextend(Datalist* dl);
extern void dlsetalloc(Datalist* dl, size_t newalloc);
extern Datalist* clonedatalist(Datalist* dl);
extern void reclaimalldatalists(void);
extern void reclaimdatalist(Datalist*);
extern void reclaimconstant(NCConstant*);
int datalistline(Datalist*);
#define datalistith(dl,i) ((dl)==NULL?NULL:((i) >= (dl)->length?NULL:&(dl)->data[i]))
#define datalistith(dl,i) ((dl)==NULL?NULL:((i) >= (dl)->length?NULL:(dl)->data[i]))
#define datalistlen(dl) ((dl)==NULL?0:(dl)->length)
Datasrc* datalist2src(Datalist* list);
Datasrc* const2src(NCConstant*);
NCConstant list2const(Datalist*);
NCConstant* list2const(Datalist*);
Datalist* const2list(NCConstant* con);
void freedatasrc(Datasrc* src);
int issublist(Datasrc* src);
int isstring(Datasrc* src);
int isfillvalue(Datasrc* src);
int istype(Datasrc* src, nc_type);
int isstringable(nc_type nctype);
void srcpush(Datasrc*);
void srcpushlist(Datasrc* src, Datalist* cmpd);
void srcpop(Datasrc*);
void srcsetfill(Datasrc* ds, Datalist* list);
NCConstant* srcnext(Datasrc*);
int srcmore(Datasrc*);
int srcline(Datasrc* ds);
void srcreset(Datasrc* ds);
#define srclen(s) ((s)==NULL?0:(s)->length)
#define islistconst(con) ((con)!=NULL && (con)->nctype == NC_COMPOUND)
#define isfillconst(con) ((con)!=NULL && (con)->nctype == NC_FILLVALUE)
#define constline(con) (con==NULL?0:(con)->lineno)
@ -133,9 +124,13 @@ void srcreset(Datasrc* ds);
#define isnilconst(con) ((con)!=NULL && (con)->nctype == NC_NIL)
#define compoundfor(con) ((con)==NULL?NULL:(con)->value.compoundv)
NCConstant* emptystringconst(int,NCConstant*);
NCConstant* emptycompoundconst(int lineno);
NCConstant* emptystringconst(int);
NCConstant cloneconstant(NCConstant* con); /* shallow clone*/
NCConstant* cloneconstant(NCConstant* con); /* deep clone*/
void clearconstant(NCConstant* con); /* deep clear*/
#define freeconst(con) freeconstant(con,DEEP);
void freeconstant(NCConstant* con, int shallow);
void alignbuffer(struct NCConstant* prim, Bytebuffer* buf);
@ -158,23 +153,18 @@ char* word(char* p, Bytebuffer* buf);
extern Bytebuffer* codebuffer; /* buffer over the std output */
extern Bytebuffer* stmt; /* single stmt text generation */
#ifdef FASTDATASRC
#define srcpeek(ds) ((ds)==NULL || (ds)->index >= (ds)->max?NULL:(ds)->data+(ds)->index)
#else
NCConstant* srcpeek(Datasrc*);
#endif
/* Aliases */
#define srcincr(src) srcnext(src)
#define srcget(src) srcpeek(src)
extern NCConstant* nullconst(void);
extern NCConstant nullconstant;
extern NCConstant fillconstant;
extern NCConstant nilconstant;
/* From genchar.c */
void gen_charattr(Datalist*, Bytebuffer*);
void gen_charvlen(Datalist*, Bytebuffer*);
void gen_charseq(Datalist*, Bytebuffer*);
void gen_chararray(struct Dimset*, int, Datalist*, Bytebuffer*, Datalist* fillsrc);
typedef enum ListClass {
@ -202,5 +192,30 @@ extern void generate_vardata(struct Symbol*, Generator*, Writer writer,Bytebuffe
extern void generate_basetype(struct Symbol*,NCConstant*,Bytebuffer*,Datalist*,Generator*);
/* Obsolete */
#if 0
Datasrc* datalist2src(Datalist* list);
Datasrc* const2src(NCConstant*);
void freedatasrc(Datasrc* src);
int issublist(Datasrc* src);
int isstring(Datasrc* src);
int isfillvalue(Datasrc* src);
int istype(Datasrc* src, nc_type);
void srcpush(Datasrc*);
void srcpushlist(Datasrc* src, Datalist* cmpd);
void srcpop(Datasrc*);
void srcsetfill(Datasrc* ds, Datalist* list);
NCConstant* srcnext(Datasrc*);
int srcmore(Datasrc*);
int srcline(Datasrc* ds);
void srcreset(Datasrc* ds);
#define srclen(s) ((s)==NULL?0:(s)->length)
#ifdef FASTDATASRC
#define srcpeek(ds) ((ds)==NULL || (ds)->index >= (ds)->max?NULL:(ds)->data+(ds)->index)
#else
NCConstant* srcpeek(Datasrc*);
#endif
#endif /*0*/
#endif /*DATA_H*/

36
ncgen/debug.c
View File

@ -5,14 +5,27 @@ See LICENSE.txt for license information.
#include "includes.h"
#define TRACE
extern char* ncclassname(nc_class);
#ifdef DEBUG
int debug = 1;
#ifdef TRACE
#define T(fcn,mem) {if(trace) {fprintf(stderr,"X: %s: %p\n", fcn,mem);}}
#else
int debug = 0;
#define T(fcn,mem)
#endif
int debug = 0;
static int trace = 0;
int
settrace(int tf)
{
trace = tf;
return 1;
}
void fdebug(const char *fmt, ...)
{
va_list argv;
@ -31,16 +44,29 @@ chkfree(void* memory)
if(memory == NULL) {
panic("free: null memory");
}
T("free",memory);
free(memory);
}
void*
chkmalloc(size_t size)
{
void* memory = malloc(size);
if(memory == NULL) {
panic("malloc:out of memory");
}
T("malloc",memory);
return memory;
}
void*
chkcalloc(size_t size)
{
void* memory = calloc(size,1); /* use calloc to zero memory*/
if(memory == NULL) {
panic("malloc:out of memory");
panic("calloc:out of memory");
}
T("calloc",memory);
return memory;
}
@ -51,6 +77,7 @@ chkrealloc(void* ptr, size_t size)
if(memory == NULL) {
panic("realloc:out of memory");
}
if(ptr != memory) {T("free",memory); T("realloc",memory);}
return memory;
}
@ -65,6 +92,7 @@ chkstrdup(const char* s)
if(dup == NULL) {
panic("strdup: out of memory");
}
T("strdup",dup);
return dup;
}

7
ncgen/debug.h
View File

@ -28,11 +28,10 @@
# endif
#endif
extern int settrace(int);
extern int ncgdebug;
extern int debug;
extern int ncgdebug;
extern void fdebug(const char *fmt, ...);
@ -50,10 +49,12 @@ The wrapped version:
3. zeros all allocated memory.
*/
#define emalloc(x) chkmalloc(x) /*note only single arg */
#define ecalloc(x) chkcalloc(x) /*note only single arg */
#define erealloc(p,x) chkrealloc(p,x)
#define efree(x) chkfree(x)
#define estrdup(x) chkstrdup(x)
extern void* chkmalloc(size_t);
extern void* chkcalloc(size_t);
extern void* chkrealloc(void*,size_t);
extern void chkfree(void*);

11
ncgen/dump.c
View File

@ -47,7 +47,7 @@ void
bufdump(Datalist* list, Bytebuffer* buf)
{
int i;
NCConstant* dp;
NCConstant** dpl;
unsigned int count;
if(list == NULL) {
@ -56,7 +56,8 @@ bufdump(Datalist* list, Bytebuffer* buf)
}
count = list->length;
for(dp=list->data,i=0;i<count;i++,dp++) {
for(dpl=list->data,i=0;i<count;i++,dpl++) {
NCConstant* dp = *dpl;
switch (dp->nctype) {
case NC_COMPOUND:
bbCat(buf,"{");
@ -258,6 +259,7 @@ dumpconstant1(NCConstant* con)
#define MAXELEM 8
#define MAXDEPTH 4
#if 0
void
dumpsrc0(Datasrc* src,char* tag)
{
@ -272,12 +274,12 @@ dumpsrc0(Datasrc* src,char* tag)
fprintf(stderr,"[%d/%d]",src->index,src->length);
for(i=0;i<index;i++) {
fprintf(stderr," ");
dumpconstant1(&src->data[i]);
dumpconstant1(src->data[i]);
}
fprintf(stderr,"^");
for(i=index;i<count;i++) {
fprintf(stderr," ");
dumpconstant1(&src->data[i]);
dumpconstant1(src->data[i]);
}
if(count < src->length) fprintf(stderr,"...");
fprintf(stderr," | ");
@ -296,3 +298,4 @@ dumpsrc(Datasrc* src,char* tag)
#endif
dumpsrc0(src,tag);
}
#endif

7
ncgen/dump.h
View File

@ -14,12 +14,5 @@ extern void dumpdatalist(Datalist*,char*);
extern void dumpconstant(NCConstant*,char*);
extern void bufdump(Datalist*,Bytebuffer*);
extern void dumpgroup(Symbol* g);
extern void dumpsrc(Datasrc*,char*);
#ifdef F
#define DUMPSRC(src,tag) dumpsrc(src,tag)
#else
#define DUMPSRC(src,tag)
#endif
#endif

74
ncgen/escapes.c
View File

@ -606,16 +606,22 @@ unescapeoct(const char* s)
int
unescape(
char *s, /* fill with contents of yytext, with escapes removed.
s and yytext may be same*/
const char *yytext,
int yyleng,
int isident)
const char *yytext, /* text to unescape */
int yyleng, /* length of yytext */
int isident, /* Is this an identifier? */
char** sp /* Return the unescaped version of yytext */
)
{
char* s = NULL; /* unescaped string */
const char *t, *tend;
char* p;
int b;
/* expand "\" escapes, e.g. "\t" to tab character */
s = (char*)emalloc(yyleng+1);
memcpy(s,yytext,yyleng);
s[yyleng] = '\0';
/* translate "\" escapes, e.g. "\t" to tab character */
t = yytext;
tend = t + yyleng;
p = s;
@ -689,5 +695,59 @@ unescape(
}
}
*p = '\0';
return (p - s);
if(sp) *sp = s;
return (p-s);
}
static int
ishex(int c)
{
return ((c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9'));
}
static int
isoct(int c)
{
return ((c >= '0' && c <= '7'));
}
/**
Do equivalent of strchr, but taking escapes into account
Set octhex to true if the string might contain \ddd or \xdd.
Note that escaped versions of the stop character are not
considered to match.
WARNING: if char is not found, then return pointer to the
trailing nul char
*/
char*
esc_strchr(char* s, int stopc, int octhex)
{
char* p;
int c;
for(p=s;(c=*p);) {
if(c == '\\') {
if(p[1] == '\0' && c == stopc)
return p; /* special case of '\\' at end of string */
p++;
c = *p;
/* FiX: allow utf8 digits? */
if(octhex && (c == 'x' || c == 'X')
&& ishex(p[1]) && ishex(p[2]))
p += 3;
else if(octhex && isoct(c) && isoct(p[1]) && isoct(p[2]))
p += 3;
else /* treat like just '\<c>' */
p++;
} else if(stopc == c)
break;
else
p++;
}
return p; /* not found */
}

2
ncgen/f77data.c
View File

@ -19,7 +19,7 @@ f77_charconstant(Generator* generator, Symbol* sym, Bytebuffer* codebuf, ...)
/* Just transfer charbuf to codebuf */
Bytebuffer* charbuf;
va_list ap;
vastart(ap,codebuf);
va_start(ap,codebuf);
charbuf = va_arg(ap, Bytebuffer*);
va_end(ap);
bbNull(charbuf);

525
ncgen/genbin.c
View File

@ -5,39 +5,39 @@
*********************************************************************/
#include "includes.h"
#include <ctype.h> /* for isprint() */
#include <ctype.h> /* for isprint() */
#include "netcdf_aux.h"
#ifdef ENABLE_BINARY
#undef TRACE
/* Forward*/
static void genbin_defineattr(Symbol* asym);
static void genbin_definevardata(Symbol* vsym);
static int genbin_defineattr(Symbol* asym);
static int genbin_definevardata(Symbol* vsym);
static int genbin_write(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writevar(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
static int genbin_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
#ifdef USE_NETCDF4
static void genbin_deftype(Symbol* tsym);
static void genbin_definespecialattributes(Symbol* var);
static int genbin_deftype(Symbol* tsym);
static int genbin_definespecialattributes(Symbol* var);
#endif
/*
* Generate C code for creating netCDF from in-memory structure.
*/
void
gen_netcdf(const char *filename)
genbin_netcdf(void)
{
int stat, ncid;
int idim, ivar, iatt;
int ndims, nvars, natts, ngatts;
const char* filename = rootgroup->file.filename;
#ifdef USE_NETCDF4
int ntyps, ngrps, igrp;
#endif
Bytebuffer* databuf = bbNew();
ndims = listlength(dimdefs);
nvars = listlength(vardefs);
natts = listlength(attdefs);
@ -63,27 +63,27 @@ gen_netcdf(const char *filename)
check_err(stat,__LINE__,__FILE__);
/* ncid created above is also root group*/
rootgroup->ncid = ncid;
rootgroup->nc_id = ncid;
#ifdef USE_NETCDF4
/* Define the group structure */
/* walking grdefs list will do a preorder walk of all defined groups*/
for(igrp=0;igrp<ngrps;igrp++) {
Symbol* gsym = (Symbol*)listget(grpdefs,igrp);
if(gsym == rootgroup) continue; /* ignore root group*/
stat = nc_def_grp(gsym->container->ncid,gsym->name,&gsym->ncid);
check_err(stat,__LINE__,__FILE__);
Symbol* gsym = (Symbol*)listget(grpdefs,igrp);
if(gsym == rootgroup) continue; /* ignore root group*/
stat = nc_def_grp(gsym->container->nc_id,gsym->name,&gsym->nc_id);
check_err(stat,__LINE__,__FILE__);
}
#endif
#ifdef USE_NETCDF4
/* Define the types*/
if (ntyps > 0) {
int ityp;
for(ityp = 0; ityp < ntyps; ityp++) {
Symbol* tsym = (Symbol*)listget(typdefs,ityp);
genbin_deftype(tsym);
}
int ityp;
for(ityp = 0; ityp < ntyps; ityp++) {
Symbol* tsym = (Symbol*)listget(typdefs,ityp);
genbin_deftype(tsym);
}
}
#endif
@ -91,74 +91,74 @@ gen_netcdf(const char *filename)
if (ndims > 0) {
for(idim = 0; idim < ndims; idim++) {
Symbol* dsym = (Symbol*)listget(dimdefs,idim);
stat = nc_def_dim(dsym->container->ncid,
dsym->name,
(dsym->dim.isunlimited?NC_UNLIMITED:dsym->dim.declsize),
&dsym->ncid);
check_err(stat,__LINE__,__FILE__);
stat = nc_def_dim(dsym->container->nc_id,
dsym->name,
(dsym->dim.isunlimited?NC_UNLIMITED:dsym->dim.declsize),
&dsym->nc_id);
check_err(stat,__LINE__,__FILE__);
}
}
/* define variables from info in vars array */
if (nvars > 0) {
for(ivar = 0; ivar < nvars; ivar++) {
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* vsym = (Symbol*)listget(vardefs,ivar);
if (vsym->typ.dimset.ndims > 0) { /* a dimensioned variable */
/* construct a vector of dimension ids*/
int dimids[NC_MAX_VAR_DIMS];
for(idim=0;idim<vsym->typ.dimset.ndims;idim++)
dimids[idim] = vsym->typ.dimset.dimsyms[idim]->ncid;
stat = nc_def_var(vsym->container->ncid,
vsym->name,
vsym->typ.basetype->ncid,
vsym->typ.dimset.ndims,
dimids,
&vsym->ncid);
} else { /* a scalar */
stat = nc_def_var(vsym->container->ncid,
vsym->name,
vsym->typ.basetype->ncid,
vsym->typ.dimset.ndims,
NULL,
&vsym->ncid);
}
check_err(stat,__LINE__,__FILE__);
}
if (vsym->typ.dimset.ndims > 0) { /* a dimensioned variable */
/* construct a vector of dimension ids*/
int dimids[NC_MAX_VAR_DIMS];
for(idim=0;idim<vsym->typ.dimset.ndims;idim++)
dimids[idim] = vsym->typ.dimset.dimsyms[idim]->nc_id;
stat = nc_def_var(vsym->container->nc_id,
vsym->name,
vsym->typ.basetype->nc_id,
vsym->typ.dimset.ndims,
dimids,
&vsym->nc_id);
} else { /* a scalar */
stat = nc_def_var(vsym->container->nc_id,
vsym->name,
vsym->typ.basetype->nc_id,
vsym->typ.dimset.ndims,
NULL,
&vsym->nc_id);
}
check_err(stat,__LINE__,__FILE__);
}
}
#ifdef USE_NETCDF4
/* define special variable properties */
if(nvars > 0) {
for(ivar = 0; ivar < nvars; ivar++) {
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* var = (Symbol*)listget(vardefs,ivar);
genbin_definespecialattributes(var);
}
genbin_definespecialattributes(var);
}
}
#endif /*USE_NETCDF4*/
/* define global attributes */
if(ngatts > 0) {
for(iatt = 0; iatt < ngatts; iatt++) {
Symbol* gasym = (Symbol*)listget(gattdefs,iatt);
genbin_defineattr(gasym);
}
for(iatt = 0; iatt < ngatts; iatt++) {
Symbol* gasym = (Symbol*)listget(gattdefs,iatt);
genbin_defineattr(gasym);
}
}
/* define per-variable attributes */
if(natts > 0) {
for(iatt = 0; iatt < natts; iatt++) {
Symbol* asym = (Symbol*)listget(attdefs,iatt);
genbin_defineattr(asym);
}
for(iatt = 0; iatt < natts; iatt++) {
Symbol* asym = (Symbol*)listget(attdefs,iatt);
genbin_defineattr(asym);
}
}
if (nofill_flag) {
stat = nc_set_fill(rootgroup->ncid, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0);
check_err(stat,__LINE__,__FILE__);
}
/* leave define mode */
stat = nc_enddef(rootgroup->ncid);
stat = nc_enddef(rootgroup->nc_id);
check_err(stat,__LINE__,__FILE__);
if(!header_only) {
@ -167,13 +167,11 @@ gen_netcdf(const char *filename)
for(ivar = 0; ivar < nvars; ivar++) {
Symbol* vsym = (Symbol*)listget(vardefs,ivar);
if(vsym->data != NULL) {
bbClear(databuf);
genbin_definevardata(vsym);
}
}
}
}
bbFree(databuf);
}
#ifdef USE_NETCDF4
@ -190,36 +188,35 @@ genbin_defineglobalspecials(void)
if(!/*Main.*/format_attribute) return;
/* Watch out, this is a global Attribute */
format = kind_string(/*Main.*/format_flag);
stat = nc_put_att_text(rootgroup->ncid,NC_GLOBAL,"_Format",strlen(format),format);
stat = nc_put_att_text(rootgroup->nc_id,NC_GLOBAL,"_Format",strlen(format),format);
check_err(stat,__LINE__,__FILE__);
}
#endif /*0*/
static void
static int
genbin_definespecialattributes(Symbol* var)
{
int stat = NC_NOERR;
Specialdata* special = &var->var.special;
Specialdata* special = var->var.special;
if(special->flags & _STORAGE_FLAG) {
int storage = special->_Storage;
size_t* chunks = special->_ChunkSizes;
if(special->nchunks == 0 || chunks == NULL) chunks = NULL;
stat = nc_def_var_chunking(var->container->ncid,
var->ncid,
(storage == NC_CONTIGUOUS?NC_CONTIGUOUS
:NC_CHUNKED),
chunks);
if(special->_Storage == NC_CONTIGUOUS) {
stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CONTIGUOUS, NULL);
} else { /* chunked */
if(special->nchunks == 0 || special->_ChunkSizes == NULL)
derror("NC_CHUNKED requested, but no chunksizes specified");
stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CHUNKED, special->_ChunkSizes);
}
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _FLETCHER32_FLAG) {
stat = nc_def_var_fletcher32(var->container->ncid,
var->ncid,
stat = nc_def_var_fletcher32(var->container->nc_id,
var->nc_id,
special->_Fletcher32);
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & (_DEFLATE_FLAG | _SHUFFLE_FLAG)) {
stat = nc_def_var_deflate(var->container->ncid,
var->ncid,
stat = nc_def_var_deflate(var->container->nc_id,
var->nc_id,
(special->_Shuffle == 1?1:0),
(special->_DeflateLevel >= 0?1:0),
(special->_DeflateLevel >= 0?special->_DeflateLevel
@ -227,56 +224,57 @@ genbin_definespecialattributes(Symbol* var)
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _ENDIAN_FLAG) {
stat = nc_def_var_endian(var->container->ncid,
var->ncid,
stat = nc_def_var_endian(var->container->nc_id,
var->nc_id,
(special->_Endianness == NC_ENDIAN_LITTLE?
NC_ENDIAN_LITTLE
:NC_ENDIAN_BIG));
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _NOFILL_FLAG) {
stat = nc_def_var_fill(var->container->ncid,
var->ncid,
(special->_Fill?NC_FILL:NC_NOFILL),
stat = nc_def_var_fill(var->container->nc_id,
var->nc_id,
(special->_Fill?NC_FILL:NC_NOFILL),
NULL);
check_err(stat,__LINE__,__FILE__);
}
if(special->flags & _FILTER_FLAG) {
/* Special check for alternate way to specify _Deflate */
if(special->_FilterID == ZIP_ID) {
unsigned int level;
if(special->nparams == 0 || special->_FilterParams == NULL)
level = 9; /* default */
else
level = special->_FilterParams[0];
if(level > 9)
derror("Illegal deflate level");
else {
stat = nc_def_var_deflate(var->container->ncid,
var->ncid,
(special->_Shuffle == 1?1:0),
(level > 0?1:0),
level);
}
} else {
stat = nc_def_var_filter(var->container->ncid,
var->ncid,
special->_FilterID,
special->nparams,
special->_FilterParams
);
}
/* Special check for alternate way to specify _Deflate */
if(special->_FilterID == ZIP_ID) {
unsigned int level;
if(special->nparams == 0 || special->_FilterParams == NULL)
level = 9; /* default */
else
level = special->_FilterParams[0];
if(level > 9)
derror("Illegal deflate level");
else {
stat = nc_def_var_deflate(var->container->nc_id,
var->nc_id,
(special->_Shuffle == 1?1:0),
(level > 0?1:0),
level);
}
} else {
stat = nc_def_var_filter(var->container->nc_id,
var->nc_id,
special->_FilterID,
special->nparams,
special->_FilterParams
);
}
check_err(stat,__LINE__,__FILE__);
}
return stat;
}
#endif /*USE_NETCDF4*/
void
cl_netcdf(void)
genbin_close(void)
{
int stat;
stat = nc_close(rootgroup->ncid);
stat = nc_close(rootgroup->nc_id);
check_err(stat,__LINE__,__FILE__);
}
@ -284,7 +282,7 @@ cl_netcdf(void)
/*
Generate type definitions
*/
static void
static int
genbin_deftype(Symbol* tsym)
{
unsigned long i;
@ -294,109 +292,114 @@ genbin_deftype(Symbol* tsym)
switch (tsym->subclass) {
case NC_PRIM: break; /* these are already taken care of*/
case NC_OPAQUE:
stat = nc_def_opaque(tsym->container->ncid,
tsym->typ.size,
tsym->name,
&tsym->ncid);
stat = nc_def_opaque(tsym->container->nc_id,
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
break;
break;
case NC_ENUM:
{
Bytebuffer* datum;
Datalist* ecdl;
stat = nc_def_enum(tsym->container->ncid,
tsym->typ.basetype->typ.typecode,
tsym->name,
&tsym->ncid);
stat = nc_def_enum(tsym->container->nc_id,
tsym->typ.basetype->nc_id,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
datum = bbNew();
ecdl = builddatalist(1);
dlextend(ecdl); /* make room for one constant*/
ecdl->length = 1;
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
generator_reset(bin_generator,NULL);
bbClear(datum);
generate_basetype(econst->typ.basetype,&econst->typ.econst,datum,NULL,bin_generator);
stat = nc_insert_enum(tsym->container->ncid,
tsym->ncid,
econst->name,
bbContents(datum));
check_err(stat,__LINE__,__FILE__);
}
bbFree(datum);
dlfree(&ecdl);
datum = bbNew();
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
generator_reset(bin_generator,NULL);
bbClear(datum);
generate_basetype(econst->typ.basetype,econst->typ.econst,datum,NULL,bin_generator);
stat = nc_insert_enum(tsym->container->nc_id,
tsym->nc_id,
econst->name,
bbContents(datum));
check_err(stat,__LINE__,__FILE__);
}
bbFree(datum);
}
break;
case NC_VLEN:
stat = nc_def_vlen(tsym->container->ncid,
tsym->name,
tsym->typ.basetype->ncid,
&tsym->ncid);
stat = nc_def_vlen(tsym->container->nc_id,
tsym->name,
tsym->typ.basetype->nc_id,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
break;
break;
case NC_COMPOUND:
stat = nc_def_compound(tsym->container->ncid,
tsym->typ.size,
tsym->name,
&tsym->ncid);
stat = nc_def_compound(tsym->container->nc_id,
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,__FILE__);
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* efield = (Symbol*)listget(tsym->subnodes,i);
ASSERT(efield->subclass == NC_FIELD);
if(efield->typ.dimset.ndims == 0){
stat = nc_insert_compound(
tsym->container->ncid,
tsym->ncid,
efield->name,
efield->typ.offset,
efield->typ.basetype->ncid);
} else {
int j;
int dimsizes[NC_MAX_VAR_DIMS]; /* int because inside compound */
/* Generate the field dimension constants*/
for(j=0;j<efield->typ.dimset.ndims;j++) {
unsigned int size = efield->typ.dimset.dimsyms[j]->dim.declsize;
dimsizes[j] = size;
}
stat = nc_insert_array_compound(
tsym->container->ncid,
tsym->ncid,
efield->name,
efield->typ.offset,
efield->typ.basetype->ncid,
efield->typ.dimset.ndims,
dimsizes);
}
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* efield = (Symbol*)listget(tsym->subnodes,i);
ASSERT(efield->subclass == NC_FIELD);
if(efield->typ.dimset.ndims == 0){
stat = nc_insert_compound(
tsym->container->nc_id,
tsym->nc_id,
efield->name,
efield->typ.offset,
efield->typ.basetype->nc_id);
} else {
int j;
int dimsizes[NC_MAX_VAR_DIMS]; /* int because inside compound */
/* Generate the field dimension constants*/
for(j=0;j<efield->typ.dimset.ndims;j++) {
unsigned int size = efield->typ.dimset.dimsyms[j]->dim.declsize;
dimsizes[j] = size;
}
stat = nc_insert_array_compound(
tsym->container->nc_id,
tsym->nc_id,
efield->name,
efield->typ.offset,
efield->typ.basetype->nc_id,
efield->typ.dimset.ndims,
dimsizes);
}
check_err(stat,__LINE__,__FILE__);
}
break;
}
break;
default: panic("definectype: unexpected type subclass");
}
return stat;
}
#endif /*USE_NETCDF4*/
static void
static int
genbin_defineattr(Symbol* asym)
{
int stat = NC_NOERR;
Bytebuffer* databuf = bbNew();
generator_reset(bin_generator,NULL);
generate_attrdata(asym,bin_generator,(Writer)genbin_write,databuf);
if((stat = ncaux_reclaim_data(asym->container->nc_id,asym->typ.basetype->nc_id,bbContents(databuf),datalistlen(asym->data))))
goto done;
done:
bbFree(databuf);
return stat;
}
/* Following is patterned after the walk functions in semantics.c */
static void
static int
genbin_definevardata(Symbol* vsym)
{
Bytebuffer* databuf;
if(vsym->data == NULL) return;
int stat = NC_NOERR;
Bytebuffer* databuf = NULL;
if(vsym->data == NULL) goto done;
databuf = bbNew();
generator_reset(bin_generator,NULL);
generate_vardata(vsym,bin_generator,(Writer)genbin_write,databuf);
ncaux_reclaim_data(vsym->container->nc_id,vsym->typ.basetype->nc_id,bbContents(databuf),datalistlen(vsym->data));
done:
bbFree(databuf);
return stat;
}
static int
@ -404,54 +407,52 @@ genbin_write(Generator* generator, Symbol* sym, Bytebuffer* memory,
int rank, size_t* start, size_t* count)
{
if(sym->objectclass == NC_ATT)
return genbin_writeattr(generator,sym,memory,rank,start,count);
return genbin_writeattr(generator,sym,memory,rank,start,count);
else if(sym->objectclass == NC_VAR)
return genbin_writevar(generator,sym,memory,rank,start,count);
return genbin_writevar(generator,sym,memory,rank,start,count);
else
PANIC("illegal symbol for genbin_write");
PANIC("illegal symbol for genbin_write");
return NC_EINVAL;
}
static int
genbin_writevar(Generator* generator, Symbol* vsym, Bytebuffer* memory,
int rank, size_t* start,
#ifdef USE_NOFILL
size_t* indices
#else
size_t* count
#endif
)
int rank, size_t* start, size_t* count)
{
int stat = NC_NOERR;
int i;
char* data = bbContents(memory);
#ifdef USE_NOFILL
size_t count[NC_MAX_VAR_DIMS];
{ int i; for(i=0;i<rank;i++) count[i] = indices[i] - start[i];}
#endif
size_t nelems;
/* Compute total number of elements */
for(nelems=1,i=0;i<rank;i++) nelems *= count[i];
#ifdef GENDEBUG
{
int i;
fprintf(stderr,"startset = [");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)start[i]);
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)start[i]);
fprintf(stderr,"] ");
fprintf(stderr,"countset = [");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)count[i]);
fprintf(stderr,"%s%lu",(i>0?", ":""),(unsigned long)count[i]);
fprintf(stderr,"]\n");
fflush(stderr);
}
#endif
if(rank == 0) {
size_t count[1] = {1};
stat = nc_put_var1(vsym->container->ncid, vsym->ncid, count, data);
size_t count[1] = {1};
stat = nc_put_var1(vsym->container->nc_id, vsym->nc_id, count, data);
} else {
stat = nc_put_vara(vsym->container->ncid, vsym->ncid, start, count, data);
stat = nc_put_vara(vsym->container->nc_id, vsym->nc_id, start, count, data);
}
check_err(stat,__LINE__,__FILE__);
bbClear(memory);
// /* Reclaim the data */
// stat = ncaux_reclaim_data(vsym->container->nc_id, vsym->typ.basetype->nc_id, data, nelems);
// check_err(stat,__LINE__,__FILE__);
// bbClear(memory); /* reclaim top-level memory */
return stat;
}
@ -465,9 +466,9 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
int varid, grpid, typid;
Symbol* basetype = asym->typ.basetype;
grpid = asym->container->ncid;
varid = (asym->att.var == NULL?NC_GLOBAL : asym->att.var->ncid);
typid = basetype->ncid;
grpid = asym->container->nc_id;
varid = (asym->att.var == NULL?NC_GLOBAL : asym->att.var->nc_id);
typid = basetype->nc_id;
list = asym->data;
len = list->length;
@ -482,9 +483,9 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
} break;
case NC_CHAR: {
char* data = (char*)bbContents(databuf);
size_t slen = bbLength(databuf);
/* Revise length if slen == 0 */
if(slen == 0) {
size_t slen = bbLength(databuf);
/* Revise length if slen == 0 */
if(slen == 0) {
bbAppend(databuf,'\0');
/* bbAppend frees the memory pointed to by char* data,
so re-assign. See Coverity issue: 1265731.*/
@ -498,71 +499,69 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
short* data = (short*)bbContents(databuf);
stat = nc_put_att_short(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_STRING: {
const char** data;
data = (const char**)bbContents(databuf);
stat = nc_put_att_string(grpid,varid,asym->name,
bbLength(databuf)/sizeof(char*),
data);
} break;
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
#ifdef USE_NETCDF4
case NC_STRING: {
const char** data;
data = (const char**)bbContents(databuf);
stat = nc_put_att_string(grpid,varid,asym->name,
bbLength(databuf)/sizeof(char*),
data);
} break;
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
check_err2(stat,asym->lineno,__LINE__,__FILE__);
} break;
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
#endif
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
check_err2(stat,asym->lineno,__LINE__,__FILE__);
} break;
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,__FILE__);
} break;
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
} else { /* use the generic put_attribute for user defined types*/
const char* data;
data = (const char*)bbContents(databuf);
const char* data;
data = (const char*)bbContents(databuf);
stat = nc_put_att(grpid,varid,asym->name,typid,
len,(void*)data);
len,(void*)data);
check_err(stat,__LINE__,__FILE__);
#ifdef GENDEBUG
{
char out[4096];
memset(out,0x77,sizeof(out));
stat = nc_get_att(grpid,varid,asym->name,&out);
{
char out[4096];
memset(out,0x77,sizeof(out));
stat = nc_get_att(grpid,varid,asym->name,&out);
check_err(stat,__LINE__,__FILE__);
}
}
#endif
}
return stat;

13
ncgen/genc.c
View File

@ -36,11 +36,12 @@ static void genc_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
* Generate C code for creating netCDF from in-memory structure.
*/
void
gen_ncc(const char *filename)
genc_netcdf(void)
{
int idim, ivar, iatt, maxdims;
int ndims, nvars, natts, ngatts;
char* cmode_string;
const char *filename = rootgroup->file.filename;
#ifdef USE_NETCDF4
int igrp,ityp, ngrps, ntyps;
@ -95,7 +96,7 @@ gen_ncc(const char *filename)
for(ivar=0;ivar<nvars;ivar++) {
Bytebuffer* tmp = bbNew();
Symbol* var = (Symbol*)listget(vardefs,ivar);
Specialdata* special = &var->var.special;
Specialdata* special = var->var.special;
if(special->flags & _CHUNKSIZES_FLAG) {
int i;
size_t* chunks = special->_ChunkSizes;
@ -448,7 +449,7 @@ genc_defineglobalspecials(void)
static void
genc_definespecialattributes(Symbol* vsym)
{
Specialdata* special = &vsym->var.special;
Specialdata* special = vsym->var.special;
if(usingclassic) return;
if(special->flags & _STORAGE_FLAG) {
int storage = special->_Storage;
@ -554,7 +555,7 @@ genc_definespecialattributes(Symbol* vsym)
#endif /*USE_NETCDF4*/
void
cl_c(void)
genc_close(void)
{
bbprintf0(stmt,"%sstat = nc_close(%s);\n",indented(1),groupncid(rootgroup));
codedump(stmt);
@ -771,7 +772,7 @@ definectype(Symbol* tsym)
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
Bytebuffer* econststring = bbNew();
ASSERT(econst->subclass == NC_ECONST);
c_generator->constant(c_generator,tsym,&econst->typ.econst,econststring);
c_generator->constant(c_generator,tsym,econst->typ.econst,econststring);
bbNull(econststring);
/* Enum constants must be converted to a fully qualified name */
bbprintf0(stmt,"#define %s ((%s)%s)\n",
@ -869,7 +870,7 @@ genc_deftype(Symbol* tsym)
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
Bytebuffer* econststring = bbNew();
ASSERT(econst->subclass == NC_ECONST);
c_generator->constant(c_generator,tsym,&econst->typ.econst,econststring);
c_generator->constant(c_generator,tsym,econst->typ.econst,econststring);
bbNull(econststring);
bbprintf0(stmt,"%seconst = %s;\n",
indented(1),bbContents(econststring));

56
ncgen/genchar.c
View File

@ -15,9 +15,6 @@
static size_t gen_charconstant(NCConstant*, Bytebuffer*, int fillchar);
static int getfillchar(Datalist* fillsrc);
static void gen_leafchararray(Dimset*,int,Datalist*,Bytebuffer*, int);
#if 0
static void gen_chararrayr(Dimset*,int,int,Bytebuffer*,Datalist*,int,int,int);
#endif
static NCConstant* makeconst(int lineno, int len, char* str);
/*
@ -128,11 +125,11 @@ gen_chararrayr(Dimset* dimset, int dimindex,
void
gen_charattr(Datalist* data, Bytebuffer* databuf)
{
gen_charvlen(data,databuf);
gen_charseq(data,databuf);
}
void
gen_charvlen(Datalist* data, Bytebuffer* databuf)
gen_charseq(Datalist* data, Bytebuffer* databuf)
{
int i;
NCConstant* c;
@ -168,7 +165,8 @@ gen_charconstant(NCConstant* con, Bytebuffer* databuf, int fillchar)
break;
case NC_STRING:
constsize = con->value.stringv.len;
bbAppendn(databuf,con->value.stringv.stringv,
if(constsize > 0)
bbAppendn(databuf,con->value.stringv.stringv,
con->value.stringv.len);
bbNull(databuf);
break;
@ -187,7 +185,7 @@ getfillchar(Datalist* fillsrc)
/* Determine the fill char */
int fillchar = 0;
if(fillsrc != NULL && fillsrc->length > 0) {
NCConstant* ccon = fillsrc->data;
NCConstant* ccon = fillsrc->data[0];
if(ccon->nctype == NC_CHAR) {
fillchar = ccon->value.charv;
} else if(ccon->nctype == NC_STRING) {
@ -242,7 +240,9 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
concatenated with any trailing or leading string (with double quotes).
*/
/* Rebuild the datalist to merge 'x' constants */
/* Rebuild the datalist to merge '0x' constants;
WARNING: this is tricky.
*/
{
int i,cccount = 0;
/* Do initial walk */
@ -253,36 +253,50 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
}
}
if(cccount > 1) {
char* accum = (char*)ecalloc(cccount+1);
int len = 0;
Bytebuffer* accum = bbNew();
int len = 0; /* >0 implies doing accum */
Datalist* newlist = builddatalist(datalistlen(data));
int lineno = 0;
NCConstant* con;
/* We are going to construct a single string constant for each
contiguous sequence of single char values.
Assume that the constants are all primitive types */
for(i=0;i<datalistlen(data);i++) {
con = datalistith(data,i);
if(consttype(con) == NC_CHAR || consttype(con) == NC_BYTE) {
if(len == 0)
if(len == 0) { /* Start an accumulation */
lineno = constline(con);
accum[len] = con->value.charv;
bbClear(accum);
}
bbAppend(accum,con->value.charv);
len++;
/* Discard this constant */
reclaimconstant(con);
} else {
if(len > 0) {
con = makeconst(lineno,len,accum);
if(len > 0) { /* End the accumulation */
bbNull(accum);
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
}
dlappend(newlist,con);
dlappend(newlist,con);
} else
dlappend(newlist,con);
}
}
/* deal with any unclosed strings */
if(len > 0) {
con = makeconst(lineno,len,accum);
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
dlappend(newlist,con);
}
free(accum);
data = newlist;
bbFree(accum);
/* Move the newlist sequence of constants into the old list */
efree(data->data);
data->data = newlist->data;
data->length = newlist->length;
data->alloc = newlist->alloc;
efree(newlist);
}
}
@ -323,7 +337,7 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
if(bbLength(charbuf) == 0 && expectedsize == 1) {
/* this is okay */
} else if(bbLength(charbuf) > expectedsize) {
semwarn(data->data[0].lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(charbuf));
semwarn(data->data[0]->lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(charbuf));
} else {
size_t bufsize = bbLength(charbuf);
/* Pad to size dimproduct size */
@ -338,7 +352,7 @@ gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
static NCConstant*
makeconst(int lineno, int len, char* str)
{
NCConstant* con = (NCConstant*)ecalloc(sizeof(NCConstant));
NCConstant* con = nullconst();
con->nctype = NC_STRING;
con->lineno = lineno;
con->filled = 0;

7
ncgen/gencml.c
View File

@ -55,10 +55,11 @@ Bytebuffer* xcode;
* Generate C code for creating netCDF from in-memory structure.
*/
void
gen_nccml(const char *filename)
gencml_netcdf(void)
{
xcode = bbNew();
bbSetalloc(xcode,C_MAX_STMT);
const char *filename = rootgroup->file.filename;
/* Dump XML header */
xline("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
@ -75,7 +76,7 @@ gen_nccml(const char *filename)
}
void
cl_cml(void)
gencml_close(void)
{
xline("</netcdf>");
xflush();
@ -285,7 +286,7 @@ definextype(Symbol* tsym)
ASSERT(econst->subclass == NC_ECONST);
nprintf(stmt,sizeof(stmt),"<enumconst name=\"%s\" value=\"%s\" />",
xname(econst),
xconst(&econst->typ.econst));
xconst(econst->typ.econst));
xline(stmt);
}
xline("</enum>");

48
ncgen/generate.c
View File

@ -7,6 +7,7 @@
#include "nc_iter.h"
#include "odom.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
/**************************************************/
/* Code for generating data lists*/
@ -81,11 +82,6 @@ generate_vardata(Symbol* vsym, Generator* generator, Writer writer, Bytebuffer*
int rank = dimset->ndims;
Symbol* basetype = vsym->typ.basetype;
Datalist* filler = getfiller(vsym);
#if 0
const size_t* start;
const size_t* count;
Odometer* odom;
#endif
if(vsym->data == NULL) return;
@ -136,7 +132,6 @@ generate_array(Symbol* vsym,
int rank = dimset->ndims;
Symbol* basetype = vsym->typ.basetype;
nc_type typecode = basetype->typ.typecode;
Odometer* odom = NULL;
nciter_t iter;
int firstunlim = findunlimited(dimset,1);
int nunlim = countunlimited(dimset);
@ -147,23 +142,24 @@ generate_array(Symbol* vsym,
if(isnc3unlim) {
/* Handle NC_CHAR case separately */
if(typecode == NC_CHAR) {
Odometer* odom = newodometer(dimset,NULL,NULL);
Bytebuffer* charbuf = bbNew();
gen_chararray(dimset,0,vsym->data,charbuf,filler);
generator->charconstant(generator,vsym,code,charbuf);
/* Create an odometer to get the dimension info */
odom = newodometer(dimset,NULL,NULL);
writer(generator,vsym,code,odom->rank,odom->start,odom->count);
#if 0
writer(generator,vsym,code,odom->rank,0,bbLength(charbuf));
#endif
bbFree(charbuf);
odometerfree(odom);
} else { /* typecode != NC_CHAR */
Odometer* odom = newodometer(dimset,NULL,NULL);
/* Case: dim 1..rank-1 are not unlimited, dim 0 might be */
size_t offset = 0; /* where are we in the data list */
size_t nelems = 0; /* # of data list items to generate */
/* Create an iterator and odometer and just walk the datalist */
nc_get_iter(vsym,nciterbuffersize,&iter);
odom = newodometer(dimset,NULL,NULL);
for(;;offset+=nelems) {
int i,uid;
nelems=nc_next_iter(&iter,odometerstartvector(odom),odometercountvector(odom));
@ -179,11 +175,12 @@ generate_array(Symbol* vsym,
generator->listend(generator,vsym,NULL,LISTDATA,uid,i,code);
writer(generator,vsym,code,rank,odom->start,odom->count);
}
odometerfree(odom);
}
} else { /* Hard case: multiple unlimited dimensions or unlim in dim > 0*/
Odometer* odom = newodometer(dimset,NULL,NULL);
/* Setup iterator and odometer */
nc_get_iter(vsym,NC_MAX_UINT,&iter); /* effectively infinite */
odom = newodometer(dimset,NULL,NULL);
for(;;) {/* iterate in nelem chunks */
/* get nelems count and modify odometer */
size_t nelems=nc_next_iter(&iter,odom->start,odom->count);
@ -195,9 +192,8 @@ generate_array(Symbol* vsym,
);
writer(generator,vsym,code,odom->rank,odom->start,odom->count);
}
}
if(odom != NULL)
odometerfree(odom);
}
}
/**
@ -286,7 +282,7 @@ generate_arrayr(Symbol* vsym,
return;
}
/* Generate an instance of the basetype */
/* Generate an instance of the basetype using the value of con*/
void
generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist* filler, Generator* generator)
{
@ -308,8 +304,8 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
int i,uid, nfields, dllen;
if(con == NULL || isfillconst(con)) {
Datalist* fill = (filler==NULL?getfiller(tsym):filler);
ASSERT(fill->length == 1);
con = &fill->data[0];
ASSERT(fill->length == 1);
con = fill->data[0];
if(!islistconst(con)) {
if(con)
semerror(con->lineno,"Compound data fill value is not enclosed in {..}");
@ -352,7 +348,7 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
if(con == NULL || isfillconst(con)) {
Datalist* fill = (filler==NULL?getfiller(tsym):filler);
ASSERT(fill->length == 1);
con = &fill->data[0];
con = fill->data[0];
if(con->nctype != NC_COMPOUND) {
semerror(con->lineno,"Vlen data fill value is not enclosed in {..}");
}
@ -365,7 +361,7 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
/* generate the nc_vlen_t instance*/
vlenbuf = bbNew();
if(tsym->typ.basetype->typ.typecode == NC_CHAR) {
gen_charvlen(data,vlenbuf);
gen_charseq(data,vlenbuf);
generator->vlenstring(generator,tsym,vlenbuf,&uid,&count);
} else {
generator->listbegin(generator,tsym,NULL,LISTVLEN,data->length,codebuf,&uid);
@ -377,7 +373,7 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
}
generator->listend(generator,tsym,NULL,LISTVLEN,uid,count,codebuf,(void*)vlenbuf);
}
generator->vlendecl(generator,tsym,codebuf,uid,count,vlenbuf);
generator->vlendecl(generator,tsym,codebuf,uid,count,vlenbuf); /* Will extract contents of vlenbuf */
bbFree(vlenbuf);
} break;
@ -464,7 +460,7 @@ static void
generate_primdata(Symbol* basetype, NCConstant* prim, Bytebuffer* codebuf,
Datalist* filler, Generator* generator)
{
NCConstant target;
NCConstant* target;
int match;
if(prim == NULL || isfillconst(prim)) {
@ -521,24 +517,26 @@ generate_primdata(Symbol* basetype, NCConstant* prim, Bytebuffer* codebuf,
basetype->name);
}
target.nctype = basetype->typ.typecode;
target = nullconst();
target->nctype = basetype->typ.typecode;
if(target.nctype != NC_ECONST) {
convert1(prim,&target);
if(target->nctype != NC_ECONST) {
convert1(prim,target);
}
switch (target.nctype) {
switch (target->nctype) {
case NC_ECONST:
if(basetype->subclass != NC_ENUM) {
semerror(constline(prim),"Conversion to enum not supported (yet)");
} break;
case NC_OPAQUE:
normalizeopaquelength(&target,basetype->typ.size);
normalizeopaquelength(target,basetype->typ.size);
break;
default:
break;
}
generator->constant(generator,basetype,&target,codebuf);
generator->constant(generator,basetype,target,codebuf);
reclaimconstant(target);
target = NULL;
return;
}

62
ncgen/generr.c
View File

@ -9,18 +9,8 @@
int error_count;
#ifndef NO_STDARG
#if 0
#define vastart(argv,fmt) va_start(argv,fmt)
#else
#define vastart(argv,fmt) va_start(argv)
#endif
/*
* Invalidate variable list.
*/
#ifndef NO_STDARG
#define vaend(argv,fmt) va_end(argv)
#else
#define vaend(argv,fmt) va_end(argv)
#endif
@ -28,14 +18,8 @@ int error_count;
* For logging error conditions.
* Designed to be called by other vararg procedures
*/
#ifndef NO_STDARG
void
vderror(const char *fmt, va_list argv)
#else
/* Technically illegal; va_alist should be only arg */
void
vderror(fmt,va_alist) const char* fmt; va_dcl
#endif
{
(void) vdwarn(fmt,argv);
error_count++;
@ -45,75 +29,49 @@ vderror(fmt,va_alist) const char* fmt; va_dcl
* For logging error conditions.
* Designed to be called by other vararg procedures
*/
#ifndef NO_STDARG
void
vdwarn(const char *fmt, va_list argv)
#else
/* Technically illegal; va_alist should be only arg */
void
vdwarn(fmt,va_alist) const char* fmt; va_dcl
#endif
{
(void) vfprintf(stderr,fmt,argv) ;
(void) fputc('\n',stderr) ;
(void) fflush(stderr); /* to ensure log files are current */
}
#ifndef NO_STDARG
void
derror(const char *fmt, ...)
#else
void
derror(fmt,va_alist) const char* fmt; va_dcl
#endif
{
va_list argv;
vastart(argv,fmt);
va_start(argv,fmt);
vderror(fmt,argv);
}
/* Report version errors */
#ifndef NO_STDARG
void
verror(const char *fmt, ...)
#else
void
verror(fmt,va_alist) const char* fmt; va_dcl
#endif
{
char newfmt[2048];
va_list argv;
vastart(argv,fmt);
va_start(argv,fmt);
strcpy(newfmt,"netCDF classic: not supported: ");
strncat(newfmt,fmt,2000);
vderror(newfmt,argv);
vaend(argv,fmt);
va_end(argv);
}
#ifndef NO_STDARG
void
semwarn(const int lno, const char *fmt, ...)
#else
void
semwarn(lno,fmt,va_alist) const int lno; const char* fmt; va_dcl
#endif
{
va_list argv;
vastart(argv,fmt);
va_start(argv,fmt);
(void)fprintf(stderr,"%s: %s line %d: ", progname, cdlname, lno);
vdwarn(fmt,argv);
}
#ifndef NO_STDARG
void
semerror(const int lno, const char *fmt, ...)
#else
void
semerror(lno,fmt,va_alist) const int lno; const char* fmt; va_dcl
#endif
{
va_list argv;
vastart(argv,fmt);
va_start(argv,fmt);
(void)fprintf(stderr,"%s: %s line %d: ", progname, cdlname, lno);
vderror(fmt,argv);
finalize_netcdf(1); /* immediately fatal */
@ -155,18 +113,12 @@ getmarkcdf5(void)
/* Provide a version of snprintf that panics if*/
/* the buffer is overrun*/
#ifndef NO_STDARG
void
nprintf(char* buffer, size_t size, const char *fmt, ...)
#else
void
nprintf(buffer,size,fmt)
char* buffer; size_t size; const char* fmt; va_dcl
#endif
{
long written;
va_list args;
vastart(args,fmt);
va_start(args,fmt);
written = vsnprintf(buffer,size,fmt,args);
if(written < 0 || written >= size) {
PANIC("snprintf failure");

25
ncgen/generr.h
View File

@ -8,22 +8,15 @@
#ifndef GENERR_H
#define GENERR_H
#include <stdarg.h>
extern int error_count;
#ifndef NO_STDARG
#if 0
#define vastart(argv,fmt) va_start(argv,fmt)
#else
#define vastart(argv,fmt) va_start(argv)
#endif
#ifndef NO_STDARG
#define vaend(argv,fmt) va_end(argv)
#else
#define vaend(argv,fmt) va_end(argv)
#endif
#ifndef NO_STDARG
#include <stdarg.h>
extern void vderror(const char *fmt, va_list argv);
extern void vdwarn(const char *fmt, va_list argv);
extern void derror(const char *fmt, ...);
@ -31,17 +24,5 @@ extern int panic(const char* fmt, ...);
extern void nprintf(char* buffer, size_t size, const char *fmt, ...);
extern void semerror(const int, const char *fmt, ...);
extern void semwarn(const int, const char *fmt, ...);
#else
#include <varargs.h>
/* Technically illegal; va_alist should be only arg */
extern void vderror(fmt,va_alist) const char* fmt; va_dcl;
extern void vdwarn(fmt,va_alist) const char* fmt; va_dcl;
extern void derror(fmt,va_alist) const char* fmt; va_dcl;
extern void panic(fmt,va_alist) const char* fmt; va_dcl;
extern void nprintf(buffer,size,fmt)
char* buffer; size_t size; const char* fmt; va_dcl;
extern void semerror(lno,fmt,va_alist) const int lno; const char* fmt; va_dcl;
extern void semwarnlno,fmt,va_alist) const int lno; const char* fmt; va_dcl;
#endif
#endif /*GENERR_H*/

5
ncgen/genf77.c
View File

@ -45,11 +45,12 @@ static void genf77_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*)
* Generate code for creating netCDF from in-memory structure.
*/
void
gen_ncf77(const char *filename)
genf77_netcdf(void)
{
int idim, ivar, iatt;
int ndims, nvars, natts, ngatts;
char* cmode_string;
const char *filename = rootgroup->file.filename;
ndims = listlength(dimdefs);
nvars = listlength(vardefs);
@ -389,7 +390,7 @@ gen_ncf77(const char *filename)
}
void
cl_f77(void)
genf77_close(void)
{
/* already done above */
}

5
ncgen/genj.c
View File

@ -38,10 +38,11 @@ static void genj_writeattr(Generator*,Symbol*,Bytebuffer*,int,size_t*,size_t*);
* Generate code for creating netCDF from in-memory structure.
*/
void
gen_ncjava(const char *filename)
genjava_netcdf(void)
{
int idim, ivar, iatt, maxdims;
int ndims, nvars, natts, ngatts;
const char *filename = rootgroup->file.filename;
ndims = listlength(dimdefs);
nvars = listlength(vardefs);
@ -201,7 +202,7 @@ gen_ncjava(const char *filename)
}
void
cl_java(void)
genjava_close(void)
{
codelined(1,"ncfile.close();");
codeline("");

2
ncgen/genjjni.c
View File

@ -741,7 +741,7 @@ definejtype(Symbol* tsym)
nprintf(stmt,sizeof(stmt),"%s%s(%s)%s",
indented(1),
jtypename(econst),
jconst(&econst->typ.econst),
jconst(econst->typ.econst),
(i == (nconst - 1)?";":","));
jlined(1,stmt);
}

53
ncgen/genlib.c
View File

@ -8,59 +8,24 @@
/* invoke netcdf calls (or generate C or Fortran code) to create netcdf
* from in-memory structure.
The output file name is chosen by using the following in priority order:
1. -o flag name
2. command line input file with .cdl changed to .nc
3. dataset name as specified in netcdf <name> {...}
*/
void
define_netcdf(void)
{
char filename[2049];
/* Rule for specifying the dataset name:
1. use -o name
2. use the datasetname from the .cdl file
3. use input cdl file name (with .cdl removed)
It would be better if there was some way
to specify the datasetname independently of the
file name, but oh well.
*/
if(netcdf_name) { /* -o flag name */
strncpy(filename,netcdf_name,2048);
} else { /* construct a usable output file name */
if (cdlname != NULL && strcmp(cdlname,"-") != 0) {/* cmd line name */
char* p;
strncpy(filename,cdlname,2048);
/* remove any suffix and prefix*/
p = strrchr(filename,'.');
if(p != NULL) {*p= '\0';}
p = strrchr(filename,'/');
if(p != NULL) {memmove(filename,(p+1),(2048-strlen(cdlname)));}
} else {/* construct name from dataset name */
strncpy(filename,datasetname,2048); /* Reserve space for extension, terminating '\0' */
}
/* Append the proper extension */
strncat(filename,binary_ext,2048-(strlen(filename) + strlen(binary_ext)));
}
/* Execute exactly one of these */
#ifdef ENABLE_C
if (l_flag == L_C) gen_ncc(filename); else /* create C code to create netcdf */
if (l_flag == L_C) genc_netcdf(); else /* create C code to create netcdf */
#endif
#ifdef ENABLE_F77
if (l_flag == L_F77) gen_ncf77(filename); else /* create Fortran code */
if (l_flag == L_F77) genf77_netcdf(); else /* create Fortran code */
#endif
#ifdef ENABLE_JAVA
if(l_flag == L_JAVA) {
gen_ncjava(filename);
} else
if(l_flag == L_JAVA) genjava_netcdf(); else
#endif
/* Binary is the default */
#ifdef ENABLE_BINARY
gen_netcdf(filename); /* create netcdf */
genbin_netcdf(); /* create netcdf */
#else
derror("No language specified");
#endif
@ -72,16 +37,16 @@ void
close_netcdf(void)
{
#ifdef ENABLE_C
if (l_flag == L_C) cl_c(); else /* create C code to close netcdf */
if (l_flag == L_C) genc_close(); else /* create C code to close netcdf */
#endif
#ifdef ENABLE_F77
if (l_flag == L_F77) cl_f77(); else
if (l_flag == L_F77) genf77_close(); else
#endif
#ifdef ENABLE_JAVA
if (l_flag == L_JAVA) cl_java(); else
if (l_flag == L_JAVA) genjava_close(); else
#endif
#ifdef ENABLE_BINARY
if (l_flag == L_BINARY) cl_netcdf();
if (l_flag == L_BINARY) genbin_close();
#endif
}
@ -110,7 +75,7 @@ topfqn(Symbol* sym)
/* Recursively compute parent fqn */
if(parent == NULL) { /* implies this is the rootgroup */
assert(sym->grp.is_root);
sym->fqn = strdup("");
sym->fqn = estrdup("");
return;
} else if(parent->fqn == NULL) {
topfqn(parent);

26
ncgen/genlib.h
View File

@ -59,7 +59,7 @@ extern void nestedfqn(Symbol* sym);
extern void attfqn(Symbol* sym);
/* from: escapes.c */
extern int unescape(char*, const char*, int, int);
extern int unescape(const char*, int, int, char**);
extern int unescapeoct(const char* s);
extern int unescapehex(const char* s);
extern char* cescapifychar(unsigned int c, int quote);
@ -73,9 +73,10 @@ extern char* xescapify(char* s0, int quote, size_t len);
extern char* jescapify(char* s0, int quote, size_t len);
extern char* jescapifyname(char* s0);
extern char* fqnescape(const char* s);
extern char* esc_strchr(char* s, int c, int octhex);
/* from: getfill.c */
extern void nc_getfill(NCConstant*);
extern void nc_getfill(NCConstant*,Symbol*);
extern char* nc_dfaltfillname(nc_type);
extern struct Datalist* getfiller(Symbol*); /* symbol isa variable|type */
@ -112,15 +113,18 @@ extern char* indented(int n);
#ifdef ENABLE_BINARY
/* from: genbin.c */
extern Generator* bin_generator;
extern void gen_netcdf(const char *filename);
extern void cl_netcdf(void);
extern void genbin_netcdf(void);
extern void genbin_close(void);
/* from: bindata.c */
extern int binary_generate_data(Datalist* data, Symbol* tsym, Datalist* fillvalue, Bytebuffer* databuf);
extern int binary_reclaim_data(Symbol* tsym, void* memory, size_t count);
#endif
#ifdef ENABLE_C
/* from: genc.c */
extern Generator* c_generator;
extern void gen_ncc(const char *filename);
extern void cl_c(void);
extern void genc_netcdf(void);
extern void genc_close(void);
extern const char* ctypename(Symbol*);
extern const char* nctype(nc_type type);
extern const char* ncctype(nc_type type);
@ -132,8 +136,8 @@ extern const char* cname(Symbol* sym);
#ifdef ENABLE_F77
/* from: genf77.c */
extern Generator* f77_generator;
extern void gen_ncf77(const char *filename);
extern void cl_f77(void);
extern void genf77_netcdf(void);
extern void genf77_close(void);
extern const char* f77name(Symbol*);
extern const char* f77typename(Symbol*);
#endif
@ -141,8 +145,8 @@ extern const char* f77typename(Symbol*);
#ifdef ENABLE_JAVA
/* from: genj.c */
extern Generator* j_generator;
extern void gen_ncjava(const char *filename);
extern void cl_java(void);
extern void genjava_netcdf(void);
extern void genjava_close(void);
extern void jpartial(char*);
extern void jline(char*);
extern void jlined(int,char*);
@ -163,7 +167,7 @@ extern GlobalSpecialData globalspecials;
/* Global data */
extern Symbol* symlist; /* all symbol objects created */
extern List* symlist; /* all symbol objects created */
extern Symbol* rootgroup;
/* Track definitions of dims, types, attributes, and vars*/

152
ncgen/getfill.c
View File

@ -15,70 +15,96 @@ static void fill(Symbol* tsym, Datalist*);
static void fillarray(Symbol* tsym, Dimset* dimset, int index, Datalist*);
static void filllist(Symbol* tvsym, Datalist* dl);
/* Construct a Datalist representing a complete fill value*/
/* for a specified variable */
/* Cache if needed later*/
/* Construct a Datalist representing a complete fill value.
for a specified variable or type. Cache if needed later.
The rules are as follows
1. If the tvsym argument is a variable and it has a _FillValue
attribute, then use that value.
2. If the tvsym argunment is a variable with no specified
_FillValue, then create one based on the variable's basetype.
3. If the tvsym argument is a type, then if it already has a fill value,
then use that, otherwise build and cache a fillvalue appropriate
for that type.
*/
Datalist*
getfiller(Symbol* tvsym)
{
Datalist* filler = NULL;
Symbol* tsym = tvsym;
ASSERT(tvsym->objectclass == NC_VAR || tvsym->objectclass == NC_TYPE);
if(tvsym->objectclass == NC_VAR) {
tsym = tvsym->typ.basetype;
filler = tvsym->var.special._Fillvalue; /* get cached value (if any)*/
}
if(filler == NULL || tvsym->objectclass == NC_TYPE) {
filler = tvsym->typ._Fillvalue; /* get cached value (if any)*/
}
if(filler == NULL) { /* need to compute it*/
filler = builddatalist(0);
fill(tsym,filler);
if(tvsym->var.special->_Fillvalue != NULL) {
/* We have a _FillValue Attribute specified */
filler = tvsym->var.special->_Fillvalue;
} else { /* otherwise create a fillvalue for the base type */
filler = getfiller(tvsym->typ.basetype);
}
} else { /* (tvsym->objectclass == NC_TYPE) */
if(tvsym->typ._Fillvalue == NULL) {
/* create and cache */
filler = builddatalist(0);
fill(tvsym,filler);
tvsym->typ._Fillvalue = filler;
}
filler = tvsym->typ._Fillvalue;
}
#ifdef GENDEBUG2
dumpdatalist(filler,"getfiller");
#endif
if(tvsym->objectclass == NC_VAR) {
tvsym->var.special._Fillvalue = filler;
} else if(tvsym->objectclass == NC_TYPE) {
tvsym->typ._Fillvalue = filler; /* cache value*/
}
return filler;
return filler;
}
static void
fill(Symbol* tsym, Datalist* filler)
{
unsigned long i;
NCConstant con = nullconstant;
Datalist* sublist;
NCConstant* con = NULL;
ASSERT(tsym->objectclass == NC_TYPE);
switch (tsym->subclass) {
case NC_ENUM: case NC_OPAQUE: case NC_PRIM:
con.nctype = tsym->typ.typecode;
nc_getfill(&con);
con = nullconst();
con->nctype = tsym->typ.typecode;
nc_getfill(con,tsym);
dlappend(filler,con);
break;
case NC_COMPOUND:
sublist = builddatalist(listlength(tsym->subnodes));
case NC_COMPOUND: {
/* Given a compound type, the fill will be a sublist
consisting itself of N constants, where N is the number of fields.
Non-array fields will be direct, array fields will be sublists.
*/
Datalist* cmpdlist = builddatalist(listlength(tsym->subnodes)); /* list of field constants */
for(i=0;i<listlength(tsym->subnodes);i++) {
NCConstant* fieldinstance = NULL;
Symbol* field = (Symbol*)listget(tsym->subnodes,i);
if(field->typ.dimset.ndims > 0) {
fillarray(field->typ.basetype,&field->typ.dimset,0,filler);
/* Build a sublist for this field */
Datalist* arraydata = builddatalist(0);
fillarray(field->typ.basetype,&field->typ.dimset,0,arraydata);
/* Convert to a compound constant */
fieldinstance = list2const(arraydata);
dlappend(cmpdlist,fieldinstance);
fieldinstance = NULL;
} else
filllist(field->typ.basetype,sublist);
/* Append directly to cmpdlist*/
filllist(field->typ.basetype,cmpdlist);
}
con = builddatasublist(sublist);
break;
case NC_VLEN:
sublist = builddatalist(0);
filllist(tsym->typ.basetype,sublist); /* generate a single instance*/
con = builddatasublist(sublist);
break;
/* Add compound instance to the filler list */
con = list2const(cmpdlist);
dlappend(filler,con);
con = NULL;
} break;
case NC_VLEN: {
Datalist* vlensublist = NULL;
/* The vlist will have a sublist hanging off of fille, with
a single compound constant in filler */
vlensublist = builddatalist(0);
filllist(tsym->typ.basetype,vlensublist); /* generate a single instance*/
con = builddatasublist(vlensublist);
dlappend(filler,con);
} break;
default: PANIC1("fill: unexpected subclass %d",tsym->subclass);
}
dlappend(filler,&con);
}
static void
@ -86,14 +112,15 @@ filllist(Symbol* tsym, Datalist* dl)
{
int i;
Datalist* sublist;
NCConstant con = nullconstant;
NCConstant* con = NULL;
ASSERT(tsym->objectclass == NC_TYPE);
switch (tsym->subclass) {
case NC_ENUM: case NC_OPAQUE: case NC_PRIM:
con.nctype = tsym->typ.typecode;
nc_getfill(&con);
dlappend(dl,&con);
con = nullconst();
con->nctype = tsym->typ.typecode;
nc_getfill(con,tsym);
dlappend(dl,con);
break;
case NC_COMPOUND:
sublist = builddatalist(listlength(tsym->subnodes));
@ -102,18 +129,21 @@ filllist(Symbol* tsym, Datalist* dl)
filllist(field->typ.basetype,sublist);
}
con = builddatasublist(sublist);
dlappend(dl,&con);
dlappend(dl,con);
break;
case NC_VLEN:
sublist = builddatalist(0);
filllist(tsym->typ.basetype,sublist); /* generate a single instance*/
con = builddatasublist(sublist);
dlappend(dl,&con);
dlappend(dl,con);
break;
default: PANIC1("fill: unexpected subclass %d",tsym->subclass);
}
}
/* Create an array of fill values of basetype given the dimset. This
is recursive over the dimensions as specified by the index argumen
*/
static void
fillarray(Symbol* basetype, Dimset* dimset, int index, Datalist* arraylist)
{
@ -122,23 +152,20 @@ fillarray(Symbol* basetype, Dimset* dimset, int index, Datalist* arraylist)
unsigned int size = dim->dim.declsize;
int isunlimited = (size == 0);
int lastdim = (index == (dimset->ndims - 1));
int firstdim = (index == 0);
Datalist* sublist;
sublist = (firstdim?builddatalist(0):arraylist);
if(isunlimited) {
/* do a single entry to satisfy*/
if(lastdim) {
filllist(basetype,sublist);
filllist(basetype,arraylist);
} else {
fillarray(basetype->typ.basetype,dimset,index+1,sublist);
fillarray(basetype->typ.basetype,dimset,index+1,arraylist);
}
} else { /* bounded*/
if(lastdim) {
for(i=0;i<size;i++) filllist(basetype,sublist);
for(i=0;i<size;i++) filllist(basetype,arraylist);
} else {
for(i=0;i<size;i++) {
fillarray(basetype->typ.basetype,dimset,index+1,sublist);
fillarray(basetype->typ.basetype,dimset,index+1,arraylist);
}
}
}
@ -149,7 +176,7 @@ fillarray(Symbol* basetype, Dimset* dimset, int index, Datalist* arraylist)
* that type.
*/
void
nc_getfill(NCConstant* value)
nc_getfill(NCConstant* value, Symbol* tsym)
{
switch(value->nctype) {
case NC_CHAR: value->value.charv = NC_FILL_CHAR; break;
@ -174,6 +201,33 @@ nc_getfill(NCConstant* value)
value->value.opaquev.len = 2;
value->value.opaquev.stringv = nulldup("00");
break;
case NC_ENUM: {
Symbol* econst;
NCConstant* eccon;
if(tsym == NULL)
derror("nc_getfill: no enum type specified");
/* Get the first value */
if(tsym->subclass != NC_ENUM)
derror("nc_getfill: expected enum type");
if(listlength(tsym->subnodes) == 0)
derror("nc_getfill: empty enum type");
econst = listget(tsym->subnodes,0);
eccon = econst->typ.econst;
switch (eccon->nctype) {
case NC_BYTE:
case NC_SHORT:
case NC_INT:
case NC_UBYTE:
case NC_USHORT:
case NC_UINT:
case NC_INT64:
case NC_UINT64:
value->value = eccon->value;
break;
default:
derror("nc_getfill: illegal enum basetype");
}
} break;
default:
derror("nc_getfill: unrecognized type: %d",value->nctype);
}

5
ncgen/includes.h
View File

@ -10,8 +10,6 @@
#ifndef NCGEN_INCLUDES_H
#define NCGEN_INCLUDES_H
#undef USE_NOFILL
#include "config.h"
#include <stdlib.h>
@ -57,4 +55,7 @@ extern int specialconstants;
#undef ITERBUG
#undef CHARBUG
#undef nullfree
#define nullfree(x) {if((x)) free(x);}
#endif /* NCGEN_INCLUDES_H */

2
ncgen/jdata.c
View File

@ -19,7 +19,7 @@ j_charconstant(Generator* generator, Symbol* sym, Bytebuffer* codebuf, ...)
/* Just transfer charbuf to codebuf */
Bytebuffer* charbuf;
va_list ap;
vastart(ap,codebuf);
va_start(ap,codebuf);
charbuf = va_arg(ap, Bytebuffer*);
va_end(ap);
bbNull(charbuf);

4
ncgen/jdatajni.c
View File

@ -331,8 +331,8 @@ genjjni_vlendata(List* vlenconstants, Bytebuffer* databuf)
vlensrc = datalist2src(cmpd->value.compoundv);
bbClear(vlenbuf);
if(typecode == NC_CHAR) {
/* Collect the char vlen in a separate buffer */
gen_charvlen(vlensrc,vlenbuf);
/* Collect the char sequence in a separate buffer */
gen_charseq(vlensrc,vlenbuf);
count = bbLength(vlenbuf);
/* Add to the existing data buf as a single constant */
jquotestring(vlenbuf,'"');

15
ncgen/list.c
View File

@ -3,8 +3,7 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "list.h"
#include "includes.h"
int listnull(void* e) {return e == NULL;}
@ -27,7 +26,7 @@ List* listnew(void)
initialized = 1;
}
*/
l = (List*)malloc(sizeof(List));
l = (List*)emalloc(sizeof(List));
if(l) {
l->alloc=0;
l->length=0;
@ -41,8 +40,8 @@ listfree(List* l)
{
if(l) {
l->alloc = 0;
if(l->content != NULL) {free(l->content); l->content = NULL;}
free(l);
if(l->content != NULL) {efree(l->content); l->content = NULL;}
efree(l);
}
return TRUE;
}
@ -54,11 +53,11 @@ listsetalloc(List* l, unsigned long sz)
if(l == NULL) return FALSE;
if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);}
if(l->alloc >= sz) {return TRUE;}
newcontent=(void**)calloc(sz,sizeof(void*));
newcontent=(void**)ecalloc(sz*sizeof(void*));
if(newcontent != NULL && l->alloc > 0 && l->length > 0 && l->content != NULL) {
memcpy((void*)newcontent,(void*)l->content,sizeof(void*)*l->length);
}
if(l->content != NULL) free(l->content);
if(l->content != NULL) efree(l->content);
l->content=newcontent;
l->alloc=sz;
return TRUE;
@ -146,7 +145,7 @@ listremove(List* l, unsigned long i)
void**
listdup(List* l)
{
void** result = (void**)malloc(sizeof(void*)*(l->length+1));
void** result = (void**)emalloc(sizeof(void*)*(l->length+1));
memcpy((void*)result,(void*)l->content,sizeof(void*)*l->length);
result[l->length] = (void*)0;
return result;

23
ncgen/main.c
View File

@ -52,8 +52,8 @@ size_t nciterbuffersize;
struct Vlendata* vlendata;
char *netcdf_name; /* command line -o file name */
char *datasetname; /* name from the netcdf <name> {} || from -N */
char *netcdf_name = NULL; /* command line -o file name */
char *datasetname = NULL; /* name from the netcdf <name> {} || from -N */
extern FILE *ncgin;
@ -158,19 +158,6 @@ static char* LE16 = "\xFF\xFE"; /* UTF-16; little-endian */
#define DFALTBINNCITERBUFFERSIZE 0x40000 /* about 250k bytes */
#define DFALTLANGNCITERBUFFERSIZE 0x4000 /* about 15k bytes */
void *emalloc (size_t size) { /* check return from malloc */
void *p;
if (size == 0)
return 0;
p = (void *) malloc (size);
if (p == 0) {
exit(NC_ENOMEM);
}
return p;
}
/* strip off leading path */
/* result is malloc'd */
@ -340,9 +327,11 @@ main(
binary_ext = ".cdf";
break;
case 'o': /* to explicitly specify output name */
if(netcdf_name) efree(netcdf_name);
netcdf_name = nulldup(optarg);
break;
case 'N': /* to explicitly specify dataset name */
if(datasetname) efree(datasetname);
datasetname = nulldup(optarg);
break;
case 'x': /* set nofill mode to speed up creation of large files */
@ -428,7 +417,7 @@ main(
}
#ifndef ENABLE_C
if(c_flag) {
if(l_flag == L_C) {
fprintf(stderr,"C not currently supported\n");
code=1; goto done;
}
@ -597,6 +586,8 @@ main(
define_netcdf();
done:
nullfree(netcdf_name);
nullfree(datasetname);
finalize_netcdf(code);
return code;
}

76
ncgen/ncgen.1
View File

@ -6,16 +6,25 @@ ncgen \- From a CDL file generate a netCDF-3 file, a netCDF-4 file or a C progra
.HP
ncgen
.nh
\%[\-b]
\%[\-c]
\%[\-f]
\%[\-k \fIformat_name\fP]
\%[\-\fIformat_code\fP]
\%[\-l \fIoutput language\fP]
\%[\-1|3|4|5|6|7]
\%[\-b]
\%[\-B \fIbuffersize\fP]
\%[\-c]
\%[\-d]
\%[\-D \fIdebuglevel\fP]
\%[\-f]
\%[\-h]
\%[\-H]
\%[\-k \fIformat_name\fP]
\%[\-l \fIb|c|f77|java\fP]
\%[\-L \fIloglevel\fP]
\%[\-M \fIname\fP]
\%[\-n]
\%[\-N \fIdatasetname\fP]
\%[\-o \fInetcdf_filename\fP]
\%[\-P]
\%[\-x]
\%[\fIinput_file\fP]
.hy
.ft
.SH DESCRIPTION
@ -43,17 +52,35 @@ in a netCDF file, use \fBncdump\fP to get a CDL version of the netCDF file,
edit the CDL file to change the name of the dimensions, and use \fBncgen\fP
to generate the corresponding netCDF file from the edited CDL file.
.SH OPTIONS
.IP "\fB-1|3|4|5|6|7\fP"
Alternate method to specify the format.
.RS
.RS
.IP "3 => netcdf classic format"
.IP "4 => netCDF-4 format (enhanced data model)"
.IP "5 => netcdf 5 format"
.IP "6 => netCDF 64-bit format"
.IP "7 => netCDF-4 classic model format (3+4 == 7)"
.RE
.RE
See the \fB-k\fP flag.
.IP "\fB-b\fP"
Create a (binary) netCDF file. If the \fB-o\fP option is absent, a
default file name will be constructed from the basename of the CDL
file, with any suffix replaced by the `.nc' extension. If a
file already exists with the specified name, it will be overwritten.
.IP "\fB-B\fP \fRbuffersize\fP\fP"
Specify the internal iterator buffer size.
.IP "\fB-c\fP"
Generate
.B C
source code that will create a netCDF file
matching the netCDF specification. The C source code is written to
standard output; equivalent to \-lc.
.IP "\fB-d\fP"
Same as \fB-D1\fP.
.IP "\fB-D\fP \fRdebuglevel\fP"
Set the level of debug output.
.IP "\fB-f\fP"
Generate
.B FORTRAN 77
@ -61,13 +88,10 @@ source code that will create a netCDF file
matching the netCDF specification.
The source code is written to
standard output; equivalent to \-lf77.
.IP "\fB-o\fP \fRnetcdf_file\fP"
Name of the file to pass to calls to "nc_create()".
If this option is specified it implies
(in the absence of any explicit \-l flag) the "\fB-b\fP" option.
This option is necessary because netCDF files
cannot be written directly to standard output, since standard output is not
seekable.
.IP "\fB-h\fP"
Output help information.
.IP "\fB-H\fP"
Output the header only; ignore the \fIdata\fP section.
.IP "\fB-k \fIformat_name\fP"
.IP "\fB-\fIformat_code\fP"
The \-k flag specifies the format of the file to be created and, by inference,
@ -105,11 +129,7 @@ format numbers and format names. Various old format name aliases are
also accepted but deprecated, e.g. 'hdf5', 'enhanced-nc3', etc.
Also, note that \-v is accepted to mean the same thing as
\-k for backward compatibility.
.IP "\fB-x\fP"
Don't initialize data with fill values. This can speed up creation of
large netCDF files greatly, but later attempts to read unwritten data
from the generated file will not be easily detectable.
.IP "\fB-l \fRoutput_language\fP"
.IP "\fB-l\fP \fRb|c|f77|java\fP"
The \-l flag specifies the output language to use
when generating source code that will create or define a netCDF file
matching the netCDF specification.
@ -170,6 +190,26 @@ a \-k flag value of 4 (netCDF-4 classic model) will be used.
.IP "\fB6.\fP"
Otherwise ncgen will set the \-k flag to 1 (classic model).
.RE
.IP "\fB-L\fP \fRloglevel\fP"
.IP "\fB-M\fP \fRname\fP"
Specify the name for the main function for C, F77, or Java.
.IP "\fB-n\fP"
.IP "\fB-N\fP \fRdatasetname\fP"
.IP "\fB-o\fP \fRnetcdf_file\fP"
Name of the file to pass to calls to "nc_create()".
If this option is specified it implies
(in the absence of any explicit \-l flag) the "\fB-b\fP" option.
This option is necessary because netCDF files
cannot be written directly to standard output, since standard output is not
seekable.
.IP "\fB-P\fP"
Use NC_DISKLESS mode to create the file totally in memory
before persisting it to disk.
.IP "\fB-x\fP"
Don't initialize data with fill values. This can speed up creation of
large netCDF files greatly, but later attempts to read unwritten data
from the generated file will not be easily detectable.
.SH EXAMPLES
.LP
Check the syntax of the CDL file `\fBfoo.cdl\fP':

27
ncgen/ncgen.h
View File

@ -131,7 +131,7 @@ unsigned int id;
/* Note: some non-var specials (i.e. _Format) are not included in this struct*/
typedef struct Specialdata {
int flags;
Datalist* _Fillvalue; /* This is a per-type */
Datalist* _Fillvalue; /* This is a per-type ; points to the _FillValue attribute node */
int _Storage; /* NC_CHUNKED | NC_CONTIGUOUS*/
size_t* _ChunkSizes; /* NULL => defaults*/
int nchunks; /* |_Chunksize| ; 0 => not specified*/
@ -152,6 +152,18 @@ typedef struct GlobalSpecialdata {
int _Superblock ; /* HDF5 file superblock version */
} GlobalSpecialData;
/*
During the generation of binary data,
we will generate a number of references
to strings and opaques that should
be reclaimed to keep the memory
checkers happy.
*/
typedef struct BinBuffer {
Bytebuffer* buf; /* top level data */
List* reclaim; /* objects that need to be free'd */
} BinBuffer;
/* Track a set of dimensions*/
/* (Note: the netcdf type system is deficient here)*/
typedef struct Dimset {
@ -175,7 +187,7 @@ typedef struct Typeinfo {
nc_type typecode;
unsigned long offset; /* fields in struct*/
unsigned long alignment;/* fields in struct*/
NCConstant econst; /* for enum values*/
NCConstant* econst; /* for enum values*/
Dimset dimset; /* for NC_VAR/NC_FIELD/NC_ATT*/
size_t size; /* for opaque, compound, etc.*/
size_t cmpdalign; /* alignment needed for total size instances */
@ -188,13 +200,17 @@ typedef struct Typeinfo {
typedef struct Varinfo {
int nattributes; /* |attributes|*/
List* attributes; /* List<Symbol*>*/
Specialdata special;
Specialdata* special;
} Varinfo;
typedef struct Groupinfo {
int is_root;
} Groupinfo;
typedef struct Fileinfo {
char* filename;
} Fileinfo;
/* store info when the symbol
is really a reference to another
symbol
@ -206,7 +222,6 @@ typedef struct Reference {
} Reference;
typedef struct Symbol { /* symbol table entry*/
struct Symbol* next; /* Linked list of all defined symbols*/
nc_class objectclass; /* NC_DIM|NC_VLEN|NC_OPAQUE...*/
nc_class subclass; /* NC_STRUCT|...*/
char* name;
@ -226,11 +241,13 @@ typedef struct Symbol { /* symbol table entry*/
Attrinfo att;
Diminfo dim;
Groupinfo grp;
Fileinfo file;
Reference ref; /* symbol is really a reference to another symbol*/
/* Misc pieces of info*/
int lineno; /* at point of creation*/
int touched; /* for sorting*/
int ncid; /* from netcdf API: varid, or dimid, or etc.*/
/* for use by -lb */
int nc_id; /* from netcdf API: varid, or dimid, or etc.*/
} Symbol;

92
ncgen/ncgen.l
View File

@ -90,7 +90,7 @@ static unsigned int MAX_UINT = NC_MAX_UINT;
char errstr[100]; /* for short error messages */
int lineno; /* line number for error messages */
int lineno; /* line number for error messages */
Bytebuffer* lextext; /* name or string with escapes removed */
#define YY_BREAK /* defining as nothing eliminates unreachable
@ -224,7 +224,7 @@ USASCII [\x01-\x7F]
break;
}
\"{nonquotes}\" {int len;
\"{nonquotes}\" {int len; char* s = NULL;
/* In netcdf4, this will be used in a variety
of places, so only remove escapes */
/*
@ -233,18 +233,15 @@ yyerror("string too long, truncated\n");
yytext[MAXTRST-1] = '\0';
}
*/
/* FIX: Assumes unescape also does normalization */
bbSetalloc(lextext,yyleng+1); /*+1 for nul */
/* Adjust length */
bbSetlength(lextext,yyleng-2); /*-2 for quotes */
len = unescape(bbContents(lextext),
(char *)yytext+1,yyleng-2,!ISIDENT);
len = unescape((char *)yytext+1,yyleng-2,!ISIDENT,&s);
if(len < 0) {
sprintf(errstr,"Illegal character: %s",yytext);
yyerror(errstr);
}
bbSetlength(lextext,len);
bbClear(lextext);
bbAppendn(lextext,s,len);
bbNull(lextext);
if(s) efree(s);
return lexdebug(TERMSTRING);
}
@ -364,15 +361,15 @@ NIL|nil|Nil {
return lexdebug(DATASETID);
}
{ID} { char* id; int len;
bbClear(lextext);
bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */
bbNull(lextext);
id = bbContents(lextext);
len = unescape(id,id,bbLength(lextext),ISIDENT);
bbSetlength(lextext,len);
if (NCSTREQ(id, FILL_STRING)) return lexdebug(FILLMARKER);
{ID} { char* id = NULL; int len;
len = strlen(yytext);
len = unescape(yytext,len,ISIDENT,&id);
if(NCSTREQ(id, FILL_STRING)) {
efree(id);
return lexdebug(FILLMARKER);
}
yylval.sym = install(id);
efree(id);
return lexdebug(IDENT);
}
@ -608,45 +605,38 @@ makepath(char* text0)
List* prefix = listnew();
/* split the text into IDENT chunks, convert to symbols */
Symbol* container = rootgroup;
char *ident, *p;
char *ident, *p, *match;
char* text = estrdup(text0);
int c,lastident;
int lastident;
ident=text+1; p=ident; /* skip leading '/' */
lastident = 0;
do {
lastident = 0;
switch ((c=*p)) {
default: p++; break;
case '\\': p++; if(*p == '/') p++; break;
case '\0': /* treat null terminator like trailing '/' (mostly) */
lastident=1; /* this is the last ident in the path */
/*fall thru */
case '/':
*p='\0';
if(!lastident) {
unescape(ident,ident,strlen(ident),ISIDENT);
refsym = lookupingroup(NC_GRP,ident,container);
if(refsym == NULL) {
sprintf(errstr,"Undefined or forward referenced group: %s",ident);
yyerror(errstr);
refsym = rootgroup;
} else {
listpush(prefix,(void*)refsym);
}
} else { /* lastident is true */
unescape(ident,ident,strlen(ident),ISIDENT);
refsym = install(ident);
refsym->objectclass = NC_GRP;/* tentative */
refsym->ref.is_ref = 1;
refsym->container = container;
refsym->subnodes = listnew();
}
container = refsym;
ident=p+1; p=ident;
break;
match = esc_strchr(p,'/',0);
lastident = (*match == '\0');
*match='\0';
(void)unescape(p,strlen(p),ISIDENT,&ident);
refsym = lookupingroup(NC_GRP,ident,container);
if(!lastident) {
if(refsym == NULL) {
sprintf(errstr,"Undefined or forward referenced group: %s",ident);
yyerror(errstr);
refsym = rootgroup;
} else
listpush(prefix,(void*)refsym);
} else {/* lasiident is true */
refsym = install(ident); /* create as symbol */
refsym->objectclass = NC_GRP;/* tentative */
refsym->ref.is_ref = 1;
refsym->container = container;
refsym->subnodes = listnew();
}
} while(c != '\0');
container = refsym;
p = (lastident?match:match+1);
if(ident) efree(ident);
} while(!lastident);
refsym->prefix = prefix;
free(text);
efree(text);
}
return refsym;
}

290
ncgen/ncgen.y
View File

@ -13,7 +13,7 @@
static char SccsId[] = "$Id: ncgen.y,v 1.42 2010/05/18 21:32:46 dmh Exp $";
*/
#include "includes.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
#include "ncgeny.h"
#include "ncgen.h"
#ifdef USE_NETCDF4
@ -36,15 +36,18 @@ static char SccsId[] = "$Id: ncgen.y,v 1.42 2010/05/18 21:32:46 dmh Exp $";
#define VLENSIZE (sizeof(nc_vlen_t))
#define MAXFLOATDIM 4294967295.0
/* mnemonic */
/* mnemonics */
typedef enum Attrkind {ATTRVAR, ATTRGLOBAL, DONTKNOW} Attrkind;
#define ISCONST 1
#define ISLIST 0
typedef nc_vlen_t vlen_t;
/* We retain the old representation of the symbol list
as a linked list.
*/
Symbol* symlist;
List* symlist;
/* Track rootgroup separately*/
Symbol* rootgroup;
@ -109,9 +112,9 @@ List* condefs; /* non-dimension constants used in type defs*/
List* tmp;
/* Forward */
static NCConstant makeconstdata(nc_type);
static NCConstant evaluate(Symbol* fcn, Datalist* arglist);
static NCConstant makeenumconstref(Symbol*);
static NCConstant* makeconstdata(nc_type);
static NCConstant* evaluate(Symbol* fcn, Datalist* arglist);
static NCConstant* makeenumconstref(Symbol*);
static void addtogroup(Symbol*);
static Symbol* currentgroup(void);
static Symbol* createrootgroup(const char*);
@ -122,9 +125,8 @@ static Symbol* makeattribute(Symbol*,Symbol*,Symbol*,Datalist*,Attrkind);
static Symbol* makeprimitivetype(nc_type i);
static Symbol* makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst);
static int containsfills(Datalist* list);
static void datalistextend(Datalist* dl, NCConstant* con);
static void vercheck(int ncid);
static long long extractint(NCConstant con);
static long long extractint(NCConstant* con);
#ifdef USE_NETCDF4
static int parsefilterflag(const char* sdata0, Specialdata* special);
#endif
@ -150,7 +152,7 @@ unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
long mark; /* track indices into the sequence*/
int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant constant;
NCConstant* constant;
}
%token <sym>
@ -361,7 +363,7 @@ opaquedecl: OPAQUE_ '(' INT_CONST ')' typename
$5->subclass=NC_OPAQUE;
$5->typ.typecode=NC_OPAQUE;
$5->typ.size=int32_val;
$5->typ.alignment=nctypealignment(NC_OPAQUE);
$5->typ.alignment=ncaux_class_alignment(NC_OPAQUE);
}
;
@ -375,7 +377,7 @@ vlendecl: typeref '(' '*' ')' typename
$5->typ.basetype=basetype;
$5->typ.typecode=NC_VLEN;
$5->typ.size=VLENSIZE;
$5->typ.alignment=nctypealignment(NC_VLEN);
$5->typ.alignment=ncaux_class_alignment(NC_VLEN);
}
;
@ -466,6 +468,7 @@ dimdecl:
#ifdef GENDEBUG1
fprintf(stderr,"dimension: %s = %llu\n",$1->name,(unsigned long long)$1->dim.declsize);
#endif
reclaimconstant($3);
}
| dimd '=' NC_UNLIMITED_K
{
@ -516,6 +519,9 @@ vardecl: typeref varlist
sym->typ.basetype = $1;
addtogroup(sym);
listpush(vardefs,(void*)sym);
sym->var.special = ecalloc(sizeof(Specialdata));
if(sym->var.special == NULL)
derror("out of memory");
}
}
listsetlength(stack,stackbase);/* remove stack nodes*/
@ -534,6 +540,7 @@ varspec: ident dimspec
{
int i;
Dimset dimset;
Symbol* var = $1; /* for debugging */
stacklen=listlength(stack);
stackbase=$2;
count = stacklen - stackbase;
@ -549,11 +556,12 @@ varspec: ident dimspec
Symbol* dsym = (Symbol*)listget(stack,stackbase+i);
dimset.dimsyms[i] = dsym;
}
$1->typ.dimset = dimset;
var->typ.dimset = dimset;
}
$1->typ.basetype = NULL; /* not yet known*/
$1->objectclass=NC_VAR;
var->typ.basetype = NULL; /* not yet known*/
var->objectclass=NC_VAR;
listsetlength(stack,stackbase);/* remove stack nodes*/
$$ = var;
}
;
@ -711,11 +719,11 @@ attrdecllist: /*empty*/ {} | attrdecl ';' attrdecllist {} ;
attrdecl:
':' _NCPROPS '=' conststring
{$$ = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)&$4,ATTRGLOBAL);}
{$$ = makespecial(_NCPROPS_FLAG,NULL,NULL,(void*)$4,ISCONST);}
| ':' _ISNETCDF4 '=' constbool
{$$ = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)&$4,ATTRGLOBAL);}
{$$ = makespecial(_ISNETCDF4_FLAG,NULL,NULL,(void*)$4,ISCONST);}
| ':' _SUPERBLOCK '=' constint
{$$ = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)&$4,ATTRGLOBAL);}
{$$ = makespecial(_SUPERBLOCK_FLAG,NULL,NULL,(void*)$4,ISCONST);}
| ':' ident '=' datalist
{ $$=makeattribute($2,NULL,NULL,$4,ATTRGLOBAL);}
| typeref type_var_ref ':' ident '=' datalist
@ -739,27 +747,27 @@ attrdecl:
}
}
| type_var_ref ':' _FILLVALUE '=' datalist
{$$ = makespecial(_FILLVALUE_FLAG,$1,NULL,(void*)$5,0);}
{$$ = makespecial(_FILLVALUE_FLAG,$1,NULL,(void*)$5,ISLIST);}
| typeref type_var_ref ':' _FILLVALUE '=' datalist
{$$ = makespecial(_FILLVALUE_FLAG,$2,$1,(void*)$6,0);}
{$$ = makespecial(_FILLVALUE_FLAG,$2,$1,(void*)$6,ISLIST);}
| type_var_ref ':' _STORAGE '=' conststring
{$$ = makespecial(_STORAGE_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_STORAGE_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _CHUNKSIZES '=' intlist
{$$ = makespecial(_CHUNKSIZES_FLAG,$1,NULL,(void*)$5,0);}
{$$ = makespecial(_CHUNKSIZES_FLAG,$1,NULL,(void*)$5,ISLIST);}
| type_var_ref ':' _FLETCHER32 '=' constbool
{$$ = makespecial(_FLETCHER32_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_FLETCHER32_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _DEFLATELEVEL '=' constint
{$$ = makespecial(_DEFLATE_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_DEFLATE_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _SHUFFLE '=' constbool
{$$ = makespecial(_SHUFFLE_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_SHUFFLE_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _ENDIANNESS '=' conststring
{$$ = makespecial(_ENDIAN_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_ENDIAN_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _FILTER '=' conststring
{$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_FILTER_FLAG,$1,NULL,(void*)$5,ISCONST);}
| type_var_ref ':' _NOFILL '=' constbool
{$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)&$5,1);}
{$$ = makespecial(_NOFILL_FLAG,$1,NULL,(void*)$5,ISCONST);}
| ':' _FORMAT '=' conststring
{$$ = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)&$4,1);}
{$$ = makespecial(_FORMAT_FLAG,NULL,NULL,(void*)$4,ISCONST);}
;
path:
@ -801,9 +809,9 @@ datalist0:
;
datalist1: /* Must have at least 1 element */
dataitem {$$ = builddatalist(0); datalistextend($$,&($1));}
dataitem {$$ = const2list($1);}
| datalist ',' dataitem
{datalistextend($1,&($3)); $$=$1;}
{dlappend($1,($3)); $$=$1; }
;
dataitem:
@ -830,9 +838,9 @@ function:
arglist:
simpleconstant
{$$ = builddatalist(0); datalistextend($$,&($1));}
{$$ = const2list($1);}
| arglist ',' simpleconstant
{datalistextend($1,&($3)); $$=$1;}
{dlappend($1,($3)); $$=$1;}
;
simpleconstant:
@ -851,8 +859,8 @@ simpleconstant:
;
intlist:
constint {$$ = builddatalist(0); datalistextend($$,&($1));}
| intlist ',' constint {$$=$1; datalistextend($1,&($3));}
constint {$$ = const2list($1);}
| intlist ',' constint {$$=$1; dlappend($1,($3));}
;
constint:
@ -892,10 +900,10 @@ yyerror(fmt,va_alist) const char* fmt; va_dcl
#endif
{
va_list argv;
vastart(argv,fmt);
va_start(argv,fmt);
(void)fprintf(stderr,"%s: %s line %d: ", progname, cdlname, lineno);
vderror(fmt,argv);
vaend(argv,fmt);
va_end(argv);
}
/* undefine yywrap macro, in case we are using bison instead of yacc */
@ -919,7 +927,7 @@ parse_init(void)
int i;
opaqueid = 0;
arrayuid = 0;
symlist = NULL;
symlist = listnew();
stack = listnew();
groupstack = listnew();
consttype = NC_NAT;
@ -945,11 +953,11 @@ makeprimitivetype(nc_type nctype)
Symbol* sym = install(primtypenames[nctype]);
sym->objectclass=NC_TYPE;
sym->subclass=NC_PRIM;
sym->ncid = nctype;
sym->nc_id = nctype;
sym->typ.typecode = nctype;
sym->typ.size = ncsize(nctype);
sym->typ.nelems = 1;
sym->typ.alignment = nctypealignment(nctype);
sym->typ.alignment = ncaux_class_alignment(nctype);
/* Make the basetype circular so we can always ask for it */
sym->typ.basetype = sym;
sym->prefix = listnew();
@ -964,15 +972,13 @@ install(const char *sname)
Symbol* sp;
sp = (Symbol*) ecalloc (sizeof (struct Symbol));
sp->name = nulldup(sname);
sp->next = symlist;
sp->lineno = lineno;
sp->location = currentgroup();
sp->container = currentgroup();
symlist = sp;
listpush(symlist,sp);
return sp;
}
static Symbol*
currentgroup(void)
{
@ -1010,40 +1016,39 @@ creategroup(Symbol * gsym)
return gsym;
}
static NCConstant
static NCConstant*
makeconstdata(nc_type nctype)
{
NCConstant con = nullconstant;
NCConstant* con = nullconst();
consttype = nctype;
con.nctype = nctype;
con.lineno = lineno;
con.filled = 0;
con->nctype = nctype;
con->lineno = lineno;
con->filled = 0;
switch (nctype) {
case NC_CHAR: con.value.charv = char_val; break;
case NC_BYTE: con.value.int8v = byte_val; break;
case NC_SHORT: con.value.int16v = int16_val; break;
case NC_INT: con.value.int32v = int32_val; break;
case NC_CHAR: con->value.charv = char_val; break;
case NC_BYTE: con->value.int8v = byte_val; break;
case NC_SHORT: con->value.int16v = int16_val; break;
case NC_INT: con->value.int32v = int32_val; break;
case NC_FLOAT:
con.value.floatv = float_val;
con->value.floatv = float_val;
break;
case NC_DOUBLE:
con.value.doublev = double_val;
con->value.doublev = double_val;
break;
case NC_STRING: { /* convert to a set of chars*/
size_t len;
len = bbLength(lextext);
con.value.stringv.len = len;
con.value.stringv.stringv = bbDup(lextext);
bbClear(lextext);
con->value.stringv.len = len;
con->value.stringv.stringv = bbExtract(lextext);
}
break;
/* Allow these constants even in netcdf-3 */
case NC_UBYTE: con.value.uint8v = ubyte_val; break;
case NC_USHORT: con.value.uint16v = uint16_val; break;
case NC_UINT: con.value.uint32v = uint32_val; break;
case NC_INT64: con.value.int64v = int64_val; break;
case NC_UINT64: con.value.uint64v = uint64_val; break;
case NC_UBYTE: con->value.uint8v = ubyte_val; break;
case NC_USHORT: con->value.uint16v = uint16_val; break;
case NC_UINT: con->value.uint32v = uint32_val; break;
case NC_INT64: con->value.int64v = int64_val; break;
case NC_UINT64: con->value.uint64v = uint64_val; break;
#ifdef USE_NETCDF4
case NC_OPAQUE: {
@ -1053,8 +1058,8 @@ makeconstdata(nc_type nctype)
s = (char*)ecalloc(len+1);
strncpy(s,bbContents(lextext),len);
s[len] = '\0';
con.value.opaquev.stringv = s;
con.value.opaquev.len = len;
con->value.opaquev.stringv = s;
con->value.opaquev.len = len;
} break;
case NC_NIL:
@ -1067,25 +1072,25 @@ makeconstdata(nc_type nctype)
default:
yyerror("Data constant: unexpected NC type: %s",
nctypename(nctype));
con.value.stringv.stringv = NULL;
con.value.stringv.len = 0;
con->value.stringv.stringv = NULL;
con->value.stringv.len = 0;
}
return con;
}
static NCConstant
static NCConstant*
makeenumconstref(Symbol* refsym)
{
NCConstant con;
NCConstant* con = nullconst();
markcdf4("Enum type");
consttype = NC_ENUM;
con.nctype = NC_ECONST;
con.lineno = lineno;
con.filled = 0;
con->nctype = NC_ECONST;
con->lineno = lineno;
con->filled = 0;
refsym->objectclass = NC_TYPE;
refsym->subclass = NC_ECONST;
con.value.enumv = refsym;
con->value.enumv = refsym;
return con;
}
@ -1158,9 +1163,9 @@ static Symbol*
makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
{
Symbol* attr = NULL;
Datalist* list;
NCConstant* con;
NCConstant iconst;
Datalist* list = NULL;
NCConstant* con = NULL;
NCConstant* tmp = NULL;
int tf = 0;
char* sdata = NULL;
int idata = -1;
@ -1180,44 +1185,48 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
if(tag != _FILLVALUE_FLAG && tag != _FORMAT_FLAG)
/*Main.*/specials_flag++;
if(isconst) {
if(isconst)
con = (NCConstant*)data;
list = builddatalist(1);
dlappend(list,(NCConstant*)data);
} else {
else
list = (Datalist*)data;
con = (NCConstant*)list->data;
}
switch (tag) {
case _FLETCHER32_FLAG:
case _SHUFFLE_FLAG:
case _ISNETCDF4_FLAG:
case _NOFILL_FLAG:
iconst.nctype = (con->nctype == NC_STRING?NC_STRING:NC_INT);
convert1(con,&iconst);
tf = truefalse(&iconst,tag);
tmp = nullconst();
tmp->nctype = (con->nctype == NC_STRING?NC_STRING:NC_INT);
convert1(con,tmp);
tf = truefalse(tmp,tag);
reclaimconstant(tmp);
break;
case _FORMAT_FLAG:
case _STORAGE_FLAG:
case _NCPROPS_FLAG:
case _ENDIAN_FLAG:
case _FILTER_FLAG:
iconst.nctype = NC_STRING;
convert1(con,&iconst);
if(iconst.nctype == NC_STRING)
sdata = iconst.value.stringv.stringv;
else
tmp = nullconst();
tmp->nctype = NC_STRING;
convert1(con,tmp);
if(tmp->nctype == NC_STRING) {
sdata = tmp->value.stringv.stringv;
tmp->value.stringv.stringv = NULL;
tmp->value.stringv.len = 0;
} else
derror("%s: illegal value",specialname(tag));
reclaimconstant(tmp);
break;
case _SUPERBLOCK_FLAG:
case _DEFLATE_FLAG:
iconst.nctype = NC_INT;
convert1(con,&iconst);
if(iconst.nctype == NC_INT)
idata = iconst.value.int32v;
tmp = nullconst();
tmp->nctype = NC_INT;
convert1(con,tmp);
if(tmp->nctype == NC_INT)
idata = tmp->value.int32v;
else
derror("%s: illegal value",specialname(tag));
reclaimconstant(tmp);
break;
case _CHUNKSIZES_FLAG:
case _FILLVALUE_FLAG:
@ -1248,18 +1257,28 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
globalspecials._IsNetcdf4 = tf;
else if(tag == _SUPERBLOCK_FLAG)
globalspecials._Superblock = idata;
else if(tag == _NCPROPS_FLAG)
globalspecials._NCProperties = estrdup(sdata);
else if(tag == _NCPROPS_FLAG) {
globalspecials._NCProperties = sdata;
sdata = NULL;
}
} else {
Specialdata* special;
/* Set up special info */
special = &vsym->var.special;
if(vsym->var.special == NULL) {
vsym->var.special = ecalloc(sizeof(Specialdata));
if(vsym->var.special == NULL)
derror("Out of memory");
}
special = vsym->var.special;
if(tag == _FILLVALUE_FLAG) {
special->_Fillvalue = list;
/* fillvalue must be a single value*/
if(list->length != 1)
if(!isconst && datalistlen(list) != 1)
derror("_FillValue: must be a single (possibly compound) value",
vsym->name);
if(isconst) {
list = const2list(con);
con = NULL;
}
/* check that the attribute value contains no fill values*/
if(containsfills(list)) {
derror("Attribute data may not contain fill values (i.e. _ )");
@ -1272,7 +1291,10 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
else if(vsym->typ.basetype != tsym) {
derror("_FillValue attribute type does not match variable type: %s",vsym->name);
}
special->_Fillvalue = clonedatalist(list);
/* Create the corresponding attribute */
attr = makeattribute(install("_FillValue"),vsym,tsym,list,ATTRVAR);
list = NULL;
} else switch (tag) {
/* These will be output as attributes later */
case _STORAGE_FLAG:
@ -1315,17 +1337,20 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
break;
case _CHUNKSIZES_FLAG: {
int i;
list = (isconst ? const2list(con) : list);
special->nchunks = list->length;
special->_ChunkSizes = (size_t*)ecalloc(sizeof(size_t)*special->nchunks);
for(i=0;i<special->nchunks;i++) {
iconst.nctype = NC_INT;
convert1(&list->data[i],&iconst);
if(iconst.nctype == NC_INT) {
special->_ChunkSizes[i] = (size_t)iconst.value.int32v;
tmp = nullconst();
tmp->nctype = NC_INT;
convert1(list->data[i],tmp);
if(tmp->nctype == NC_INT) {
special->_ChunkSizes[i] = (size_t)tmp->value.int32v;
} else {
efree(special->_ChunkSizes);
derror("%s: illegal value",specialname(tag));
}
reclaimconstant(tmp);
}
special->flags |= _CHUNKSIZES_FLAG;
/* Chunksizes => storage == chunked */
@ -1342,12 +1367,15 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
derror("_Filter: unparseable filter spec: %s",sdata);
}
#else
derror("%s: the filter attribute requires netcdf-4 to be enabled",specialname(tag));
derror("%s: the filter attribute requires netcdf-4 to be enabled",specialname(tag));
#endif
break;
default: PANIC1("makespecial: illegal token: %d",tag);
}
}
if(sdata) efree(sdata);
if(con) reclaimconstant(con);
if(list) reclaimdatalist(list);
return attr;
}
@ -1383,18 +1411,18 @@ makeattribute(Symbol* asym,
}
static long long
extractint(NCConstant con)
extractint(NCConstant* con)
{
switch (con.nctype) {
case NC_BYTE: return (long long)(con.value.int8v);
case NC_SHORT: return (long long)(con.value.int16v);
case NC_INT: return (long long)(con.value.int32v);
case NC_UBYTE: return (long long)(con.value.uint8v);
case NC_USHORT: return (long long)(con.value.uint16v);
case NC_UINT: return (long long)(con.value.uint32v);
case NC_INT64: return (long long)(con.value.int64v);
switch (con->nctype) {
case NC_BYTE: return (long long)(con->value.int8v);
case NC_SHORT: return (long long)(con->value.int16v);
case NC_INT: return (long long)(con->value.int32v);
case NC_UBYTE: return (long long)(con->value.uint8v);
case NC_USHORT: return (long long)(con->value.uint16v);
case NC_UINT: return (long long)(con->value.uint32v);
case NC_INT64: return (long long)(con->value.int64v);
default:
derror("Not a signed integer type: %d",con.nctype);
derror("Not a signed integer type: %d",con->nctype);
break;
}
return 0;
@ -1405,23 +1433,17 @@ containsfills(Datalist* list)
{
if(list != NULL) {
int i;
NCConstant* con = list->data;
for(i=0;i<list->length;i++,con++) {
if(con->nctype == NC_COMPOUND) {
if(containsfills(con->value.compoundv)) return 1;
} else if(con->nctype == NC_FILLVALUE)
NCConstant** cons = list->data;
for(i=0;i<list->length;i++) {
if(cons[i]->nctype == NC_COMPOUND) {
if(containsfills(cons[i]->value.compoundv)) return 1;
} else if(cons[i]->nctype == NC_FILLVALUE)
return 1;
}
}
return 0;
}
static void
datalistextend(Datalist* dl, NCConstant* con)
{
dlappend(dl,con);
}
/*
Try to infer the file type from the
kinds of constructs used in the cdl file.
@ -1481,29 +1503,29 @@ Note that currently, only a single value can
be returned.
*/
static NCConstant
static NCConstant*
evaluate(Symbol* fcn, Datalist* arglist)
{
NCConstant result = nullconstant;
NCConstant* result = nullconst();
/* prepare the result */
result.lineno = fcn->lineno;
result->lineno = fcn->lineno;
if(strcasecmp(fcn->name,"time") == 0) {
char* timekind = NULL;
char* timevalue = NULL;
result.nctype = NC_DOUBLE;
result.value.doublev = 0;
result->nctype = NC_DOUBLE;
result->value.doublev = 0;
/* int time([string],string) */
switch (arglist->length) {
case 2:
if(arglist->data[1].nctype != NC_STRING) {
if(arglist->data[1]->nctype != NC_STRING) {
derror("Expected function signature: time([string,]string)");
goto done;
}
/* fall thru */
case 1:
if(arglist->data[0].nctype != NC_STRING) {
if(arglist->data[0]->nctype != NC_STRING) {
derror("Expected function signature: time([string,]string)");
goto done;
}
@ -1514,10 +1536,10 @@ evaluate(Symbol* fcn, Datalist* arglist)
goto done;
}
if(arglist->length == 2) {
timekind = arglist->data[0].value.stringv.stringv;
timevalue = arglist->data[1].value.stringv.stringv;
timekind = arglist->data[0]->value.stringv.stringv;
timevalue = arglist->data[1]->value.stringv.stringv;
} else
timevalue = arglist->data[0].value.stringv.stringv;
timevalue = arglist->data[0]->value.stringv.stringv;
if(timekind == NULL) { /* use cd time as the default */
cdCompTime comptime;
CdTime cdtime;
@ -1531,7 +1553,7 @@ evaluate(Symbol* fcn, Datalist* arglist)
cdtime.baseYear = 1970;
cdtime.timeType = CdChron;
/* convert to double value */
Cdh2e(&cdtime,&result.value.doublev);
Cdh2e(&cdtime,&result->value.doublev);
} else {
derror("Time conversion '%s' not supported",timekind);
goto done;

977
ncgen/ncgenl.c
View File

File diff suppressed because it is too large Load Diff

854
ncgen/ncgeny.c
View File

File diff suppressed because it is too large Load Diff

4
ncgen/ncgeny.h
View File

@ -106,14 +106,14 @@ extern int ncgdebug;
union YYSTYPE
{
#line 147 "ncgen.y" /* yacc.c:1909 */
#line 149 "ncgen.y" /* yacc.c:1909 */
Symbol* sym;
unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/
long mark; /* track indices into the sequence*/
int nctype; /* for tracking attribute list type*/
Datalist* datalist;
NCConstant constant;
NCConstant* constant;
#line 119 "ncgeny.h" /* yacc.c:1909 */
};

245
ncgen/semantics.c
View File

@ -8,9 +8,9 @@
#include "includes.h"
#include "dump.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
/* Forward*/
static void computefqns(void);
static void filltypecodes(void);
static void processenums(void);
static void processeconstrefs(void);
@ -20,21 +20,25 @@ static void processvars(void);
static void processattributes(void);
static void processunlimiteddims(void);
static void processeconstrefs(void);
static void processeconstrefsR(Datalist*);
static void processeconstrefsR(Symbol*,Datalist*);
static void processroot(void);
static List* ecsearchgrp(Symbol* grp, List* candidates);
static List* findecmatches(char* ident);
static void fixeconstref(NCConstant* con);
static void computefqns(void);
static void fixeconstref(Symbol*,NCConstant* con);
static void inferattributetype(Symbol* asym);
static void validateNIL(Symbol* sym);
static void checkconsistency(void);
static int tagvlentypes(Symbol* tsym);
static void computefqns(void);
static Symbol* uniquetreelocate(Symbol* refsym, Symbol* root);
static Symbol* checkeconst(Symbol* en, const char* refname);
static char* createfilename(void);
#if 0
static Symbol* locateenumtype(Symbol* econst, Symbol* group, NCConstant*);
static List* findecmatches(char* ident);
static List* ecsearchgrp(Symbol* grp, List* candidates);
static Symbol* checkeconst(Symbol* en, const char* refname);
#endif
List* vlenconstants; /* List<Constant*>;*/
/* ptr to vlen instances across all datalists*/
@ -43,6 +47,8 @@ List* vlenconstants; /* List<Constant*>;*/
void
processsemantics(void)
{
/* Fix up the root name to match the chosen filename */
processroot();
/* Fill in the fqn for every defining symbol */
computefqns();
/* Process each type and sort by dependency order*/
@ -193,7 +199,6 @@ uniquetreelocate(Symbol* refsym, Symbol* root)
return sym;
}
/*
Compute the fqn for every top-level definition symbol
*/
@ -252,6 +257,17 @@ computefqns(void)
}
}
/**
Process the root group.
Currently mean:
1. Compute and store the filename
*/
static void
processroot(void)
{
rootgroup->file.filename = createfilename();
}
/* 1. Do a topological sort of the types based on dependency*/
/* so that the least dependent are first in the typdefs list*/
/* 2. fill in type typecodes*/
@ -376,8 +392,9 @@ tagvlentypes(Symbol* tsym)
static void
filltypecodes(void)
{
Symbol* sym;
for(sym=symlist;sym != NULL;sym = sym->next) {
int i;
for(i=0;i<listlength(symlist);i++) {
Symbol* sym = listget(symlist,i);
if(sym->typ.basetype != NULL && sym->typ.typecode == NC_NAT)
sym->typ.typecode = sym->typ.basetype->typ.typecode;
}
@ -409,10 +426,11 @@ processenums(void)
if(tsym->subclass != NC_ENUM) continue;
for(j=0;j<listlength(tsym->subnodes);j++) {
Symbol* esym = (Symbol*)listget(tsym->subnodes,j);
NCConstant newec;
NCConstant* newec = nullconst();
ASSERT(esym->subclass == NC_ECONST);
newec.nctype = esym->typ.typecode;
convert1(&esym->typ.econst,&newec);
newec->nctype = esym->typ.typecode;
convert1(esym->typ.econst,newec);
reclaimconstant(esym->typ.econst);
esym->typ.econst = newec;
}
}
@ -429,54 +447,104 @@ processeconstrefs(void)
for(i=0;i<listlength(gattdefs);i++) {
Symbol* att = (Symbol*)listget(gattdefs,i);
if(att->data != NULL && listlength(att->data) > 0)
processeconstrefsR(att->data);
processeconstrefsR(att,att->data);
}
for(i=0;i<listlength(attdefs);i++) {
Symbol* att = (Symbol*)listget(attdefs,i);
if(att->data != NULL && listlength(att->data) > 0)
processeconstrefsR(att->data);
processeconstrefsR(att,att->data);
}
for(i=0;i<listlength(vardefs);i++) {
Symbol* var = (Symbol*)listget(vardefs,i);
if(var->data != NULL && listlength(var->data) > 0)
processeconstrefsR(var->data);
processeconstrefsR(var,var->data);
if(var->var.special->_Fillvalue != NULL)
processeconstrefsR(var,var->var.special->_Fillvalue);
}
}
/* Recursive helper for processeconstrefs */
static void
processeconstrefsR(Datalist* data)
processeconstrefsR(Symbol* avsym, Datalist* data)
{
NCConstant* con = NULL;
NCConstant** dlp = NULL;
int i;
for(i=0,con=data->data;i<data->length;i++,con++) {
for(i=0,dlp=data->data;i<data->length;i++,dlp++) {
NCConstant* con = *dlp;
if(con->nctype == NC_COMPOUND) {
/* Iterate over the sublists */
processeconstrefsR(con->value.compoundv);
} else if(con->nctype == NC_ECONST) {
fixeconstref(con);
processeconstrefsR(avsym,con->value.compoundv);
} else if(con->nctype == NC_ECONST || con->nctype == NC_FILLVALUE) {
fixeconstref(avsym,con);
}
}
}
static void
fixeconstref(NCConstant* con)
fixeconstref(Symbol* avsym, NCConstant* con)
{
Symbol* basetype = NULL;
Symbol* refsym = con->value.enumv;
Symbol* varsym = NULL;
int i;
/* Figure out the proper type associated with avsym */
ASSERT(avsym->objectclass == NC_VAR || avsym->objectclass == NC_ATT);
if(avsym->objectclass == NC_VAR) {
basetype = avsym->typ.basetype;
varsym = avsym;
} else { /*(avsym->objectclass == NC_ATT)*/
basetype = avsym->typ.basetype;
varsym = avsym->container;
if(varsym->objectclass == NC_GRP)
varsym = NULL;
}
if(basetype->objectclass != NC_TYPE && basetype->subclass != NC_ENUM)
semerror(con->lineno,"Enumconstant associated with a non-econst type");
if(con->nctype == NC_FILLVALUE) {
Datalist* filllist = NULL;
NCConstant* filler = NULL;
filllist = getfiller(varsym == NULL?basetype:varsym);
if(filllist == NULL)
semerror(con->lineno, "Cannot determine enum constant fillvalue");
filler = datalistith(filllist,0);
con->value.enumv = filler->value.enumv;
return;
}
for(i=0;i<listlength(basetype->subnodes);i++) {
Symbol* econst = listget(basetype->subnodes,i);
ASSERT(econst->subclass == NC_ECONST);
if(strcmp(econst->name,refsym->name)==0) {
con->value.enumv = econst;
return;
}
}
semerror(con->lineno,"Undefined enum or enum constant reference: %s",refsym->name);
}
#if 0
/* If we have an enum-valued group attribute, then we need to do
extra work to find the containing enum type
*/
static Symbol*
locateenumtype(Symbol* refsym, Symbol* parent, NCConstant* con)
{
Symbol* match = NULL;
Symbol* parent = NULL;
Symbol* refsym = con->value.enumv;
List* grpmatches;
/* Locate all possible matching enum constant definitions */
List* candidates = findecmatches(refsym->name);
if(candidates == NULL) {
semerror(con->lineno,"Undefined enum or enum constant reference: %s",refsym->name);
return;
return NULL;
}
/* One hopes that 99% of the time, the match is unique */
if(listlength(candidates) == 1) {
con->value.enumv = (Symbol*)listget(candidates,0);
match = listget(candidates,0);
goto done;
}
/* If this ref has a specified group prefix, then find that group
@ -499,12 +567,11 @@ fixeconstref(NCConstant* con)
default:
semerror(con->lineno,"Ambiguous enum constant reference: %s", fullname(refsym));
}
con->value.enumv = listget(grpmatches,0);
match = listget(grpmatches,0);
listfree(grpmatches);
goto done;
}
/* Sigh, we have to search up the tree to see if any of our candidates are there */
parent = refsym->container;
assert(parent == NULL || parent->objectclass == NC_GRP);
while(parent != NULL && match == NULL) {
grpmatches = ecsearchgrp(parent,candidates);
@ -518,15 +585,13 @@ fixeconstref(NCConstant* con)
}
listfree(grpmatches);
}
if(match != NULL) {
con->value.enumv = match;
goto done;
}
if(match != NULL) goto done;
/* Not unique and not in the parent tree, so complains and pick the first candidate */
semerror(con->lineno,"Ambiguous enum constant reference: %s", fullname(refsym));
con->value.enumv = (Symbol*)listget(candidates,0);
match = (Symbol*)listget(candidates,0);
done:
listfree(candidates);
return match;
}
/*
@ -602,7 +667,7 @@ checkeconst(Symbol* en, const char* refname)
}
return NULL;
}
#endif
/* Compute type sizes and compound offsets*/
void
@ -618,12 +683,12 @@ computesize(Symbol* tsym)
case NC_VLEN: /* actually two sizes for vlen*/
computesize(tsym->typ.basetype); /* first size*/
tsym->typ.size = ncsize(tsym->typ.typecode);
tsym->typ.alignment = nctypealignment(tsym->typ.typecode);
tsym->typ.alignment = ncaux_class_alignment(tsym->typ.typecode);
tsym->typ.nelems = 1; /* always a single compound datalist */
break;
case NC_PRIM:
tsym->typ.size = ncsize(tsym->typ.typecode);
tsym->typ.alignment = nctypealignment(tsym->typ.typecode);
tsym->typ.alignment = ncaux_class_alignment(tsym->typ.typecode);
tsym->typ.nelems = 1;
break;
case NC_OPAQUE:
@ -736,13 +801,14 @@ processattributes(void)
if(asym->typ.basetype == NULL) inferattributetype(asym);
/* fill in the typecode*/
asym->typ.typecode = asym->typ.basetype->typ.typecode;
if(asym->data->length == 0) {
if(asym->data != NULL && asym->data->length == 0) {
NCConstant* empty = NULL;
/* If the attribute has a zero length, then default it;
note that it must be of type NC_CHAR */
if(asym->typ.typecode != NC_CHAR)
semerror(asym->lineno,"Empty datalist can only be assigned to attributes of type char",fullname(asym));
asym->data = builddatalist(1);
emptystringconst(asym->lineno,&asym->data->data[asym->data->length]);
empty = emptystringconst(asym->lineno);
dlappend(asym->data,empty);
}
validateNIL(asym);
}
@ -761,18 +827,21 @@ processattributes(void)
/* Generate a default fill value */
asym->data = getfiller(asym->typ.basetype);
}
asym->att.var->var.special._Fillvalue = asym->data;
if(asym->att.var->var.special->_Fillvalue != NULL)
reclaimdatalist(asym->att.var->var.special->_Fillvalue);
asym->att.var->var.special->_Fillvalue = clonedatalist(asym->data);
} else if(asym->typ.basetype == NULL) {
inferattributetype(asym);
}
/* fill in the typecode*/
asym->typ.typecode = asym->typ.basetype->typ.typecode;
if(asym->data->length == 0) {
NCConstant* empty = NULL;
/* If the attribute has a zero length, and is char type, then default it */
if(asym->typ.typecode != NC_CHAR)
semerror(asym->lineno,"Empty datalist can only be assigned to attributes of type char",fullname(asym));
asym->data = builddatalist(1);
emptystringconst(asym->lineno,&asym->data->data[asym->data->length]);
empty = emptystringconst(asym->lineno);
dlappend(asym->data,empty);
}
validateNIL(asym);
}
@ -817,7 +886,7 @@ infertype(nc_type prior, nc_type next, int hasneg)
Collect info by repeated walking of the attribute value list.
*/
static nc_type
inferattributetype1(Datasrc* src)
inferattributetype1(Datalist* adata)
{
nc_type result = NC_NAT;
int hasneg = 0;
@ -826,39 +895,37 @@ inferattributetype1(Datasrc* src)
int forcefloat = 0;
int forcedouble = 0;
int forceuint64 = 0;
int i;
/* Walk the top level set of attribute values to ensure non-nesting */
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
if(con == NULL) return NC_NAT;
if(con->nctype > NC_MAX_ATOMIC_TYPE) { /* illegal */
return NC_NAT;
}
srcnext(src);
}
/* Walk repeatedly to get info for inference (loops could be combined) */
/* Walk repeatedly to get info for inference (loops could be combined) */
/* Compute: all strings or chars? */
srcreset(src);
stringcount = 0;
charcount = 0;
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
if(con->nctype == NC_STRING) stringcount++;
else if(con->nctype == NC_CHAR) charcount++;
}
if((stringcount+charcount) > 0) {
if((stringcount+charcount) < srclen(src))
if((stringcount+charcount) < datalistlen(adata))
return NC_NAT; /* not all textual */
return NC_CHAR;
}
/* Compute: any floats/doubles? */
srcreset(src);
forcefloat = 0;
forcedouble = 0;
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
if(con->nctype == NC_FLOAT) forcefloat = 1;
else if(con->nctype == NC_DOUBLE) {forcedouble=1; break;}
}
@ -868,10 +935,9 @@ inferattributetype1(Datasrc* src)
/* At this point all the constants should be integers */
/* Compute: are there any uint64 values > NC_MAX_INT64? */
srcreset(src);
forceuint64 = 0;
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
if(con->nctype != NC_UINT64) continue;
if(con->value.uint64v > NC_MAX_INT64) {forceuint64=1; break;}
}
@ -879,10 +945,9 @@ inferattributetype1(Datasrc* src)
return NC_UINT64;
/* Compute: are there any negative constants? */
srcreset(src);
hasneg = 0;
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
switch (con->nctype) {
case NC_BYTE : if(con->value.int8v < 0) {hasneg = 1;} break;
case NC_SHORT: if(con->value.int16v < 0) {hasneg = 1;} break;
@ -891,10 +956,9 @@ inferattributetype1(Datasrc* src)
}
/* Compute: inferred integer type */
srcreset(src);
result = NC_NAT;
while(srcmore(src)) {
NCConstant* con = srcnext(src);
for(i=0;i<datalistlen(adata);i++) {
NCConstant* con = datalistith(adata,i);
result = infertype(result,con->nctype,hasneg);
if(result == NC_NAT) break; /* something wrong */
}
@ -905,7 +969,6 @@ static void
inferattributetype(Symbol* asym)
{
Datalist* datalist;
Datasrc* src;
nc_type nctype;
ASSERT(asym->data != NULL);
datalist = asym->data;
@ -914,9 +977,7 @@ inferattributetype(Symbol* asym)
asym->typ.basetype = basetypefor(NC_CHAR);
return;
}
src = datalist2src(datalist);
nctype = inferattributetype1(src);
freedatasrc(src);
nctype = inferattributetype1(datalist);
if(nctype == NC_NAT) { /* Illegal attribute value list */
semerror(asym->lineno,"Non-simple list of values for untyped attribute: %s",fullname(asym));
return;
@ -1119,7 +1180,7 @@ thisunlim->name,
thisunlim->dim.declsize = unlimsize;
/*!lastunlim => data is list of sublists, recurse on each sublist*/
for(i=0;i<data->length;i++) {
NCConstant* con = data->data+i;
NCConstant* con = data->data[i];
if(con->nctype != NC_COMPOUND) {
semerror(con->lineno,"UNLIMITED dimension (other than first) must be enclosed in {}");
}
@ -1131,7 +1192,7 @@ thisunlim->name,
compute total number of characters */
length = 0;
for(i=0;i<data->length;i++) {
NCConstant* con = &data->data[i];
NCConstant* con = data->data[i];
switch (con->nctype) {
case NC_CHAR: case NC_BYTE: case NC_UBYTE:
length++;
@ -1189,7 +1250,7 @@ processunlimiteddims(void)
} else {
int j;
for(j=0;j<var->data->length;j++) {
NCConstant* con = var->data->data+j;
NCConstant* con = var->data->data[j];
if(con->nctype != NC_COMPOUND)
semerror(con->lineno,"UNLIMITED dimension (other than first) must be enclosed in {}");
else
@ -1210,3 +1271,41 @@ processunlimiteddims(void)
}
#endif
}
/* Rules for specifying the dataset name:
1. use -o name
2. use the datasetname from the .cdl file
3. use input cdl file name (with .cdl removed)
It would be better if there was some way
to specify the datasetname independently of the
file name, but oh well.
*/
static char*
createfilename(void)
{
char filename[4096];
filename[0] = '\0';
if(netcdf_name) { /* -o flag name */
strlcat(filename,netcdf_name,sizeof(filename));
} else { /* construct a usable output file name */
if (cdlname != NULL && strcmp(cdlname,"-") != 0) {/* cmd line name */
char* p;
strlcat(filename,cdlname,sizeof(filename));
/* remove any suffix and prefix*/
p = strrchr(filename,'.');
if(p != NULL) {*p= '\0';}
p = strrchr(filename,'/');
if(p != NULL) {
char* q = filename;
p++; /* skip the '/' */
while((*q++ = *p++));
}
} else {/* construct name from dataset name */
strlcat(filename,datasetname,sizeof(filename));
}
/* Append the proper extension */
strlcat(filename,binary_ext,sizeof(filename));
}
return strdup(filename);
}

110
ncgen/util.c
View File

@ -99,44 +99,58 @@ tztrim(
return;
}
#if 0
/* Assume bytebuffer contains pointers to char**/
void
reclaimattptrs(void* buf, long count)
{
int i;
char** ptrs = (char**)buf;
for(i=0;i<count;i++) {free((void*)ptrs[i]);}
for(i=0;i<count;i++) {efree((void*)ptrs[i]);}
}
#endif
static void
freeSpecialdata(Specialdata* data)
{
if(data == NULL) return;
reclaimdatalist(data->_Fillvalue);
if(data->_ChunkSizes)
efree(data->_ChunkSizes);
if(data->_FilterParams)
efree(data->_FilterParams);
efree(data);
}
void
freeSymbol(Symbol* sym)
{
#ifdef FIX
/* recurse first */
switch (sym->objectclass) {
case NG_VAR:
reclaimconstlist(vsym->var.data);
if(vsym->var.dims != NULL) efree(vsym->var.dims);
case NC_VAR:
freeSpecialdata(sym->var.special);
listfree(sym->var.attributes);
break;
case NG_ATT:
if(asym->att.basetype == primsymbols[NC_STRING])
reclaimattptrs(asym->att.data,asym->att.count);
else
efree(asym->att.data);
case NC_TYPE:
if(sym->typ.econst)
reclaimconstant(sym->typ.econst);
if(sym->typ._Fillvalue)
reclaimdatalist(sym->typ._Fillvalue);
break;
case NC_GRP:
if(sym->file.filename)
efree(sym->file.filename);
break;
case NG_GRP:
case NG_DIM:
case NG_TYP:
case NG_ENUM:
case NG_ECONST:
case NG_VLEN:
case NG_STRUCT:
case NG_FIELD:
case NG_OPAQUE:
default: break;
}
efree(sym->name);
/* Universal */
if(sym->name) efree(sym->name);
if(sym->fqn) efree(sym->fqn);
listfree(sym->prefix);
if(sym->data)
reclaimdatalist(sym->data);
listfree(sym->subnodes);
efree(sym);
#endif
}
char* nctypenames[17] = {
@ -523,67 +537,19 @@ getpadding(int offset, int alignment)
static void
reclaimSymbols(void)
{
Symbol* sym;
for(sym=symlist;sym;) {
Symbol* next = sym->next;
int i;
for(i=0;i<listlength(symlist);i++) {
Symbol* sym = listget(symlist,i);
freeSymbol(sym);
sym = next;
}
}
static void
constantFree(NCConstant* con)
{
switch(con->nctype) {
case NC_COMPOUND:
/* do nothing; ReclaimDatalists below will take care of the datalist */
break;
case NC_STRING:
if(con->value.stringv.len > 0 && con->value.stringv.stringv != NULL)
efree(con->value.stringv.stringv);
break;
case NC_OPAQUE:
if(con->value.opaquev.len > 0 && con->value.opaquev.stringv != NULL)
efree(con->value.opaquev.stringv);
break;
default:
break;
}
}
static void
reclaimDatalists(void)
{
Datalist* list;
NCConstant* con;
/* Step 1: free up the constant content of each datalist*/
for(list=alldatalists;list != NULL; list = list->next) {
if(list->data != NULL) { /* avoid multiple free attempts*/
int i;
for(i=0,con=list->data;i<list->length;i++,con++)
constantFree(con);
list->data = NULL;
}
}
/* Step 2: free up the datalist itself*/
for(list=alldatalists;list != NULL;) {
Datalist* current = list;
list = list->next;
efree(current);
}
}
void
cleanup()
{
reclaimDatalists();
reclaimSymbols();
}
/* compute the total n-dimensional size as 1 long array;
if stop == 0, then stop = dimset->ndims.
*/