mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-04-24 18:30:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix ncgen handling of big data sections

Browse Source 
The current ncgen does not properly handle very large
data sections. Apparently this is very uncommon because
it was only discovered in testing the new zarr code.

The fix required a new approach to processing data sections.
Unfortunately, the resulting ncgen is slower than before
but at least it is, I think, now correct.

The added test cases are in libnczarr, and so will
not show up until that is incorporated into master.

Note also that fortran code generation changed, but
has not been tested here.

Misc. Changes
1. Cleanup error handling in ncgen -lc and -lb output
2. Cleanup Makefiles for ncgen to remove unused code
3. Added a program, ncgen/ncdumpchunks, to print
   the data for a .nc file on a per-chunk format.
4. Made the XGetOpt change in PR https://github.com/Unidata/netcdf-c/pull/1694
   for ncdump/ncvalidator
This commit is contained in:
Dennis Heimbigner 2020-05-14 11:20:46 -06:00
parent 9e5f08d0e1
commit 68a98f6e81
49 changed files with 2762 additions and 2591 deletions

4
RELEASE_NOTES.md

@ -7,6 +7,10 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.0 - TBD
* [Bug Fix] IMPORTANT: Ncgen was not properly handling large data
sections and was storing data in incorrect order. For small data sections
is was correct. The value of "large" has not been determined, so
users should verify the output of ncgen.
* [Enhancement] When a filter is applied twice with different
parameters, then the second set is used for writing the dataset
[https://github.com/Unidata/netcdf-c/issues/1713].

9
libdap4/d4curlfunctions.c

@ -26,19 +26,12 @@
#endif
#endif
#define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != NC_NOERR) {goto done;}}
#define CHECK(state,flag,value) {if(set_curlopt(state,flag,(void*)value) != NC_NOERR) {goto done;}}
/* forward */
static int set_curlflag(NCD4INFO*, int flag);
static int set_curlopt(NCD4INFO*, int flag, void* value);
static int
check(NCD4INFO* info, int flag, void* value)
{
int ret = set_curlopt(info,flag,value);
return THROW(ret);
}
/*
Set a specific curl flag; primary wrapper for curl_easy_setopt
*/

3
libdap4/d4includes.h

@ -15,6 +15,9 @@
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <curl/curl.h>

30
nc_test4/ref_bzip2.c

@ -7,14 +7,16 @@ static size_t var_chunksizes[4] = {4, 4, 4, 4} ;
static unsigned int var_0_filterparams[1] = {9U} ;
void
check_err(const int stat, const int line, const char *file) {
check_err(const int stat, int line, const char* file, const char* func) {
if (stat != NC_NOERR) {
(void)fprintf(stderr,"line %d of %s: %s\n", line, file, nc_strerror(stat));
(void)fprintf(stderr,"line %d of %s.%s: %s\n", line, file, func, nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
#define CHECK_ERR(err) check_err(err, __LINE__, __FILE__, __func__)
int
main() {/* create bzip2.nc */
@ -47,18 +49,18 @@ main() {/* create bzip2.nc */
/* enter define mode */
stat = nc_create("bzip2.nc", NC_CLOBBER|NC_NETCDF4, &ncid);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
bzip2_grp = ncid;
/* define dimensions */
stat = nc_def_dim(bzip2_grp, "dim0", dim0_len, &dim0_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_dim(bzip2_grp, "dim1", dim1_len, &dim1_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_dim(bzip2_grp, "dim2", dim2_len, &dim2_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_dim(bzip2_grp, "dim3", dim3_len, &dim3_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* define variables */
@ -67,17 +69,17 @@ main() {/* create bzip2.nc */
var_dims[2] = dim2_dim;
var_dims[3] = dim3_dim;
stat = nc_def_var(bzip2_grp, "var", NC_FLOAT, RANK_var, var_dims, &var_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(bzip2_grp, var_id, NC_CHUNKED, var_chunksizes);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_fill(bzip2_grp, var_id, NC_NOFILL, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_filter(bzip2_grp, var_id, 307, 1, var_0_filterparams);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* leave define mode */
stat = nc_enddef (bzip2_grp);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* assign variable data */
@ -86,11 +88,11 @@ main() {/* create bzip2.nc */
size_t var_startset[4] = {0, 0, 0, 0} ;
size_t var_countset[4] = {4, 4, 4, 4};
stat = nc_put_vara(bzip2_grp, var_id, var_startset, var_countset, var_data);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
}
stat = nc_close(bzip2_grp);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
return 0;
}

19
ncdump/nciter.c

@ -21,8 +21,8 @@ nc_blkio_init(size_t bufsize, /* size in bytes of in-memory copy buffer */
size_t value_size, /* size in bytes of each variable element */
int rank, /* number of dimensions for variable */
int chunked, /* 1 if variable is chunked, 0 otherwise */
nciter_t *iter /* returned iteration state, don't mess with it */
) {
nciter_t *iter) /* returned iteration state, don't mess with it */
{
int stat = NC_NOERR;
int i;
long long prod;
@ -73,9 +73,9 @@ nc_blkio_init(size_t bufsize, /* size in bytes of in-memory copy buffer */
* value. Wouldn't be needed if nc_inq_type() was a netCDF-3 function
* too. */
static int
inq_value_size(int igrp, nc_type vartype, size_t *value_sizep) {
inq_value_size(int igrp, nc_type vartype, size_t *value_sizep)
{
int stat = NC_NOERR;
#ifdef USE_NETCDF4
NC_CHECK(nc_inq_type(igrp, vartype, NULL, value_sizep));
#else
@ -174,8 +174,8 @@ up_start_by_chunks(
int
nc_get_iter(int ncid,
int varid,
size_t bufsize, /* size in bytes of memory buffer */
nciter_t **iterpp /* returned opaque iteration state */)
size_t bufsize, /* size in bytes of memory buffer */
nciter_t **iterpp) /* returned opaque iteration state */
{
int stat = NC_NOERR;
nciter_t *iterp;
@ -239,8 +239,8 @@ nc_get_iter(int ncid,
size_t
nc_next_iter(nciter_t *iter, /* returned opaque iteration state */
size_t *start, /* returned start vector for next vara call */
size_t *count /* returned count vector for next vara call */
) {
size_t *count) /* returned count vector for next vara call */
{
int i;
/* Note: special case for chunked variables is just an
* optimization, the contiguous code below is OK even
@ -307,7 +307,8 @@ nc_next_iter(nciter_t *iter, /* returned opaque iteration state */
/* Free iterator and its internally allocated memory */
int
nc_free_iter(nciter_t *iterp) {
nc_free_iter(nciter_t *iterp)
{
if(iterp->dimsizes)
free(iterp->dimsizes);
if(iterp->chunksizes)

2
ncdump/ncvalidator.c

@ -73,7 +73,7 @@ THIS SOFTWARE.
#include <unistd.h> /* read() getopt() */
#endif
#ifdef _WIN32
#ifdef _MSC_VER
#include <io.h>
#define snprintf _snprintf
#include "XGetopt.h"

601
ncdump/ref_ctest.c

File diff suppressed because it is too large Load Diff

601
ncdump/ref_ctest64.c

File diff suppressed because it is too large Load Diff

92
ncdump/ref_ctest_special_atts_4.c

@ -21,14 +21,16 @@ static size_t var4_chunksizes[3] = {6, 7, 8} ;
static size_t var5_chunksizes[1] = {6} ;
void
check_err(const int stat, const int line, const char *file) {
check_err(const int stat, int line, const char* file, const char* func) {
if (stat != NC_NOERR) {
(void)fprintf(stderr,"line %d of %s: %s\n", line, file, nc_strerror(stat));
(void)fprintf(stderr,"line %d of %s.%s: %s\n", line, file, func, nc_strerror(stat));
fflush(stderr);
exit(1);
}
}
#define CHECK_ERR(err) check_err(err, __LINE__, __FILE__, __func__)
int
main() {/* create ref_tst_special_atts.nc */
@ -79,112 +81,112 @@ main() {/* create ref_tst_special_atts.nc */
/* enter define mode */
stat = nc_create("ref_tst_special_atts.nc", NC_CLOBBER|NC_NETCDF4, &ncid);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
tst_special_atts_grp = ncid;
stat = nc_def_compound(tst_special_atts_grp, sizeof(obs_t), "obs_t", &obs_t_typ); check_err(stat,__LINE__,__FILE__);
stat = nc_def_compound(tst_special_atts_grp, sizeof(obs_t), "obs_t", &obs_t_typ); CHECK_ERR(stat);
{
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "day", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_day), NC_BYTE); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "elev", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_elev), NC_SHORT); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "count", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_count), NC_INT); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "relhum", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_relhum), NC_FLOAT); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "time", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_time), NC_DOUBLE); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "category", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_category), NC_UBYTE); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "id", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_id), NC_USHORT); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "particularity", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_particularity), NC_UINT); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "attention_span", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_attention_span), NC_INT64); check_err(stat,__LINE__,__FILE__);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "day", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_day), NC_BYTE); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "elev", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_elev), NC_SHORT); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "count", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_count), NC_INT); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "relhum", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_relhum), NC_FLOAT); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "time", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_time), NC_DOUBLE); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "category", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_category), NC_UBYTE); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "id", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_id), NC_USHORT); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "particularity", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_particularity), NC_UINT); CHECK_ERR(stat);
stat = nc_insert_compound(tst_special_atts_grp, obs_t_typ, "attention_span", NC_COMPOUND_OFFSET(obs_t,obs_t_PERIOD_attention_span), NC_INT64); CHECK_ERR(stat);
}
/* define dimensions */
stat = nc_def_dim(tst_special_atts_grp, "dim1", dim1_len, &dim1_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_dim(tst_special_atts_grp, "dim2", dim2_len, &dim2_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_dim(tst_special_atts_grp, "dim3", dim3_len, &dim3_dim);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* define variables */
var1_dims[0] = dim1_dim;
stat = nc_def_var(tst_special_atts_grp, "var1", NC_INT, RANK_var1, var1_dims, &var1_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var1_id, NC_CONTIGUOUS, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var1_id, NC_ENDIAN_LITTLE);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
var2_dims[0] = dim1_dim;
var2_dims[1] = dim2_dim;
stat = nc_def_var(tst_special_atts_grp, "var2", NC_INT, RANK_var2, var2_dims, &var2_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var2_id, NC_CHUNKED, var2_chunksizes);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_fletcher32(tst_special_atts_grp, var2_id, 1);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var2_id, NC_ENDIAN_BIG);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
var3_dims[0] = dim1_dim;
var3_dims[1] = dim2_dim;
var3_dims[2] = dim3_dim;
stat = nc_def_var(tst_special_atts_grp, "var3", NC_INT, RANK_var3, var3_dims, &var3_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var3_id, NC_CHUNKED, var3_chunksizes);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_deflate(tst_special_atts_grp, var3_id, NC_NOSHUFFLE, 1, 2);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var3_id, NC_ENDIAN_LITTLE);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
var4_dims[0] = dim1_dim;
var4_dims[1] = dim2_dim;
var4_dims[2] = dim3_dim;
stat = nc_def_var(tst_special_atts_grp, "var4", NC_INT, RANK_var4, var4_dims, &var4_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var4_id, NC_CHUNKED, var4_chunksizes);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_deflate(tst_special_atts_grp, var4_id, NC_SHUFFLE, 1, 2);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var4_id, NC_ENDIAN_LITTLE);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_fill(tst_special_atts_grp, var4_id, NC_NOFILL, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
var5_dims[0] = dim1_dim;
stat = nc_def_var(tst_special_atts_grp, "var5", obs_t_typ, RANK_var5, var5_dims, &var5_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var5_id, NC_CHUNKED, var5_chunksizes);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_fletcher32(tst_special_atts_grp, var5_id, 1);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_deflate(tst_special_atts_grp, var5_id, NC_SHUFFLE, 1, 2);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_fill(tst_special_atts_grp, var5_id, NC_NOFILL, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
var6_dims[0] = dim1_dim;
stat = nc_def_var(tst_special_atts_grp, "var6", NC_INT, RANK_var6, var6_dims, &var6_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var6_id, NC_COMPACT, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var6_id, NC_ENDIAN_LITTLE);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var(tst_special_atts_grp, "var7", NC_INT, RANK_var7, 0, &var7_id);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_chunking(tst_special_atts_grp, var7_id, NC_COMPACT, NULL);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
stat = nc_def_var_endian(tst_special_atts_grp, var7_id, NC_ENDIAN_LITTLE);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* leave define mode */
stat = nc_enddef (tst_special_atts_grp);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
/* assign variable data */
stat = nc_close(tst_special_atts_grp);
check_err(stat,__LINE__,__FILE__);
CHECK_ERR(stat);
return 0;
}

1
ncdump/tst_nccopy3.sh

@ -34,7 +34,6 @@ fi
echo "*** Testing netCDF-3 features of nccopy on ncdump/*.nc files"
for i in $TESTFILES ; do
echo "*** Testing nccopy $i.nc nccopy3_copy_of_$i.nc ..."
ls -l $i.nc
${NCCOPY} $i.nc nccopy3_copy_of_$i.nc
${NCDUMP} -n nccopy3_copy_of_$i $i.nc > tmp_tst_nccopy3.cdl
${NCDUMP} nccopy3_copy_of_$i.nc > nccopy3_copy_of_$i.cdl

32
ncgen/CMakeLists.txt

@ -9,13 +9,22 @@ IF(BUILD_SHARED_LIBS AND WIN32)
remove_definitions(-DDLL_NETCDF)
ENDIF()
SET(ncgen_FILES generate.c main.c cdata.c bindata.c genchar.c cvt.c data.c debug.c escapes.c genc.c genbin.c generr.c genlib.c getfill.c odom.c semantics.c ncgeny.c dump.c util.c bytebuffer.c list.c genf77.c f77data.c genj.c jdata.c nc_iter.c ncgen.h)
SET(ncgen_FILES bindata.c bytebuffer.c cdata.c cvt.c data.c
debug.c dump.c escapes.c f77data.c genbin.c
genc.c genchar.c generate.c generr.c genf77.c
genj.c genlib.c getfill.c jdata.c list.c
main.c ncgeny.c semantics.c
util.c bytebuffer.h data.h debug.h dump.h
generate.h generr.h genlib.h includes.h list.h
ncgen.h ncgeny.h util.h)
# Obsolete
SET(OBSOLETE odom.c odom.h jdatastd.c jdatajni.c genjni.c cdfdata.c cmldata.c)
# don't add the automatically determined parts of the RPATH
# which point to directories outside the build tree to the install RPATH
SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
IF(USE_X_GETOPT)
SET(ncgen_FILES ${ncgen_FILES} XGetopt.c)
ENDIF()
@ -23,6 +32,25 @@ ENDIF()
ADD_EXECUTABLE(ncgen ${ncgen_FILES})
TARGET_LINK_LIBRARIES(ncgen netcdf ${ALL_TLL_LIBS})
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
IF(USE_NETCDF4)
SET(ncdumpchunks_FILES ncdumpchunks.c)
ADD_EXECUTABLE(ncdumpchunks ${ncdumpchunks_FILES})
TARGET_LINK_LIBRARIES(ncdumpchunks netcdf ${ALL_TLL_LIBS})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG
${CMAKE_CURRENT_BINARY_DIR})
SET_TARGET_PROPERTIES(ncdumpchunks PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE
${CMAKE_CURRENT_BINARY_DIR})
IF(MSVC)
SET_TARGET_PROPERTIES(ncdumpchunks
PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
)
ENDIF()
ENDIF()
####
# We have to do a little tweaking
# to remove the Release/ and Debug/ directories

25
ncgen/Makefile.am

@ -9,13 +9,24 @@ LDADD = ${top_builddir}/liblib/libnetcdf.la
# Build ncgen from the listed sources.
bin_PROGRAMS = ncgen
ncgen_SOURCES=generate.c main.c cdata.c bindata.c genchar.c cvt.c data.c debug.c \
escapes.c genc.c genbin.c generr.c genlib.c getfill.c odom.c \
semantics.c dump.c util.c bytebuffer.c list.c data.h \
debug.h generr.h genlib.h includes.h ncgen.h odom.h dump.h \
util.h bytebuffer.h list.h genf77.c f77data.c genj.c jdata.c nc_iter.h \
nc_iter.c \
ncgeny.c ncgeny.h
ncgen_SOURCES = bindata.c bytebuffer.c cdata.c cvt.c data.c \
debug.c dump.c escapes.c f77data.c genbin.c \
genc.c genchar.c generate.c generr.c genf77.c \
genj.c genlib.c getfill.c jdata.c list.c \
main.c ncgeny.c semantics.c \
util.c bytebuffer.h data.h debug.h dump.h \
generate.h generr.h genlib.h includes.h list.h \
ncgen.h ncgeny.h util.h
# Obsolete
OBSOLETE = odom.c odom.h jdatastd.c jdatajni.c genjni.c cdfdata.c cmldata.c
# Given a netcdf4 file, dump the actual chunk contents.
# Used to validate nczarr chunking code.
if USE_HDF5
noinst_PROGRAMS = ncdumpchunks
ncdumpchunks_SOURCES = ncdumpchunks.c
endif
# This is the man page.
man_MANS = ncgen.1

1
ncgen/bindata.c

@ -4,7 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#include "nclog.h"
#ifdef ENABLE_BINARY

1
ncgen/cdata.c

@ -4,7 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#ifdef ENABLE_C

370
ncgen/data.c

@ -26,7 +26,7 @@ List* alldatalists = NULL;
NCConstant nullconstant;
NCConstant fillconstant;
Datalist nildatalist; /* to support NIL keyword */
Datalist* filldatalist;
Bytebuffer* codebuffer;
Bytebuffer* codetmp;
@ -80,7 +80,7 @@ list2const(Datalist* list)
NCConstant* con = nullconst();
ASSERT(list != NULL);
con->nctype = NC_COMPOUND;
con->lineno = list->data[0]->lineno;
if(!list->readonly) con->lineno = list->data[0]->lineno;
setconstlist(con,list);
con->filled = 0;
return con;
@ -198,43 +198,6 @@ freeconstant(NCConstant* con, int shallow)
/**************************************************/
#if 0
Datalist*
datalistclone(Datalist* dl)
{
int i;
Datalist* clone = builddatalist(dl->length);
for(i=0;i<dl->length;i++) {
clone->data[i] = cloneconstant(dl->data[i]);
}
return clone;
}
Datalist*
datalistappend(Datalist* dl, NCConstant* con)
{
NCConstant** vector;
ASSERT(dl != NULL);
if(con == NULL) return dl;
vector = (NCConstant**)erealloc(dl->data,sizeof(NCConstant*)*(dl->length+1));
if(vector == NULL) return NULL;
vector[dl->length] = cloneconstant(con);
dl->length++;
dl->data = vector;
return dl;
}
Datalist*
datalistreplace(Datalist* dl, unsigned int index, NCConstant* con)
{
ASSERT(dl != NULL);
ASSERT(index < dl->length);
ASSERT(con != NULL);
dl->data[index] = cloneconstant(con);
return dl;
}
#endif
int
datalistline(Datalist* ds)
{
@ -584,18 +547,28 @@ indented(int n)
return indentation;
}
void
dlsetalloc(Datalist* dl, size_t need)
{
NCConstant** newdata = NULL;
if(dl->readonly) abort();
if(dl->alloc < need) {
newdata = (NCConstant**)ecalloc(need*sizeof(NCConstant*));
if(dl->length > 0)
memcpy(newdata,dl->data,sizeof(NCConstant*)*dl->length);
dl->alloc = need;
nullfree(dl->data);
dl->data = newdata;
}
}
void
dlextend(Datalist* dl)
{
size_t newalloc;
NCConstant** newdata = NULL;
if(dl->readonly) abort();
newalloc = (dl->alloc > 0?2*dl->alloc:2);
newdata = (NCConstant**)ecalloc(newalloc*sizeof(NCConstant*));
if(dl->length > 0)
memcpy(newdata,dl->data,sizeof(NCConstant*)*dl->length);
dl->alloc = newalloc;
nullfree(dl->data);
dl->data = newdata;
dlsetalloc(dl,newalloc);
}
@ -623,6 +596,7 @@ builddatalist(int initial)
void
dlappend(Datalist* dl, NCConstant* constant)
{
if(dl->readonly) abort();
if(dl->length >= dl->alloc)
dlextend(dl);
dl->data[dl->length++] = (constant);
@ -635,6 +609,40 @@ dlset(Datalist* dl, size_t pos, NCConstant* constant)
dl->data[pos] = (constant);
}
NCConstant*
dlremove(Datalist* dl, size_t pos)
{
int i;
NCConstant* con = NULL;
ASSERT(dl->length > 0 && pos < dl->length);
con = dl->data[pos];
for(i=pos+1;i<dl->length;i++)
dl->data[i-1] = dl->data[i];
dl->length--;
return con;
}
void
dlinsert(Datalist* dl, size_t pos, Datalist* insertion)
{
int i;
int len1 = datalistlen(dl);
int len2 = datalistlen(insertion);
int delta = len1 - pos;
dlsetalloc(dl,len2+len1+1);
/* move contents of dl up to make room for insertion */
if(delta > 0)
memmove(&dl->data[pos+len2],&dl->data[pos],delta*sizeof(NCConstant*));
dl->length += len2;
for(i=0;i<len2;i++) {
NCConstant* con = insertion->data[i];
con = cloneconstant(con);
dl->data[pos+i] = con;
}
}
/* Convert a datalist to a compound constant */
NCConstant*
builddatasublist(Datalist* dl)
@ -649,6 +657,23 @@ builddatasublist(Datalist* dl)
}
/* Convert a subsequence of a datalist to its own datalist */
Datalist*
builddatasubset(Datalist* dl, size_t start, size_t count)
{
Datalist* subset;
if(dl == NULL || start >= datalistlen(dl)) return NULL;
if((start + count) > datalistlen(dl))
count = (datalistlen(dl) - start);
subset = (Datalist*)ecalloc(sizeof(Datalist));
subset->readonly = 1;
subset->length = count;
subset->alloc = count;
subset->data = &dl->data[start];
return subset;
}
/* Deep copy */
Datalist*
clonedatalist(Datalist* dl)
@ -666,30 +691,12 @@ clonedatalist(Datalist* dl)
con = cloneconstant(con);
dlappend(newdl,con);
}
#if 0
newdl->vlen = dl->vlen;
#endif
newdl->readonly = dl->readonly;
return newdl;
}
/* recursive helpers */
#if 0
static int
isdup(Datalist* dl)
{
int i;
size_t limit = listlength(alldatalists);
for(i=0;i<limit;i++) {
Datalist* di = listget(alldatalists,i);
if(di == dl) return 1;
}
return 0;
}
#endif
void
reclaimconstant(NCConstant* con)
{
@ -726,36 +733,33 @@ reclaimdatalist(Datalist* list)
{
int i;
if(list == NULL) return;
if(list->data != NULL) {
for(i=0;i<list->length;i++) {
NCConstant* con = list->data[i];
if(con != NULL) reclaimconstant(con);
}
efree(list->data);
list->data = NULL;
}
efree(list);
if(!list->readonly) {
if(list->data != NULL) {
for(i=0;i<list->length;i++) {
NCConstant* con = list->data[i];
if(con != NULL) reclaimconstant(con);
}
}
}
freedatalist(list);
}
/* Like reclaimdatalist, but do not try to reclaim contained constants */
void
freedatalist(Datalist* list)
{
if(list == NULL) return;
if(!list->readonly) {
efree(list->data);
list->data = NULL;
}
efree(list);
}
void
reclaimalldatalists(void)
{
int i;
#if 0
int j;
/* Remove duplicates */
for(i=0;i<listlength(alldatalists);i++) {
Datalist* di = listget(alldatalists,i);
if(di == NULL) continue;
for(j=i;j<listlength(alldatalists);j++) {
Datalist* dj = listget(alldatalists,j);
if(dj == di) {
listset(alldatalists,j,NULL);
fprintf(stderr,"XXX\n");
}
}
}
#endif
for(i=0;i<listlength(alldatalists);i++) {
Datalist* di = listget(alldatalists,i);
if(di != NULL)
@ -765,179 +769,33 @@ fprintf(stderr,"XXX\n");
alldatalists = NULL;
}
/* Obsolete */
#if 0
/* return 1 if the next element in the datasrc is compound*/
int
issublist(Datasrc* datasrc) {return istype(datasrc,NC_COMPOUND);}
/* return 1 if the next element in the datasrc is a string*/
int
isstring(Datasrc* datasrc) {return istype(datasrc,NC_STRING);}
/* return 1 if the next element in the datasrc is a fill value*/
int
isfillvalue(Datasrc* datasrc)
{
return srcpeek(datasrc) == NULL || istype(datasrc,NC_FILLVALUE);
}
/* return 1 if the next element in the datasrc is nc_type*/
int
istype(Datasrc* datasrc , nc_type nctype)
{
NCConstant* ci = srcpeek(datasrc);
if(ci != NULL && ci->nctype == nctype) return 1;
return 0;
}
/**************************************************/
void
freedatasrc(Datasrc* src)
{
efree(src);
}
Datasrc*
allocdatasrc(void)
{
Datasrc* src;
src = ecalloc(sizeof(Datasrc));
src->data = NULL;
src->index = 0;
src->length = 0;
src->prev = NULL;
return src;
}
Datasrc*
datalist2src(Datalist* list)
{
Datasrc* src;
ASSERT(list != NULL);
src = allocdatasrc();
src->data = list->data;
src->index = 0;
src->length = list->length;
DUMPSRC(src,"#");
return src;
}
Datasrc*
const2src(NCConstant* con)
{
Datasrc* src;
ASSERT(con != NULL);
src = allocdatasrc();
src->data = emalloc(sizeof(NCConstant*));
src->data[0] = con;
src->index = 0;
src->length = 1;
DUMPSRC(src,"#");
return src;
}
NCConstant*
srcpeek(Datasrc* ds)
{
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return ds->data[ds->index];
if(ds->spliced)
return srcpeek(ds->prev);
return NULL;
}
void
srcreset(Datasrc* ds)
{
ds->index = 0;
}
NCConstant*
srcnext(Datasrc* ds)
{
DUMPSRC(ds,"!");
if(ds == NULL) return NULL;
if(ds->index < ds->length)
return ds->data[ds->index++];
if(ds->spliced) {
srcpop(ds);
return srcnext(ds);
}
return NULL;
}
int
srcmore(Datasrc* ds)
{
if(ds == NULL) return 0;
if(ds->index < ds->length) return 1;
if(ds->spliced) return srcmore(ds->prev);
return 0;
}
int
srcline(Datasrc* ds)
{
int index = ds->index;
int len = ds->length;
/* pick closest available entry*/
if(len == 0) return 0;
if(index >= len) index = len-1;
return ds->data[index]->lineno;
}
void
srcpush(Datasrc* src)
static void
flattenR(Datalist* result, Datalist* data, int rank, int depth)
{
int i;
NCConstant* con;
ASSERT(src != NULL);
con = srcnext(src);
ASSERT(con->nctype == NC_COMPOUND);
srcpushlist(src,con->value.compoundv);
}
void
srcpushlist(Datasrc* src, Datalist* dl)
{
Datasrc* newsrc;
ASSERT(src != NULL && dl != NULL);
newsrc = allocdatasrc();
*newsrc = *src;
src->prev = newsrc;
src->index = 0;
src->data = dl->data;
src->length = dl->length;
DUMPSRC(src,">!");
}
void
srcpop(Datasrc* src)
{
if(src != NULL) {
Datasrc* prev = src->prev;
*src = *prev;
freedatasrc(prev);
if(rank == depth) return;
if(datalistlen(data) == 0) return;
for(i=0;i<datalistlen(data);i++) {
con = datalistith(data,i);
if(depth < rank - 1) {
/* Is this is a char list, then we might have short depth */
if(islistconst(con))
flattenR(result,compoundfor(con),rank,depth+1);
else
dlappend(result,con);
} else { /* depth == rank -1, last dimension */
dlappend(result,con);
}
}
DUMPSRC(src,"<");
}
void
srcsplice(Datasrc* ds, Datalist* list)
/* Produce a new list that is the concat of all the leaf constants */
Datalist*
flatten(Datalist* list,int rank)
{
srcpushlist(ds,list);
ds->spliced = 1;
Datalist* result = builddatalist(0);
flattenR(result,list,rank,0);
return result;
}
void
srcsetfill(Datasrc* ds, Datalist* list)
{
if(ds->index >= ds->length) PANIC("srcsetfill: no space");
if(ds->data[ds->index]->nctype != NC_FILLVALUE) PANIC("srcsetfill: not fill");
ds->data[ds->index]->nctype = NC_COMPOUND;
setconstlist(ds->data[ds->index],list);
}
#endif /*0*/

125
ncgen/data.h

@ -14,12 +14,12 @@
/* nmemonics*/
#define TOPLEVEL 1
#define DEEP 0
/* Forward types */
struct Datalist;
struct Symbol;
struct Dimset;
typedef struct Generator Generator;
/* any one possible value*/
typedef union Constvalue {
@ -50,6 +50,7 @@ typedef union Constvalue {
typedef struct NCConstant {
nc_type nctype; /* NC_INT,... */
nc_type subtype; /* NC_DIM | NC_NAT */
int lineno;
Constvalue value;
int filled; /* was this originally NC_FILLVALUE? */
@ -62,35 +63,8 @@ typedef struct Datalist {
NCConstant** data; /* actual list of constants constituting the datalist*/
/* Track various values associated with the datalist*/
/* (used to be in Constvalue.compoundv)*/
#if 0
struct Vlen {
struct Symbol* schema; /* type/var that defines structure of this*/
unsigned int count; /* # of vlen basetype instances*/
unsigned int uid; /* unique id for NC_VLEN*/
} vlen;
#endif
} Datalist;
/* Define a structure to track
location of current read point in the Datalist sequence
In effect, we are parsing the data sequence.
Push and pop of data sources is supported (see srcpush() below).*/
typedef struct Datasrc {
NCConstant** data; /* duplicate pointer; so do not free.*/
int index;
int length;
int spliced; /* Was this list spliced into our parent ? */
struct Datasrc* prev; /* linked list for debugging */
} Datasrc;
#if 0
/* Define a holder for passing a start/count array */
struct Vlendata {
char* data;
unsigned long count;
};
extern struct Vlendata* vlendata;
#endif
extern List* alldatalists;
@ -98,33 +72,41 @@ extern List* alldatalists;
extern Datalist* builddatalist(int initialize);
extern void capture(Datalist* dl);
extern void dlappend(Datalist*, NCConstant*);
extern void dlinsert(Datalist* dl, size_t pos, Datalist* insertion);
extern void dlset(Datalist*, size_t, NCConstant*);
extern NCConstant* dlremove(Datalist*, size_t);
extern NCConstant* builddatasublist(Datalist* dl);
extern Datalist* builddatasubset(Datalist* dl, size_t start, size_t count);
extern void dlextend(Datalist* dl);
extern void dlsetalloc(Datalist* dl, size_t newalloc);
extern void dlsetalloc(Datalist* dl,size_t);
extern Datalist* clonedatalist(Datalist* dl);
extern void reclaimalldatalists(void);
extern void reclaimdatalist(Datalist*);
extern void reclaimconstant(NCConstant*);
extern void freedatalist(Datalist* list);
extern Datalist* flatten(Datalist* list,int rank);
int datalistline(Datalist*);
#define datalistith(dl,i) ((dl)==NULL?NULL:((i) >= (dl)->length?NULL:(dl)->data[i]))
#define datalistlen(dl) ((dl)==NULL?0:(dl)->length)
#define datalistclear(dl) {if((dl)!=NULL) {dl->length=0;}}
NCConstant* list2const(Datalist*);
Datalist* const2list(NCConstant* con);
int isstringable(nc_type nctype);
#define islistconst(con) ((con)!=NULL && (con)->nctype == NC_COMPOUND)
#define islistconst(con) ((con)!=NULL && ((con)->nctype == NC_COMPOUND))
#define isfillconst(con) ((con)!=NULL && (con)->nctype == NC_FILLVALUE)
#define constline(con) (con==NULL?0:(con)->lineno)
#define consttype(con) (con==NULL?NC_NAT:(con)->nctype)
#define isnilconst(con) ((con)!=NULL && (con)->nctype == NC_NIL)
#define compoundfor(con) ((con)==NULL?NULL:(con)->value.compoundv)
#define compoundfor(con) ((con)==NULL?NULL:(con)->value.compoundv)
#define setsubtype(con,type) {if((con)!=NULL){(con)->subtype=(type);}}
NCConstant* emptycompoundconst(int lineno);
NCConstant* emptystringconst(int);
NCConstant* cloneconstant(NCConstant* con); /* deep clone*/
@ -132,90 +114,11 @@ void clearconstant(NCConstant* con); /* deep clear*/
#define freeconst(con) freeconstant(con,DEEP);
void freeconstant(NCConstant* con, int shallow);
void alignbuffer(struct NCConstant* prim, Bytebuffer* buf);
/* Code dump support procedures */
void bbindent(Bytebuffer*,const int);
void bbprintf(Bytebuffer*,const char *fmt, ...); /* append */
void bbprintf0(Bytebuffer*,const char *fmt, ...); /* clear, then append*/
/* Following dump to codebuffer */
void codeprintf(const char *fmt, ...);
void codedump(Bytebuffer*);
void codepartial(const char*);
void codeline(const char*);
void codelined(int n,const char*);
void codeflush(void); /* flush codebuffer to stdout */
void commify(Bytebuffer* buf);
char* word(char* p, Bytebuffer* buf);
/* Provide buffers for language based generators */
extern Bytebuffer* codebuffer; /* buffer over the std output */
extern Bytebuffer* stmt; /* single stmt text generation */
/* Aliases */
#define srcincr(src) srcnext(src)
#define srcget(src) srcpeek(src)
extern NCConstant* nullconst(void);
extern NCConstant nullconstant;
extern NCConstant fillconstant;
extern NCConstant nilconstant;
/* From genchar.c */
void gen_charattr(Datalist*, Bytebuffer*);
void gen_charseq(Datalist*, Bytebuffer*);
void gen_chararray(struct Dimset*, int, Datalist*, Bytebuffer*, Datalist* fillsrc);
typedef enum ListClass {
LISTDATA, LISTATTR, LISTVLEN, LISTCOMPOUND, LISTFIELDARRAY
} ListClass;
struct Generator {
void* globalstate; /* per-generator; per list state is in the method args where needed */
int (*charconstant)(Generator*,struct Symbol*,Bytebuffer*,...);
int (*constant)(Generator*,struct Symbol*,NCConstant*,Bytebuffer*,...);
int (*listbegin)(Generator*,struct Symbol*,void*,ListClass,size_t,Bytebuffer*,int*,...);
int (*list)(Generator*,struct Symbol*,void*,ListClass,int,size_t,Bytebuffer*,...);
int (*listend)(Generator*,struct Symbol*,void*,ListClass,int,size_t,Bytebuffer*,...);
int (*vlendecl)(Generator*,struct Symbol*,Bytebuffer*,int,size_t,...);
int (*vlenstring)(Generator*,struct Symbol*,Bytebuffer*,int*,size_t*,...);
};
extern int generator_getstate(Generator*,void**);
extern int generator_reset(Generator*,void*);
typedef int (*Writer)(Generator*,struct Symbol*,Bytebuffer*,int,const size_t*,const size_t*);
extern void generate_attrdata(struct Symbol*, Generator*, Writer writer, Bytebuffer*);
extern void generate_vardata(struct Symbol*, Generator*, Writer writer,Bytebuffer*);
extern void generate_basetype(struct Symbol*,NCConstant*,Bytebuffer*,Datalist*,Generator*);
/* Obsolete */
#if 0
Datasrc* datalist2src(Datalist* list);
Datasrc* const2src(NCConstant*);
void freedatasrc(Datasrc* src);
int issublist(Datasrc* src);
int isstring(Datasrc* src);
int isfillvalue(Datasrc* src);
int istype(Datasrc* src, nc_type);
void srcpush(Datasrc*);
void srcpushlist(Datasrc* src, Datalist* cmpd);
void srcpop(Datasrc*);
void srcsetfill(Datasrc* ds, Datalist* list);
NCConstant* srcnext(Datasrc*);
int srcmore(Datasrc*);
int srcline(Datasrc* ds);
void srcreset(Datasrc* ds);
#define srclen(s) ((s)==NULL?0:(s)->length)
#ifdef FASTDATASRC
#define srcpeek(ds) ((ds)==NULL || (ds)->index >= (ds)->max?NULL:(ds)->data+(ds)->index)
#else
NCConstant* srcpeek(Datasrc*);
#endif
#endif /*0*/
extern Datalist* filldatalist;
#endif /*DATA_H*/

3
ncgen/debug.h

@ -46,6 +46,9 @@ extern int ncgdebug;
extern void fdebug(const char *fmt, ...);
#define CHECK_ERR(x) check_err((x),__LINE__,__FILE__,__func__)
#define CHECK_ERR2(x,cl) check_err2((x),(cl),__LINE__,__FILE__,__func__)
#define PANIC(msg) assert(panic(msg))
#define PANIC1(msg,arg) assert(panic(msg,arg))
#define PANIC2(msg,arg1,arg2) assert(panic(msg,arg1,arg2))

56
ncgen/dump.c

@ -10,6 +10,9 @@
#define DEBUGSRC
#define MAXELEM 8
#define MAXDEPTH 4
/* Forward */
static void dumpdataprim(NCConstant*,Bytebuffer*);
@ -60,9 +63,9 @@ bufdump(Datalist* list, Bytebuffer* buf)
NCConstant* dp = *dpl;
switch (dp->nctype) {
case NC_COMPOUND:
bbCat(buf,"{");
if(dp->subtype == NC_DIM) bbCat(buf,"("); else bbCat(buf,"{");
bufdump(dp->value.compoundv,buf);
bbCat(buf,"}");
if(dp->subtype == NC_DIM) bbCat(buf,")"); else bbCat(buf,"}");
break;
case NC_ARRAY:
bbCat(buf,"[");
@ -195,7 +198,10 @@ dumpconstant1(NCConstant* con)
Bytebuffer* buf = bbNew();
bufdump(dl,buf);
/* fprintf(stderr,"(0x%lx){",(unsigned long)dl);*/
fprintf(stderr,"{%s}",bbDup(buf));
if(con->subtype == NC_DIM)
fprintf(stderr,"{%s}",bbDup(buf));
else
fprintf(stderr,"{%s}",bbDup(buf));
bbFree(buf);
} break;
case NC_STRING:
@ -255,47 +261,3 @@ dumpconstant1(NCConstant* con)
}
fflush(stderr);
}
#define MAXELEM 8
#define MAXDEPTH 4
#if 0
void
dumpsrc0(Datasrc* src,char* tag)
{
int i, count, index, depth;
depth = MAXDEPTH;
count = src->length;
index = src->index;
if(count > MAXELEM) count = MAXELEM;
if(index > count) index = count;
fprintf(stderr,"%s:: ",(tag?tag:""));
do {
fprintf(stderr,"[%d/%d]",src->index,src->length);
for(i=0;i<index;i++) {
fprintf(stderr," ");
dumpconstant1(src->data[i]);
}
fprintf(stderr,"^");
for(i=index;i<count;i++) {
fprintf(stderr," ");
dumpconstant1(src->data[i]);
}
if(count < src->length) fprintf(stderr,"...");
fprintf(stderr," | ");
src = src->prev;
} while(src != NULL && depth > 0);
if(src != NULL) fprintf(stderr,"---");
fprintf(stderr,"\n");
fflush(stderr);
}
void
dumpsrc(Datasrc* src,char* tag)
{
#ifndef DEBUGSRC
if(debug == 0) return;
#endif
dumpsrc0(src,tag);
}
#endif

1
ncgen/f77data.c

@ -4,7 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#ifdef ENABLE_F77

72
ncgen/genbin.c

@ -60,7 +60,7 @@ genbin_netcdf(void)
#endif
stat = nc_create(filename, cmode_modifier, &ncid);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
/* ncid created above is also root group*/
rootgroup->nc_id = ncid;
@ -72,7 +72,7 @@ genbin_netcdf(void)
Symbol* gsym = (Symbol*)listget(grpdefs,igrp);
if(gsym == rootgroup) continue; /* ignore root group*/
stat = nc_def_grp(gsym->container->nc_id,gsym->name,&gsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
#endif
@ -95,7 +95,7 @@ genbin_netcdf(void)
dsym->name,
(dsym->dim.isunlimited?NC_UNLIMITED:dsym->dim.declsize),
&dsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
}
@ -122,7 +122,7 @@ genbin_netcdf(void)
NULL,
&vsym->nc_id);
}
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
}
@ -154,12 +154,12 @@ genbin_netcdf(void)
if (nofill_flag) {
stat = nc_set_fill(rootgroup->nc_id, NC_NOFILL, 0);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
/* leave define mode */
stat = nc_enddef(rootgroup->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
if(!header_only) {
/* Load values into those variables with defined data */
@ -189,7 +189,7 @@ genbin_defineglobalspecials(void)
/* Watch out, this is a global Attribute */
format = kind_string(/*Main.*/format_flag);
stat = nc_put_att_text(rootgroup->nc_id,NC_GLOBAL,"_Format",strlen(format),format);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
#endif /*0*/
@ -197,7 +197,7 @@ static int
genbin_definespecialattributes(Symbol* var)
{
int stat = NC_NOERR;
Specialdata* special = var->var.special;
Specialdata* special = &var->var.special;
if(special->flags & _STORAGE_FLAG) {
if(special->_Storage == NC_CONTIGUOUS
|| special->_Storage == NC_COMPACT) {
@ -207,13 +207,13 @@ genbin_definespecialattributes(Symbol* var)
derror("NC_CHUNKED requested, but no chunksizes specified");
stat = nc_def_var_chunking(var->container->nc_id, var->nc_id, NC_CHUNKED, special->_ChunkSizes);
}
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
if(special->flags & _FLETCHER32_FLAG) {
stat = nc_def_var_fletcher32(var->container->nc_id,
var->nc_id,
special->_Fletcher32);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
if(special->flags & (_DEFLATE_FLAG | _SHUFFLE_FLAG)) {
stat = nc_def_var_deflate(var->container->nc_id,
@ -222,7 +222,7 @@ genbin_definespecialattributes(Symbol* var)
(special->_DeflateLevel >= 0?1:0),
(special->_DeflateLevel >= 0?special->_DeflateLevel
:0));
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
if(special->flags & _ENDIAN_FLAG) {
stat = nc_def_var_endian(var->container->nc_id,
@ -230,14 +230,14 @@ genbin_definespecialattributes(Symbol* var)
(special->_Endianness == NC_ENDIAN_LITTLE?
NC_ENDIAN_LITTLE
:NC_ENDIAN_BIG));
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
if(special->flags & _NOFILL_FLAG) {
stat = nc_def_var_fill(var->container->nc_id,
var->nc_id,
(special->_Fill?NC_FILL:NC_NOFILL),
NULL);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
if(special->flags & _FILTER_FLAG) {
int k;
@ -250,7 +250,7 @@ genbin_definespecialattributes(Symbol* var)
nfs->params
);
}
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
return stat;
}
@ -262,7 +262,7 @@ genbin_close(void)
{
int stat;
stat = nc_close(rootgroup->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
#ifdef USE_NETCDF4
@ -283,7 +283,7 @@ genbin_deftype(Symbol* tsym)
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
break;
case NC_ENUM:
{
@ -292,7 +292,7 @@ genbin_deftype(Symbol* tsym)
tsym->typ.basetype->nc_id,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
datum = bbNew();
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
@ -304,7 +304,7 @@ genbin_deftype(Symbol* tsym)
tsym->nc_id,
econst->name,
bbContents(datum));
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
bbFree(datum);
}
@ -314,14 +314,14 @@ genbin_deftype(Symbol* tsym)
tsym->name,
tsym->typ.basetype->nc_id,
&tsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
break;
case NC_COMPOUND:
stat = nc_def_compound(tsym->container->nc_id,
tsym->typ.size,
tsym->name,
&tsym->nc_id);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* efield = (Symbol*)listget(tsym->subnodes,i);
ASSERT(efield->subclass == NC_FIELD);
@ -349,7 +349,7 @@ genbin_deftype(Symbol* tsym)
efield->typ.dimset.ndims,
dimsizes);
}
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
break;
default: panic("definectype: unexpected type subclass");
@ -435,11 +435,11 @@ genbin_writevar(Generator* generator, Symbol* vsym, Bytebuffer* memory,
} else {
stat = nc_put_vara(vsym->container->nc_id, vsym->nc_id, start, count, data);
}
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
#if 0
/* Reclaim the data */
stat = ncaux_reclaim_data(vsym->container->nc_id, vsym->typ.basetype->nc_id, data, nelems);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
bbClear(memory); /* reclaim top-level memory */
#endif
return stat;
@ -468,7 +468,7 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
case NC_BYTE: {
signed char* data = (signed char*)bbContents(databuf);
stat = nc_put_att_schar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_CHAR: {
char* data = (char*)bbContents(databuf);
@ -482,27 +482,27 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
slen++;
}
stat = nc_put_att_text(grpid,varid,asym->name,slen,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_SHORT: {
short* data = (short*)bbContents(databuf);
stat = nc_put_att_short(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_INT: {
int* data = (int*)bbContents(databuf);
stat = nc_put_att_int(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_FLOAT: {
float* data = (float*)bbContents(databuf);
stat = nc_put_att_float(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_DOUBLE: {
double* data = (double*)bbContents(databuf);
stat = nc_put_att_double(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_STRING: {
const char** data;
@ -514,27 +514,27 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
case NC_UBYTE: {
unsigned char* data = (unsigned char*)bbContents(databuf);
stat = nc_put_att_uchar(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_USHORT: {
unsigned short* data = (unsigned short*)bbContents(databuf);
stat = nc_put_att_ushort(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_UINT: {
unsigned int* data = (unsigned int*)bbContents(databuf);
stat = nc_put_att_uint(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
case NC_INT64: {
long long* data = (long long*)bbContents(databuf);
stat = nc_put_att_longlong(grpid,varid,asym->name,typid,len,data);
check_err2(stat,asym->lineno,__LINE__,"ncgen");
CHECK_ERR2(stat,asym->lineno);
} break;
case NC_UINT64: {
unsigned long long* data = (unsigned long long*)bbContents(databuf);
stat = nc_put_att_ulonglong(grpid,varid,asym->name,typid,len,data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
} break;
default: PANIC1("genbin_defineattr: unexpected basetype: %d",basetype->typ.typecode);
}
@ -543,13 +543,13 @@ genbin_writeattr(Generator* generator, Symbol* asym, Bytebuffer* databuf,
data = (const char*)bbContents(databuf);
stat = nc_put_att(grpid,varid,asym->name,typid,
len,(void*)data);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
#ifdef GENDEBUG
{
char out[4096];
memset(out,0x77,sizeof(out));
stat = nc_get_att(grpid,varid,asym->name,&out);
check_err(stat,__LINE__,"ncgen");
CHECK_ERR(stat);
}
#endif
}

53
ncgen/genc.c

@ -5,7 +5,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#include <ctype.h> /* for isprint() */
#ifdef ENABLE_C
@ -96,7 +95,7 @@ genc_netcdf(void)
for(ivar=0;ivar<nvars;ivar++) {
Bytebuffer* tmp = bbNew();
Symbol* var = (Symbol*)listget(vardefs,ivar);
Specialdata* special = var->var.special;
Specialdata* special = &var->var.special;
if(special->flags & _CHUNKSIZES_FLAG) {
int i;
size_t* chunks = special->_ChunkSizes;
@ -142,14 +141,16 @@ genc_netcdf(void)
/* Now construct the main procedures*/
codeline("void");
codeline("check_err(const int stat, const int line, const char *file) {");
codeline("check_err(const int stat, int line, const char* file, const char* func) {");
codelined(1,"if (stat != NC_NOERR) {");
codelined(2,"(void)fprintf(stderr,\"line %d of %s: %s\\n\", line, file, nc_strerror(stat));");
codelined(2,"(void)fprintf(stderr,\"line %d of %s.%s: %s\\n\", line, file, func, nc_strerror(stat));");
codelined(2,"fflush(stderr);");
codelined(2,"exit(1);");
codelined(1,"}");
codeline("}");
codeline("");
codeline("#define CHECK_ERR(err) check_err(err, __LINE__, __FILE__, __func__)");
codeline("");
codeline("int");
bbprintf0(stmt,"%s() {/* create %s */\n", mainname, filename);
codedump(stmt);
@ -281,7 +282,7 @@ genc_netcdf(void)
bbprintf0(stmt," stat = nc_create(\"%s\", %s, &ncid);\n",
filename,cmode_string);
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
codeflush();
#ifdef USE_NETCDF4
@ -305,7 +306,7 @@ genc_netcdf(void)
groupncid(gsym->container),
gsym->name, groupncid(gsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
codeflush();
}
@ -341,7 +342,7 @@ genc_netcdf(void)
dimncid(dsym));
}
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
}
codeflush();
@ -375,7 +376,7 @@ genc_netcdf(void)
(dimset->ndims == 0?"0":poolcat(cname(vsym),"_dims")),
varncid(vsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
#ifdef USE_NETCDF4
genc_definespecialattributes(vsym);
#endif /*USE_NETCDF4*/
@ -411,14 +412,14 @@ genc_netcdf(void)
codelined(1,"/* don't initialize variables with fill values */");
bbindent(stmt,1);
bbprintf0(stmt,"stat = nc_set_fill(%s, NC_NOFILL, 0);",groupncid(rootgroup));
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
codeline("");
codelined(1,"/* leave define mode */");
bbprintf0(stmt," stat = nc_enddef (%s);\n",groupncid(rootgroup));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
codeflush();
if(!header_only) {
@ -450,7 +451,7 @@ genc_defineglobalspecials(void)
static void
genc_definespecialattributes(Symbol* vsym)
{
Specialdata* special = vsym->var.special;
Specialdata* special = &vsym->var.special;
if(usingclassic) return;
if(special->flags & _STORAGE_FLAG) {
const char* storage = NULL;
@ -473,7 +474,7 @@ genc_definespecialattributes(Symbol* vsym)
codedump(stmt);
}
codeline(");");
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
if(special->flags & _FLETCHER32_FLAG) {
bbprintf0(stmt,
@ -482,7 +483,7 @@ genc_definespecialattributes(Symbol* vsym)
varncid(vsym),
special->_Fletcher32);
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
if(special->flags & (_DEFLATE_FLAG | _SHUFFLE_FLAG)) {
bbprintf0(stmt,
@ -493,7 +494,7 @@ genc_definespecialattributes(Symbol* vsym)
(special->_DeflateLevel >= 0?1:0),
(special->_DeflateLevel >= 0?special->_DeflateLevel:0));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
if(special->flags & _ENDIAN_FLAG) {
bbprintf0(stmt,
@ -504,7 +505,7 @@ genc_definespecialattributes(Symbol* vsym)
:"NC_ENDIAN_BIG")
);
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
if(special->flags & _NOFILL_FLAG) {
bbprintf0(stmt,
@ -514,7 +515,7 @@ genc_definespecialattributes(Symbol* vsym)
(special->_Fill?"NC_FILL":"NC_NOFILL")
);
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
if(special->flags & _FILTER_FLAG) {
int k;
@ -535,7 +536,7 @@ genc_definespecialattributes(Symbol* vsym)
codedump(stmt);
}
codeline(");");
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
}
}
@ -546,7 +547,7 @@ genc_close(void)
{
bbprintf0(stmt,"%sstat = nc_close(%s);\n",indented(1),groupncid(rootgroup));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
#ifndef vms
codelined(1,"return 0;");
#else
@ -837,7 +838,7 @@ genc_deftype(Symbol* tsym)
tsym->name,
typencid(tsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
break;
case NC_ENUM:
codelined(1,"{");
@ -852,7 +853,7 @@ genc_deftype(Symbol* tsym)
tsym->name,
typencid(tsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
for(i=0;i<listlength(tsym->subnodes);i++) {
Symbol* econst = (Symbol*)listget(tsym->subnodes,i);
Bytebuffer* econststring = bbNew();
@ -880,7 +881,7 @@ genc_deftype(Symbol* tsym)
typencid(tsym->typ.basetype),
typencid(tsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
break;
case NC_COMPOUND:
bbprintf0(stmt,"%sstat = nc_def_compound(%s, sizeof(%s), \"%s\", &%s);",
@ -890,7 +891,7 @@ genc_deftype(Symbol* tsym)
escapifyname(tsym->name),
typencid(tsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
/* Generate the field dimension constants*/
codelined(1,"{");
for(i=0;i<listlength(tsym->subnodes);i++) {
@ -942,7 +943,7 @@ genc_deftype(Symbol* tsym)
typencid(efield->typ.basetype));
}
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
codelined(1,"}");
break;
@ -1040,7 +1041,7 @@ genc_writevar(Generator* generator, Symbol* vsym, Bytebuffer* code,
varncid(vsym),
cname(vsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
codeflush();
} else { /* rank > 0 */
int i;
@ -1102,7 +1103,7 @@ genc_writevar(Generator* generator, Symbol* vsym, Bytebuffer* code,
cname(vsym),
cname(vsym));
codedump(stmt);
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
}
/* end defined block*/
@ -1255,7 +1256,7 @@ genc_writeattr(Generator* generator, Symbol* asym, Bytebuffer* code,
break;
}
codelined(1,"check_err(stat,__LINE__,__FILE__);");
codelined(1,"CHECK_ERR(stat);");
codelined(1,"}");
}

373
ncgen/genchar.c

@ -4,7 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "odom.h"
/******************************************************/
/* Code for generating char variables etc; mostly
@ -16,6 +15,7 @@ static size_t gen_charconstant(NCConstant*, Bytebuffer*, int fillchar);
static int getfillchar(Datalist* fillsrc);
static void gen_leafchararray(Dimset*,int,Datalist*,Bytebuffer*, int);
static NCConstant* makeconst(int lineno, int len, char* str);
static void rebuildsingletons(Datalist* data);
/*
Matching strings to char variables, attributes, and vlen
@ -61,66 +61,122 @@ gen_chararray(Dimset* dimset, int dimindex, Datalist* data, Bytebuffer* charbuf,
/* Case: netcdf3 case */
if(nc3unlim) {
gen_leafchararray(dimset,0,data,charbuf,fillchar);
return;
gen_leafchararray(dimset,0,data,charbuf,fillchar);
return;
}
/* else generate should have done all the hard work */
gen_leafchararray(dimset,dimindex,data,charbuf,fillchar);
#if 0
if(dimindex < (rank - 1)) {
size_t stop = datalistlen(data);
size_t offset;
for(offset=0;offset<stop;offset++) {
NCConstant* con = datalistith(data,offset);
ASSERT((islistconst(con)));
gen_chararray(dimset,dimindex+1,compoundfor(con),charbuf,fillsrc);
}
} else
#endif
gen_leafchararray(dimset,dimindex,data,charbuf,fillchar);
}
#if 0
/* Recursive helper */
/* I think there is a flaw in the ncgen manual
about handling something like this:
dimensions:
n = 8 ;
variables:
char cdata2(n) ;
data:
cdata2 = '\000','\001','\002','\177','\200','\201','\376','\377';
The rules would say that each of the 8 char constants must
be padded to length 8. I think this is only true if the dimension
is unlimited, and even then, I am not sure.
*/
static void
gen_chararrayr(Dimset* dimset, int dimindex,
Bytebuffer* databuf, Datalist* data, int fillchar,
int unitsize, int expectedsize)
gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
Bytebuffer* charbuf, int fillchar)
{
int i;
size_t dimsize = declsizefor(dimset,dimindex);
int rank = dimset->ndims;
int firstunlim = findunlimited(dimset,0);
int lastunlimited = findlastunlimited(dimset);
int nextunlimited = findunlimited(dimset,dimindex+1);
int islastgroup = (lastunlimited == rank || dimindex >= lastunlimited || dimindex == rank-1);
Odometer* subodom = NULL;
size_t expectedsize,xproduct,unitsize;
int rank = rankfor(dimset);
Datalist* flat = NULL;
ASSERT(rank > 0);
ASSERT((islastgroup));
#if 0
ASSERT(bbLength(charbuf) == 0);
ASSERT((findlastunlimited(dimset) == rank || findlastunlimited(dimset) == dimindex));
#endif
/* we should be at a list of simple constants */
for(i=0;i<data->length;i++) {
NCConstant* c = datalistith(data,i);
/*
There are a number of special cases that must be
considered, mostly driven by the need to keep consistent
with ncgen3. These cases are driven by the number of
dimensions, which dimensions are unlimited (if any), etc.
The general rule is based on the size of the last
dimension, we compute the required size (after padding)
of each string constant. Expected size is then the size
of concat of the string constants after padding.
*/
/*
There is another special case used for back compatibility with ncgen3.
In the datalist, all sequences of character constants (i.e. 'X')
are concatenated into a single string; the result, however is not
concatenated with any trailing or leading string (with double quotes).
*/
rebuildsingletons(data);
/* Compute crossproduct from dimindex up to (but not including) the last dimension */
xproduct = crossproduct(dimset,dimindex,rank-1);
/* Start casing it out */
if(rank == 0) {
unitsize = 1;
expectedsize = (xproduct * unitsize);
} else if(rank == 1) {
unitsize = 1;
expectedsize = (xproduct * declsizefor(dimset,rank-1));
} else if(isunlimited(dimset,rank-1)) {/* last dimension is unlimited */
unitsize = 1;
expectedsize = (xproduct*declsizefor(dimset,rank-1));
} else if(rank > 0 && !isunlimited(dimset,rank-1)) {/*last dim is not unlimited */
unitsize = declsizefor(dimset,rank-1);
expectedsize = (xproduct * unitsize);
} else
abort(); /* in case we forgot a case */
flat = flatten(data,rank);
for(i=0;i<flat->length;i++) {
NCConstant* c = datalistith(flat,i);
ASSERT(!islistconst(c));
if(isstringable(c->nctype)) {
int j;
size_t constsize;
constsize = gen_charconstant(c,databuf,fillchar);
if(constsize % unitsize > 0) {
constsize = gen_charconstant(c,charbuf,fillchar);
if(constsize == 0 || constsize % unitsize > 0) {
size_t padsize = unitsize - (constsize % unitsize);
for(j=0;j<padsize;j++) bbAppend(databuf,fillchar);
for(j=0;j<padsize;j++) bbAppend(charbuf,fillchar);
}
} else {
semwarn(constline(c),
"Encountered non-string and non-char constant in datalist; ignored");
semwarn(constline(c),"Encountered non-string and non-char constant in datalist; ignored");
}
}/* for */
}
freedatalist(flat);
/* If |databuf| > expectedsize, complain: exception is zero length */
if(bbLength(databuf) == 0 && expectedsize == 1) {
if(bbLength(charbuf) == 0 && expectedsize == 1) {
/* this is okay */
} else if(bbLength(databuf) > expectedsize) {
semwarn(data->data[0].lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(databuf));
} else if(bbLength(charbuf) > expectedsize) {
semwarn(flat->data[0]->lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(charbuf));
} else {
size_t bufsize = bbLength(databuf);
size_t bufsize = bbLength(charbuf);
/* Pad to size dimproduct size */
if(bufsize % expectedsize > 0) {
size_t padsize = expectedsize - (bufsize % expectedsize);
for(i=0;i<padsize;i++) bbAppend(databuf,fillchar);
for(i=0;i<padsize;i++) bbAppend(charbuf,fillchar);
}
}
}
#endif
void
gen_charattr(Datalist* data, Bytebuffer* databuf)
@ -165,7 +221,7 @@ gen_charconstant(NCConstant* con, Bytebuffer* databuf, int fillchar)
break;
case NC_STRING:
constsize = con->value.stringv.len;
if(constsize > 0)
if(constsize > 0)
bbAppendn(databuf,con->value.stringv.stringv,
con->value.stringv.len);
bbNull(databuf);
@ -179,6 +235,22 @@ gen_charconstant(NCConstant* con, Bytebuffer* databuf, int fillchar)
return constsize;
}
/* Create a new string constant */
static NCConstant*
makeconst(int lineno, int len, char* str)
{
NCConstant* con = nullconst();
con->nctype = NC_STRING;
con->lineno = lineno;
con->filled = 0;
con->value.stringv.len = len;
/* We cannot use strdup because str might have embedded nuls */
con->value.stringv.stringv = (char*)ecalloc(len+1);
memcpy((void*)con->value.stringv.stringv,(void*)str,len);
con->value.stringv.stringv[len] = '\0';
return con;
}
static int
getfillchar(Datalist* fillsrc)
{
@ -198,168 +270,115 @@ getfillchar(Datalist* fillsrc)
return fillchar;
}
/* I think there is a flaw in the ncgen manual
about handling something like this:
dimensions:
n = 8 ;
variables:
char cdata2(n) ;
data:
cdata2 = '\000','\001','\002','\177','\200','\201','\376','\377';
The rules would say that each of the 8 char constants must
be padded to length 8. I think this is only true if the dimension
is unlimited, and even then, I am not sure.
*/
/* Rebuild the datalist to merge '0x' constants; WARNING: this is tricky. */
static void
gen_leafchararray(Dimset* dimset, int dimindex, Datalist* data,
Bytebuffer* charbuf, int fillchar)
rebuildsingletons(Datalist* data)
{
int i,cccount = 0;
/* Do initial walk */
for(i=0;i<datalistlen(data);i++) {
NCConstant* con = datalistith(data,i);
if(consttype(con) == NC_CHAR || consttype(con) == NC_BYTE) {
cccount++;
}
}
if(cccount > 1) {
Bytebuffer* accum = bbNew();
int len = 0; /* >0 implies doing accum */
Datalist* newlist = builddatalist(datalistlen(data));
int lineno = 0;
NCConstant* con;
/* We are going to construct a single string constant for each
contiguous sequence of single char values.
Assume that the constants are all primitive types */
for(i=0;i<datalistlen(data);i++) {
con = datalistith(data,i);
if(consttype(con) == NC_CHAR || consttype(con) == NC_BYTE) {
if(len == 0) { /* Start an accumulation */
lineno = constline(con);
bbClear(accum);
}
bbAppend(accum,con->value.charv);
len++;
/* Discard this constant */
reclaimconstant(con);
} else {
if(len > 0) { /* End the accumulation */
bbNull(accum);
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
dlappend(newlist,con);
} else
dlappend(newlist,con);
}
}
/* deal with any unclosed strings */
if(len > 0) {
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
dlappend(newlist,con);
}
bbFree(accum);
/* Move the newlist sequence of constants into the old list */
efree(data->data);
data->data = newlist->data;
data->length = newlist->length;
data->alloc = newlist->alloc;
efree(newlist);
}
}
#if 0
/* Recursive helper */
static void
gen_chararrayr(Dimset* dimset, int dimindex,
Bytebuffer* databuf, Datalist* data, int fillchar,
int unitsize, int expectedsize)
{
int i;
size_t expectedsize,xproduct,unitsize;
int rank = rankfor(dimset);
size_t dimsize = declsizefor(dimset,dimindex);
int rank = dimset->ndims;
int firstunlim = findunlimited(dimset,0);
int lastunlimited = findlastunlimited(dimset);
int nextunlimited = findunlimited(dimset,dimindex+1);
int islastgroup = (lastunlimited == rank || dimindex >= lastunlimited || dimindex == rank-1);
ASSERT(bbLength(charbuf) == 0);
ASSERT((findlastunlimited(dimset) == rank
|| findlastunlimited(dimset) == dimindex));
/*
There are a number of special cases that must be
considered, mostly driven by the need to keep consistent
with ncgen3. These cases are driven by the number of
dimensions, which dimensions are unlimited (if any), etc.
The general rule is based on the size of the last
dimension, we compute the required size (after padding)
of each string constant. Expected size is then the size
of concat of the string constants after padding.
There is another special case used for back compatibility with ncgen3.
In the datalist, all sequences of character constants (i.e. 'X')
are concatenated into a single string; the result, however is not
concatenated with any trailing or leading string (with double quotes).
*/
/* Rebuild the datalist to merge '0x' constants;
WARNING: this is tricky.
*/
{
int i,cccount = 0;
/* Do initial walk */
for(i=0;i<datalistlen(data);i++) {
NCConstant* con = datalistith(data,i);
if(consttype(con) == NC_CHAR || consttype(con) == NC_BYTE) {
cccount++;
}
}
if(cccount > 1) {
Bytebuffer* accum = bbNew();
int len = 0; /* >0 implies doing accum */
Datalist* newlist = builddatalist(datalistlen(data));
int lineno = 0;
NCConstant* con;
/* We are going to construct a single string constant for each
contiguous sequence of single char values.
Assume that the constants are all primitive types */
for(i=0;i<datalistlen(data);i++) {
con = datalistith(data,i);
if(consttype(con) == NC_CHAR || consttype(con) == NC_BYTE) {
if(len == 0) { /* Start an accumulation */
lineno = constline(con);
bbClear(accum);
}
bbAppend(accum,con->value.charv);
len++;
/* Discard this constant */
reclaimconstant(con);
} else {
if(len > 0) { /* End the accumulation */
bbNull(accum);
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
dlappend(newlist,con);
} else
dlappend(newlist,con);
}
}
/* deal with any unclosed strings */
if(len > 0) {
con = makeconst(lineno,len,bbContents(accum));
len = 0;
lineno = 0;
dlappend(newlist,con);
}
bbFree(accum);
/* Move the newlist sequence of constants into the old list */
efree(data->data);
data->data = newlist->data;
data->length = newlist->length;
data->alloc = newlist->alloc;
efree(newlist);
}
}
/* Compute crossproduct up to (but not including) the last dimension */
xproduct = crossproduct(dimset,dimindex,rank-1);
/* Start casing it out */
if(rank == 0) {
unitsize = 1;
expectedsize = (xproduct * unitsize);
} else if(rank == 1) {
unitsize = 1;
expectedsize = (xproduct * declsizefor(dimset,rank-1));
} else if(isunlimited(dimset,rank-1)) {/* last dimension is unlimited */
unitsize = 1;
expectedsize = (xproduct*declsizefor(dimset,rank-1));
} else { /* rank > 0 && last dim is not unlimited */
unitsize = declsizefor(dimset,rank-1);
expectedsize = (xproduct * unitsize);
}
ASSERT(rank > 0);
ASSERT((islastgroup));
/* we should be at a list of simple constants */
for(i=0;i<data->length;i++) {
NCConstant* c = datalistith(data,i);
ASSERT(!islistconst(c));
if(isstringable(c->nctype)) {
int j;
size_t constsize;
constsize = gen_charconstant(c,charbuf,fillchar);
if(constsize == 0 || constsize % unitsize > 0) {
constsize = gen_charconstant(c,databuf,fillchar);
if(constsize % unitsize > 0) {
size_t padsize = unitsize - (constsize % unitsize);
for(j=0;j<padsize;j++) bbAppend(charbuf,fillchar);
for(j=0;j<padsize;j++) bbAppend(databuf,fillchar);
}
} else {
semwarn(constline(c),"Encountered non-string and non-char constant in datalist; ignored");
semwarn(constline(c),
"Encountered non-string and non-char constant in datalist; ignored");
}
}
}/* for */
/* If |databuf| > expectedsize, complain: exception is zero length */
if(bbLength(charbuf) == 0 && expectedsize == 1) {
if(bbLength(databuf) == 0 && expectedsize == 1) {
/* this is okay */
} else if(bbLength(charbuf) > expectedsize) {
semwarn(data->data[0]->lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(charbuf));
} else if(bbLength(databuf) > expectedsize) {
semwarn(data->data[0].lineno,"character data list too long; expected %d character constant, found %d: ",expectedsize,bbLength(databuf));
} else {
size_t bufsize = bbLength(charbuf);
size_t bufsize = bbLength(databuf);
/* Pad to size dimproduct size */
if(bufsize % expectedsize > 0) {
size_t padsize = expectedsize - (bufsize % expectedsize);
for(i=0;i<padsize;i++) bbAppend(charbuf,fillchar);
for(i=0;i<padsize;i++) bbAppend(databuf,fillchar);
}
}
}
/* Create a new string constant */
static NCConstant*
makeconst(int lineno, int len, char* str)
{
NCConstant* con = nullconst();
con->nctype = NC_STRING;
con->lineno = lineno;
con->filled = 0;
con->value.stringv.len = len;
/* We cannot use strdup because str might have embedded nuls */
con->value.stringv.stringv = (char*)ecalloc(len+1);
memcpy((void*)con->value.stringv.stringv,(void*)str,len);
con->value.stringv.stringv[len] = '\0';
return con;
}
#endif

356
ncgen/generate.c

@ -4,8 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#include "odom.h"
#include "ncoffsets.h"
#include "netcdf_aux.h"
@ -16,7 +14,6 @@
/* Forward*/
static void generate_array(Symbol*,Bytebuffer*,Datalist*,Generator*,Writer);
static void generate_arrayr(Symbol*,Bytebuffer*,Datalist*,Odometer*,int,Datalist*,Generator*);
static void generate_primdata(Symbol*, NCConstant*, Bytebuffer*, Datalist* fillsrc, Generator*);
static void generate_fieldarray(Symbol*, NCConstant*, Dimset*, Bytebuffer*, Datalist* fillsrc, Generator*);
@ -24,6 +21,18 @@ static void generate_fieldarray(Symbol*, NCConstant*, Dimset*, Bytebuffer*, Data
#define VLENLIST1
#define FIELDARRAY 1
#define ITER_BUFSIZE_DEFAULT (2<<20)
void
pvec(int rank, size_t* vector)
{
int i;
fprintf(stderr,"(");
for(i=0;i<rank;i++)
fprintf(stderr," %lu",(long)vector[i]);
fprintf(stderr,")");
}
/**************************************************/
@ -39,18 +48,6 @@ int generator_reset(Generator* generator, void* state)
generator->globalstate = state;
return 1;
}
#ifdef IGNORe
static void
checkodom(Odometer* odom)
{
int i;
for(i=0;i<odom->rank;i++) {
ASSERT(odom->index[i] == odom->start[i]+odom->count[i]);
}
}
#endif
/**************************************************/
void
@ -85,203 +82,15 @@ generate_vardata(Symbol* vsym, Generator* generator, Writer writer, Bytebuffer*
if(vsym->data == NULL) return;
/* give the buffer a running start to be large enough*/
if(!bbSetalloc(code, nciterbuffersize))
return;
if(rank == 0) {/*scalar case*/
NCConstant* c0 = datalistith(vsym->data,0);
generate_basetype(basetype,c0,code,filler,generator);
writer(generator,vsym,code,0,NULL,NULL);
} else {/*rank > 0*/
#if 0
/* First, create an odometer using all of the dimensions */
odom = newodometer(dimset,NULL,NULL);
start = odometerstartvector(odom);
count = odometercountvector(odom);
#endif
generate_array(vsym,code,filler,generator,writer);
}
}
/**
The basic idea is to split the set of dimensions into
groups and iterate over each group by recursion.
A group is defined as the range of indices starting at an
unlimited dimension up to (but not including) the next
unlimited.
The first group starts at index 0, even if dimension 0 is not
unlimited. The last group is everything from the last
unlimited dimension thru the last dimension (index rank-1).
*/
static void
generate_array(Symbol* vsym,
Bytebuffer* code,
Datalist* filler,
Generator* generator,
Writer writer
)
{
Dimset* dimset = &vsym->typ.dimset;
int rank = dimset->ndims;
Symbol* basetype = vsym->typ.basetype;
nc_type typecode = basetype->typ.typecode;
nciter_t iter;
int firstunlim = findunlimited(dimset,1);
int nunlim = countunlimited(dimset);
int isnc3unlim = (nunlim <= 1 && (firstunlim == 0 || firstunlim == rank)); /* netcdf-3 case of at most 1 unlim in 0th dimension */
ASSERT(rank > 0);
if(isnc3unlim) {
/* Handle NC_CHAR case separately */
if(typecode == NC_CHAR) {
Odometer* odom = newodometer(dimset,NULL,NULL);
Bytebuffer* charbuf = bbNew();
gen_chararray(dimset,0,vsym->data,charbuf,filler);
generator->charconstant(generator,vsym,code,charbuf);
/* Create an odometer to get the dimension info */
writer(generator,vsym,code,odom->rank,odom->start,odom->count);
#if 0
writer(generator,vsym,code,odom->rank,0,bbLength(charbuf));
#endif
bbFree(charbuf);
odometerfree(odom);
} else { /* typecode != NC_CHAR */
Odometer* odom = newodometer(dimset,NULL,NULL);
/* Case: dim 1..rank-1 are not unlimited, dim 0 might be */
size_t offset = 0; /* where are we in the data list */
size_t nelems = 0; /* # of data list items to generate */
/* Create an iterator and odometer and just walk the datalist */
nc_get_iter(vsym,nciterbuffersize,&iter);
for(;;offset+=nelems) {
int i,uid;
nelems=nc_next_iter(&iter,odometerstartvector(odom),odometercountvector(odom));
if(nelems == 0)
break;
bbClear(code);
generator->listbegin(generator,vsym,NULL,LISTDATA,vsym->data->length,code,&uid);
for(i=0;i<nelems;i++) {
NCConstant* con = datalistith(vsym->data,i+offset);
generator->list(generator,vsym,NULL,LISTDATA,uid,i,code);
generate_basetype(basetype,con,code,filler,generator);
}
generator->listend(generator,vsym,NULL,LISTDATA,uid,i,code);
writer(generator,vsym,code,rank,odom->start,odom->count);
}
odometerfree(odom);
}
} else { /* Hard case: multiple unlimited dimensions or unlim in dim > 0*/
Odometer* odom = newodometer(dimset,NULL,NULL);
/* Setup iterator and odometer */
nc_get_iter(vsym,NC_MAX_UINT,&iter); /* effectively infinite */
for(;;) {/* iterate in nelem chunks */
/* get nelems count and modify odometer */
size_t nelems=nc_next_iter(&iter,odom->start,odom->count);
if(nelems == 0) break;
generate_arrayr(vsym,code,vsym->data,
odom,
/*dim index=*/0,
filler,generator
);
writer(generator,vsym,code,odom->rank,odom->start,odom->count);
}
odometerfree(odom);
}
}
/**
The basic idea is to split the set of dimensions into groups
and iterate over each group. A group is defined as the
range of indices starting at an unlimited dimension up to
(but not including) the next unlimited. The first group
starts at index 0, even if dimension 0 is not unlimited.
The last group is everything from the last unlimited
dimension thru the last dimension (index rank-1).
*/
static void
generate_arrayr(Symbol* vsym,
Bytebuffer* code,
Datalist* list,
Odometer* odom,
int dimindex,
Datalist* filler,
Generator* generator
)
{
int uid,i;
Symbol* basetype = vsym->typ.basetype;
Dimset* dimset = &vsym->typ.dimset;
int rank = dimset->ndims;
int lastunlimited = findlastunlimited(dimset);
int nextunlimited = findunlimited(dimset,dimindex+1);
int typecode = basetype->typ.typecode;
int islastgroup = (lastunlimited == rank || dimindex >= lastunlimited || dimindex == rank-1);
Odometer* subodom = NULL;
ASSERT(rank > 0);
ASSERT((dimindex >= 0 && dimindex < rank));
if(islastgroup) {
/* Handle NC_CHAR case separately */
if(typecode == NC_CHAR) {
Bytebuffer* charbuf = bbNew();
gen_chararray(dimset,dimindex,list,charbuf,filler);
generator->charconstant(generator,vsym,code,charbuf);
bbFree(charbuf);
} else {
/* build a special odometer to walk the last few dimensions */
subodom = newsubodometer(odom,dimset,dimindex,rank);
generator->listbegin(generator,vsym,NULL,LISTDATA,list->length,code,&uid);
for(i=0;odometermore(subodom);i++) {
size_t offset = odometeroffset(subodom);
NCConstant* con = datalistith(list,offset);
generator->list(generator,vsym,NULL,LISTDATA,uid,i,code);
generate_basetype(basetype,con,code,filler,generator);
odometerincr(subodom);
}
generator->listend(generator,vsym,NULL,LISTDATA,uid,i,code);
odometerfree(subodom); subodom = NULL;
}
} else {/* !islastgroup */
/* Our datalist must be a list of compounds representing
the next unlimited; so walk the subarray from this index
up to next unlimited.
*/
ASSERT((dimindex < nextunlimited));
ASSERT((isunlimited(dimset,nextunlimited)));
/* build a sub odometer */
subodom = newsubodometer(odom,dimset,dimindex,nextunlimited);
for(i=0;odometermore(subodom);i++) {
size_t offset = odometeroffset(subodom);
NCConstant* con = datalistith(list,offset);
if(con == NULL || con->nctype == NC_FILL) {
if(filler == NULL)
filler = getfiller(vsym);
generate_arrayr(vsym,code,filler,odom,nextunlimited,NULL,generator);
} else if(islistconst(con)) {
Datalist* sublist = compoundfor(con);
generate_arrayr(vsym,code,sublist,odom,nextunlimited,filler,generator);
} else {
semerror(constline(con),"Expected {...} representing unlimited list");
return;
}
odometerincr(subodom);
}
odometerfree(subodom); subodom = NULL;
}
if(subodom != NULL)
odometerfree(subodom);
return;
}
/* Generate an instance of the basetype using the value of con*/
void
generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist* filler, Generator* generator)
@ -294,6 +103,11 @@ generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist*
case NC_ENUM:
case NC_OPAQUE:
case NC_PRIM:
if(con == NULL || isfillconst(con)) {
Datalist* fill = (filler==NULL?getfiller(tsym):filler);
ASSERT(fill->length == 1);
con = datalistith(fill,0);
}
if(islistconst(con)) {
semerror(constline(con),"Expected primitive found {..}");
}
@ -540,3 +354,139 @@ generate_primdata(Symbol* basetype, NCConstant* prim, Bytebuffer* codebuf,
target = NULL;
return;
}
/* Avoid long argument lists */
struct Args {
Symbol* vsym;
Dimset* dimset;
int typecode;
int storage;
int rank;
Generator* generator;
Writer writer;
Bytebuffer* code;
Datalist* filler;
size_t dimsizes[NC_MAX_VAR_DIMS];
size_t chunksizes[NC_MAX_VAR_DIMS];
};
static void
generate_arrayR(struct Args* args, int dimindex, size_t* index, Datalist* data)
{
size_t counter,stop;
size_t count[NC_MAX_VAR_DIMS];
Datalist* actual;
Symbol* dim = args->dimset->dimsyms[dimindex];
stop = args->dimsizes[dimindex];
/* Four cases: (dimindex==rank-1|dimindex<rank-1) X (unlimited|!unlimited) */
if(dimindex == (args->rank - 1)) {/* base case */
int uid;
if(dimindex > 0 && dim->dim.isunlimited) {
/* Get the unlimited list */
NCConstant* con = datalistith(data,0);
actual = compoundfor(con);
} else
actual = data;
/* For last index, dump all of its elements */
args->generator->listbegin(args->generator,args->vsym,NULL,LISTDATA,datalistlen(actual),args->code,&uid);
for(counter=0;counter<stop;counter++) {
NCConstant* con = datalistith(actual,counter);
generate_basetype(args->vsym->typ.basetype,con,args->code,args->filler,args->generator);
args->generator->list(args->generator,args->vsym,NULL,LISTDATA,uid,counter,args->code);
}
args->generator->listend(args->generator,args->vsym,NULL,LISTDATA,uid,counter,args->code);
memcpy(count,onesvector,sizeof(size_t)*dimindex);
count[dimindex] = stop;
args->writer(args->generator,args->vsym,args->code,args->rank,index,count);
bbClear(args->code);
} else {
actual = data;
/* Iterate over this dimension */
for(counter = 0;counter < stop; counter++) {
Datalist* subdata = NULL;
NCConstant* con = datalistith(actual,counter);
if(con == NULL)
subdata = filldatalist;
else {
ASSERT(islistconst(con));
if(islistconst(con)) subdata = compoundfor(con);
}
index[dimindex] = counter;
generate_arrayR(args,dimindex+1,index,subdata); /* recurse */
}
}
}
static void
generate_array(Symbol* vsym, Bytebuffer* code, Datalist* filler, Generator* generator, Writer writer)
{
int i;
size_t index[NC_MAX_VAR_DIMS];
struct Args args;
size_t totalsize;
int nunlimited = 0;
assert(vsym->typ.dimset.ndims > 0);
args.vsym = vsym;
args.dimset = &vsym->typ.dimset;
args.generator = generator;
args.writer = writer;
args.filler = filler;
args.code = code;
args.rank = args.dimset->ndims;
args.storage = vsym->var.special._Storage;
args.typecode = vsym->typ.basetype->typ.typecode;
assert(args.rank > 0);
totalsize = 1; /* total # elements in the array */
for(i=0;i<args.rank;i++) {
args.dimsizes[i] = args.dimset->dimsyms[i]->dim.declsize;
totalsize *= args.dimsizes[i];
}
nunlimited = countunlimited(args.dimset);
if(vsym->var.special._Storage == NC_CHUNKED)
memcpy(args.chunksizes,vsym->var.special._ChunkSizes,sizeof(size_t)*args.rank);
memset(index,0,sizeof(index));
/* Special case for NC_CHAR */
if(args.typecode == NC_CHAR) {
size_t start[NC_MAX_VAR_DIMS];
size_t count[NC_MAX_VAR_DIMS];
Bytebuffer* charbuf = bbNew();
gen_chararray(args.dimset,0,args.vsym->data,charbuf,args.filler);
args.generator->charconstant(args.generator,args.vsym,args.code,charbuf);
memset(start,0,sizeof(size_t)*args.rank);
memcpy(count,args.dimsizes,sizeof(size_t)*args.rank);
args.writer(args.generator,args.vsym,args.code,args.rank,start,count);
bbFree(charbuf);
bbClear(args.code);
return;
}
/* If the total no. of elements is less than some max and no unlimited,
then generate a single vara that covers the whole array */
if(totalsize < wholevarsize && nunlimited == 0) {
Symbol* basetype = args.vsym->typ.basetype;
size_t counter;
int uid;
Datalist* flat = flatten(vsym->data,args.rank);
args.generator->listbegin(args.generator,basetype,NULL,LISTDATA,totalsize,args.code,&uid);
for(counter=0;counter<totalsize;counter++) {
NCConstant* con = datalistith(flat,counter);
if(con == NULL)
con = &fillconstant;
generate_basetype(basetype,con,args.code,args.filler,args.generator);
args.generator->list(args.generator,args.vsym,NULL,LISTDATA,uid,counter,args.code);
}
args.generator->listend(args.generator,args.vsym,NULL,LISTDATA,uid,counter,args.code);
args.writer(args.generator,args.vsym,args.code,args.rank,zerosvector,args.dimsizes);
freedatalist(flat);
} else
generate_arrayR(&args, 0, index, vsym->data);
}

62
ncgen/generate.h Normal file

@ -0,0 +1,62 @@
/*********************************************************************
* Copyright 2018, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#ifndef GENERATE_H
#define GENERATE_H 1
typedef struct Generator Generator;
void alignbuffer(struct NCConstant* prim, Bytebuffer* buf);
/* Code dump support procedures */
void bbindent(Bytebuffer*,const int);
void bbprintf(Bytebuffer*,const char *fmt, ...); /* append */
void bbprintf0(Bytebuffer*,const char *fmt, ...); /* clear, then append*/
/* Following dump to codebuffer */
void codeprintf(const char *fmt, ...);
void codedump(Bytebuffer*);
void codepartial(const char*);
void codeline(const char*);
void codelined(int n,const char*);
void codeflush(void); /* flush codebuffer to stdout */
void commify(Bytebuffer* buf);
char* word(char* p, Bytebuffer* buf);
/* Provide buffers for language based generators */
extern Bytebuffer* codebuffer; /* buffer over the std output */
extern Bytebuffer* stmt; /* single stmt text generation */
/* From genchar.c */
void gen_charattr(Datalist*, Bytebuffer*);
void gen_charseq(Datalist*, Bytebuffer*);
void gen_chararray(Dimset*, int, Datalist*, Bytebuffer*, Datalist*);
typedef enum ListClass {
LISTDATA, LISTATTR, LISTVLEN, LISTCOMPOUND, LISTFIELDARRAY
} ListClass;
struct Generator {
void* globalstate; /* per-generator; per list state is in the method args where needed */
int (*charconstant)(Generator*,struct Symbol*,Bytebuffer*,...);
int (*constant)(Generator*,struct Symbol*,NCConstant*,Bytebuffer*,...);
int (*listbegin)(Generator*,struct Symbol*,void*,ListClass,size_t,Bytebuffer*,int*,...);
int (*list)(Generator*,struct Symbol*,void*,ListClass,int,size_t,Bytebuffer*,...);
int (*listend)(Generator*,struct Symbol*,void*,ListClass,int,size_t,Bytebuffer*,...);
int (*vlendecl)(Generator*,struct Symbol*,Bytebuffer*,int,size_t,...);
int (*vlenstring)(Generator*,struct Symbol*,Bytebuffer*,int*,size_t*,...);
};
extern int generator_getstate(Generator*,void**);
extern int generator_reset(Generator*,void*);
typedef int (*Writer)(Generator*,struct Symbol*,Bytebuffer*,int,const size_t*,const size_t*);
extern void generate_attrdata(struct Symbol*, Generator*, Writer writer, Bytebuffer*);
extern void generate_vardata(struct Symbol*, Generator*, Writer writer,Bytebuffer*);
extern void generate_basetype(struct Symbol*,NCConstant*,Bytebuffer*,Datalist*,Generator*);
#endif /*DATA_H*/

5
ncgen/generr.c

@ -13,11 +13,6 @@ extern int vsnprintf(char*, size_t, const char*, va_list ap);
int error_count;
#if 0
#define vastart(argv,fmt) va_start(argv,fmt)
#define vaend(argv,fmt) va_end(argv)
#endif
/*
* For logging error conditions.
* Designed to be called by other vararg procedures

38
ncgen/genf77.c

@ -5,7 +5,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#ifdef ENABLE_F77
@ -571,43 +570,6 @@ f77attrifyr(Symbol* asym, char* p, Bytebuffer* buf)
return p;
}
#ifdef USE_NETCDF4
#if 0
/* Result is pool alloc'd*/
static char*
f77prefixed(List* prefix, char* suffix, char* separator)
{
int slen;
int plen;
int i;
char* result;
ASSERT(suffix != NULL);
plen = prefixlen(prefix);
if(prefix == NULL || plen == 0) return codify(suffix);
/* plen > 0*/
slen = 0;
for(i=0;i<plen;i++) {
Symbol* sym = (Symbol*)listget(prefix,i);
slen += (strlen(sym->name)+strlen(separator));
}
slen += strlen(suffix);
slen++; /* for null terminator*/
result = poolalloc(slen);
result[0] = '\0';
/* Leave off the root*/
i = (rootgroup == (Symbol*)listget(prefix,0))?1:0;
for(;i<plen;i++) {
Symbol* sym = (Symbol*)listget(prefix,i);
strcat(result,sym->name); /* append "<prefix[i]/>"*/
strcat(result,separator);
}
strcat(result,suffix); /* append "<suffix>"*/
return result;
}
#endif
#endif
/* return FORTRAN name for netCDF type, given type code */
static const char*
nftype(nc_type type)

1
ncgen/genj.c

@ -5,7 +5,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#ifdef ENABLE_JAVA

35
ncgen/genlib.c

@ -156,38 +156,3 @@ attfqn(Symbol* sym)
strcat(fqn,fqnname);
sym->fqn = fqn;
}
#if 0
/* Result is pool alloc'd*/
char*
cprefixed(List* prefix, char* suffix, char* separator)
{
int slen;
int plen;
int i;
char* result;
ASSERT(suffix != NULL);
plen = prefixlen(prefix);
if(prefix == NULL || plen == 0) return codify(suffix);
/* plen > 0*/
slen = 0;
for(i=0;i<plen;i++) {
Symbol* sym = (Symbol*)listget(prefix,i);
slen += (strlen(sym->name)+strlen(separator));
}
slen += strlen(suffix);
slen++; /* for null terminator*/
result = poolalloc(slen);
result[0] = '\0';
/* Leave off the root*/
i = (rootgroup == (Symbol*)listget(prefix,0))?1:0;
for(;i<plen;i++) {
Symbol* sym = (Symbol*)listget(prefix,i);
strcat(result,sym->name); /* append "<prefix[i]/>"*/
strcat(result,separator);
}
strcat(result,suffix); /* append "<suffix>"*/
return result;
}
#endif /*0*/

6
ncgen/genlib.h

@ -82,6 +82,7 @@ extern struct Datalist* getfiller(Symbol*); /* symbol isa variable|type */
/* from: ncgen.y */
extern Symbol* install(const char *sname);
extern void freesymbol(Symbol*);
extern Symbol* basetypefor(nc_type nctype);/* Convert nctype to a Symbol*/
extern Symbol* makearraytype(Symbol*, Dimset*);
@ -163,6 +164,7 @@ extern int specials_flag; /* 1 => special attributes are present */
extern int usingclassic; /* 1 => k_flag == 1|2|5 */
extern int k_flag;
extern int ncloglevel;
extern int wholevarsize;
extern GlobalSpecialData globalspecials;
/* Global data */
@ -191,13 +193,15 @@ extern char* binary_ext;
extern int nofill_flag;
extern int header_only;
extern char* mainname;
extern size_t nciterbuffersize;
extern char* progname; /* for error messages*/
extern char *netcdf_name; /* command line -o file name */
extern char *datasetname; /* name from the netcdf <name> {} */
extern char *cdlname; /* name from the command line */
extern size_t zerosvector[NC_MAX_VAR_DIMS];
extern size_t onesvector[NC_MAX_VAR_DIMS];
extern const char* specialname(int tag);
extern void init_netcdf(void);

4
ncgen/getfill.c

@ -33,9 +33,9 @@ getfiller(Symbol* tvsym)
Datalist* filler = NULL;
ASSERT(tvsym->objectclass == NC_VAR || tvsym->objectclass == NC_TYPE);
if(tvsym->objectclass == NC_VAR) {
if(tvsym->var.special->_Fillvalue != NULL) {
if(tvsym->var.special._Fillvalue != NULL) {
/* We have a _FillValue Attribute specified */
filler = tvsym->var.special->_Fillvalue;
filler = tvsym->var.special._Fillvalue;
} else { /* otherwise create a fillvalue for the base type */
filler = getfiller(tvsym->typ.basetype);
}

1
ncgen/includes.h

@ -44,6 +44,7 @@
#include "nc4internal.h"
#include "data.h"
#include "ncgen.h"
#include "generate.h"
#include "genlib.h"
#include "util.h"
#include "debug.h"

1
ncgen/jdata.c

@ -4,7 +4,6 @@
*********************************************************************/
#include "includes.h"
#include "nc_iter.h"
#ifdef ENABLE_JAVA

58
ncgen/main.c

@ -42,12 +42,14 @@ int usingclassic;
int cmode_modifier;
int diskless;
int ncloglevel;
int wholevarsize;
GlobalSpecialData globalspecials;
char* binary_ext = ".nc";
size_t zerosvector[NC_MAX_VAR_DIMS];
size_t onesvector[NC_MAX_VAR_DIMS];
size_t nciterbuffersize;
char* binary_ext = ".nc";
struct Vlendata* vlendata;
@ -141,22 +143,6 @@ struct Languages legallanguages[] = {
};
#endif
#if 0 /*not used*/
/* BOM Sequences */
static char* U8 = "\xEF\xBB\xBF"; /* UTF-8 */
static char* BE32 = "\x00\x00\xFE\xFF"; /* UTF-32; big-endian */
static char* LE32 = "\xFF\xFE"; /* UTF-32; little-endian */
static char* BE16 = "\xFE\xFF"; /* UTF-16; big-endian */
static char* LE16 = "\xFF\xFE"; /* UTF-16; little-endian */
#endif
/* The default minimum iterator size depends
on whether we are doing binary or language
based output.
*/
#define DFALTBINNCITERBUFFERSIZE 0x40000 /* about 250k bytes */
#define DFALTLANGNCITERBUFFERSIZE 0x4000 /* about 15k bytes */
/* strip off leading path */
/* result is malloc'd */
@ -185,7 +171,6 @@ usage(void)
" [-6]"
" [-7]"
" [-b]"
" [-B buffersize]"
" [-d]"
" [-D debuglevel]"
" [-h]"
@ -199,6 +184,7 @@ usage(void)
" [-N datasetname]"
" [-L loglevel]"
" [-H]"
" [-W generate whole var upload]"
" [file ... ]",
progname);
derror("netcdf library version %s", nc_inq_libvers());
@ -229,7 +215,6 @@ main(
syntax_only = 0;
header_only = 0;
mainname = "main";
nciterbuffersize = 0;
k_flag = 0;
format_attribute = 0;
@ -237,6 +222,7 @@ main(
cdf5_flag = 0;
specials_flag = 0;
diskless = 0;
wholevarsize = 1024;
#ifdef LOGGING
ncloglevel = NC_TURN_OFF_LOGGING;
#else
@ -244,7 +230,7 @@ main(
#endif
memset(&globalspecials,0,sizeof(GlobalSpecialData));
while ((c = getopt(argc, argv, "134567bB:cdD:fhHk:l:M:no:Pv:xL:N:")) != EOF)
while ((c = getopt(argc, argv, "134567bcdD:fhHk:l:M:no:Pv:xL:N:B:")) != EOF)
switch(c) {
case 'd':
debug = 1;
@ -275,6 +261,9 @@ main(
}
l_flag = L_BINARY;
break;
case 'B':
/* ignore, but leave for back compatibility */
break;
case 'H':
header_only = 1;
break;
@ -292,9 +281,6 @@ main(
derror("%s: output language is null", progname);
return(1);
}
#if 0
lang_name = estrdup(optarg);
#endif
lang_name = (char*) emalloc(strlen(optarg)+1);
(void)strcpy(lang_name, optarg);
@ -330,6 +316,9 @@ main(
if(datasetname) efree(datasetname);
datasetname = nulldup(optarg);
break;
case 'W':
wholevarsize = atoi(optarg);
break;
case 'x': /* set nofill mode to speed up creation of large files */
nofill_flag = 1;
break;
@ -385,9 +374,6 @@ main(
case 'M': /* Determine the name for the main function */
mainname = nulldup(optarg);
break;
case 'B':
nciterbuffersize = atoi(optarg);
break;
case 'P': /* diskless with persistence */
diskless = 1;
break;
@ -403,15 +389,6 @@ main(
syntax_only = 1;
}
/* Compute/default the iterator buffer size */
if(l_flag == L_BINARY) {
if(nciterbuffersize == 0 )
nciterbuffersize = DFALTBINNCITERBUFFERSIZE;
} else {
if(nciterbuffersize == 0)
nciterbuffersize = DFALTLANGNCITERBUFFERSIZE;
}
#ifndef ENABLE_C
if(l_flag == L_C) {
fprintf(stderr,"C not currently supported\n");
@ -591,13 +568,22 @@ done:
void
init_netcdf(void) /* initialize global counts, flags */
{
int i;
memset((void*)&nullconstant,0,sizeof(NCConstant));
fillconstant = nullconstant;
fillconstant.nctype = NC_FILLVALUE;
filldatalist = builddatalist(1);
dlappend(filldatalist,&fillconstant);
filldatalist->readonly = 1;
codebuffer = bbNew();
stmt = bbNew();
error_count = 0; /* Track # of errors */
for(i=0;i<NC_MAX_VAR_DIMS;i++) onesvector[i] = 1;
memset(zerosvector,0,sizeof(zerosvector));
}
void

243
ncgen/nc_iter.c

@ -1,243 +0,0 @@
/*********************************************************************
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/README file for copying and redistribution conditions.
* "$Id $"
*********************************************************************/
/*
This nciter code was derived from the ncdump/nciter code.
It has diverged slightly over time.
*/
#include "includes.h"
#include "nc_iter.h"
#define CHECK(stat,f) if(stat != NC_NOERR) {check(stat,#f,"ncgen",__LINE__);} else {}
/* forward declarations */
static int nc_blkio_init(size_t bufsize, size_t value_size, int rank,
const size_t *dims, nciter_t *iter);
static int up_start(int ndims, const size_t *dims, int incdim, size_t inc,
size_t* odom);
static int nciter_ndims(Symbol*,int*);
static int nciter_dimlens(Symbol*,size_t*);
static int nciter_vartype(Symbol*,Symbol**);
static int nciter_valuesize(Symbol*,size_t*);
static void
check(int err, const char* fcn, const char* file, const int line)
{
fprintf(stderr,"%s\n",nc_strerror(err));
fprintf(stderr,"Location: function %s; file %s; line %d\n",
fcn,file,line);
fflush(stderr); fflush(stdout);
exit(1);
}
/* Initialize iteration for a variable. Just a wrapper for
* nc_blkio_init() that makes the netCDF calls needed to initialize
* lower-level iterator. */
int
nc_get_iter(Symbol* vsym,
size_t bufsize, /* size in bytes of memory buffer */
nciter_t *iterp /* returned opaque iteration state */)
{
int stat = NC_NOERR;
Symbol* vartype;
size_t value_size; /* size in bytes of each variable element */
int ndims; /* number of dimensions for variable */
size_t dimsizes[NC_MAX_VAR_DIMS] = {0}; /* variable dimension sizes */
long long nvalues = 1;
int dim;
memset((void*)iterp,0,sizeof(nciter_t)); /* make sure it is initialized */
stat = nciter_ndims(vsym, &ndims);
CHECK(stat, nciter_ndims);
stat = nciter_dimlens(vsym,dimsizes);
CHECK(stat, nciter_dimlens);
/* compute total # elements */
nvalues=1;
for(dim = 0; dim < ndims; dim++) {
nvalues *= dimsizes[dim];
}
stat = nciter_vartype(vsym, &vartype);
CHECK(stat, nciter_vartype);
stat = nciter_valuesize(vartype,&value_size);
CHECK(stat, nciter_valuesize);
stat = nc_blkio_init(bufsize, value_size, ndims, dimsizes, iterp);
CHECK(stat, nc_blkio_init);
iterp->to_get = 0;
return stat;
}
/* Iterate on blocks for variables, by updating start and count vector
* for next vara call. Assumes nc_get_iter called first. Returns
* number of variable values to get, 0 if done, negative number if
* error, so use like this:
size_t to_get;
while((to_get = nc_next_iter(&iter, start, count)) > 0) {
... iteration ...
}
if(to_get < 0) { ... handle error ... }
*/
size_t
nc_next_iter(nciter_t *iter, /* returned opaque iteration state */
size_t *start, /* returned start vector for next vara call */
size_t *count /* returned count vector for next vara call */
) {
int i;
if(iter->first) {
for(i = 0; i < iter->right_dim; i++) {
start[i] = 0;
count[i] = 1;
}
start[iter->right_dim] = 0;
count[iter->right_dim] = iter->rows;
for(i = iter->right_dim + 1; i < iter->rank; i++) {
start[i] = 0;
count[i] = iter->dimsizes[i];
}
iter->first = 0;
} else {
iter->more = up_start(iter->rank, iter->dimsizes, iter->right_dim,
iter->inc, start);
/* iterate on pieces of variable */
if(iter->cur < iter->numrows) {
iter->inc = iter->rows;
count[iter->right_dim] = iter->rows;
iter->cur++;
} else {
if(iter->leftover > 0) {
count[iter->right_dim] = iter->leftover;
iter->inc = iter->leftover;
iter->cur = 0;
}
}
}
iter->to_get = 1;
for(i = 0; i < iter->rank; i++) {
iter->to_get *= count[i];
}
return iter->more == 0 ? 0 : iter->to_get ;
}
/* Initialize block iteration for variables, including those that
* won't fit in the copy buffer all at once.
*/
static int
nc_blkio_init(size_t bufsize, /* size in bytes of in-memory copy buffer */
size_t value_size, /* size in bytes of each variable element */
int rank, /* number of dimensions for variable */
const size_t *dims, /* variable dimension sizes */
nciter_t *iter /* returned iteration state, don't mess with it */
) {
int stat = NC_NOERR;
int i;
long long prod;
iter->rank = rank;
iter->first = 1;
iter->more = 1;
for(i = 0; i < rank; i++)
iter->dimsizes[i] = dims[i];
prod = value_size;
iter->right_dim = rank - 1;
for(i = rank; i > 0; i--) {
if(prod*dims[i-1] <= bufsize) {
prod *= dims[i-1];
iter->right_dim--;
} else {
break;
}
}
if (i > 0) { /* variable won't fit in bufsize bytes */
iter->rows = bufsize/prod;
iter->numrows = dims[iter->right_dim] / iter->rows;
iter->leftover = dims[iter->right_dim] - iter->numrows * iter->rows;
iter->cur = 1;
iter->inc = iter->rows;
return stat;
}
/* else, variable will fit in bufsize bytes of memory. */
iter->right_dim = 0;
iter->rows = dims[0];
iter->inc = 0;
return stat;
}
/*
* Updates a vector of size_t, odometer style. Returns 0 if odometer
* overflowed, else 1.
*/
static int
up_start(
int ndims, /* Number of dimensions */
const size_t *dims, /* The "odometer" limits for each dimension */
int incdim, /* the odmometer increment dimension */
size_t inc, /* the odometer increment for that dimension */
size_t* odom /* The "odometer" vector to be updated */
)
{
int id;
int ret = 1;
if(inc == 0) {
return 0;
}
odom[incdim] += inc;
for (id = incdim; id > 0; id--) {
if(odom[id] >= dims[id]) {
odom[id-1]++;
odom[id] -= dims[id];
}
}
if (odom[0] >= dims[0])
ret = 0;
return ret;
}
static int
nciter_ndims(Symbol* sym, int* ndimsp)
{
if(ndimsp) *ndimsp = sym->typ.dimset.ndims;
return NC_NOERR;
}
static int
nciter_dimlens(Symbol* sym, size_t* dimlens)
{
int i;
int ndims = sym->typ.dimset.ndims;
for(i=0;i<ndims;i++) {
Symbol* dim = sym->typ.dimset.dimsyms[i];
dimlens[i] = dim->dim.declsize;
}
return NC_NOERR;
}
static int
nciter_vartype(Symbol* sym, Symbol** tsymp)
{
if(tsymp)
*tsymp = sym->typ.basetype;
return NC_NOERR;
}
/* For ncgen, we want the iterator to return
dimension indices, not offsets, so force size
to be one.
*/
static int
nciter_valuesize(Symbol* tsym, size_t* valuesizep)
{
if(valuesizep)
*valuesizep = 1;
return NC_NOERR;
}

52
ncgen/nc_iter.h

@ -1,52 +0,0 @@
/*********************************************************************
* Copyright 2018, University Corporation for Atmospheric Research
* See netcdf/README file for copying and redistribution conditions.
* "$Id $"
*********************************************************************/
#ifndef _NC_ITER_H
#define _NC_ITER_H
#include <netcdf.h>
#if defined(__cplusplus)
extern "C" {
#endif
/*
* The opaque structure to hold per-variable state of iteration
*/
typedef struct {
int first; /* false after associated next function invoked */
int right_dim; /* rightmost dimension for start of variable pieces */
size_t rows; /* how many subpiece rows in bufsiz */
size_t numrows; /* how many row pieces in right_dim dimension */
size_t cur; /* current "row" in loop over row pieces */
size_t leftover; /* how many rows left over after partitioning
* bufsiz into subpiece blocks */
int more; /* whether there is more data still to get */
size_t to_get; /* number of values to get on this access */
int rank; /* number of dimensions */
size_t inc; /* increment for right_dim element of start vector */
size_t dimsizes[NC_MAX_VAR_DIMS];
} nciter_t;
/*
* The Interface
*/
/* Get iterator for a variable. */
extern int
nc_get_iter(Symbol*, size_t bufsize, nciter_t *iterp);
/* Iterate over blocks of variable values, using start and count
* vectors. Returns number of values to access (product of counts),
* or 0 if done. */
extern size_t
nc_next_iter(nciter_t *iterp, size_t *start, size_t *count);
#if defined(__cplusplus)
}
#endif
#endif /* _NC_ITER_H */

199
ncgen/ncdumpchunks.c Executable file

@ -0,0 +1,199 @@
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <netcdf.h>
#include <hdf5.h>
#include <H5DSpublic.h>
#if 0
#define FILE_NAME "tst_vars.nc"
#define VAR "v1"
#define RANK 3
static const size_t dimlens[RANK] = {4,4,2};
static const size_t chunksize[RANK] = {2,2,2};
static const size_t chunkcount[RANK] = {2,2,1};
static const size_t CHUNKPROD = 8;
#endif
typedef struct Odometer {
size_t rank; /*rank */
size_t stop[NC_MAX_VAR_DIMS];
size_t max[NC_MAX_VAR_DIMS]; /* max size of ith index */
size_t index[NC_MAX_VAR_DIMS]; /* current value of the odometer*/
} Odometer;
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
Odometer* odom_new(size_t rank, const size_t* stop, const size_t* max);
void odom_free(Odometer* odom);
int odom_more(Odometer* odom);
int odom_next(Odometer* odom);
size_t* odom_indices(Odometer* odom);
size_t odom_offset(Odometer* odom);
void setoffset(Odometer* odom, size_t* chunksizes, hsize_t* offset);
static void
usage(int err)
{
if(err != 0) {
fprintf(stderr,"Error: (%d) %s\n",err,nc_strerror(err));
}
fprintf(stderr,"usage: ncdumpchunks <file> <var>\n");
fflush(stderr);
exit(1);
}
int
main(int argc, char** argv)
{
int i,r,stat = NC_NOERR;
hid_t fileid, grpid, datasetid;
hid_t dxpl_id = H5P_DEFAULT; /*data transfer property list */
int* chunkdata = NULL; /*[CHUNKPROD];*/
unsigned int filter_mask = 0;
const char* file_name = NULL;
const char* var_name = NULL;
int ncid, varid, dimids[NC_MAX_VAR_DIMS];
int rank, vtype, storage;
size_t dimlens[NC_MAX_VAR_DIMS];
size_t chunklens[NC_MAX_VAR_DIMS];
size_t chunkcounts[NC_MAX_VAR_DIMS];
size_t chunkprod;
Odometer* odom = NULL;
hsize_t offset[NC_MAX_VAR_DIMS];
if(argc < 3)
usage(0);
file_name = argv[1];
var_name = argv[2];
/* Get the info about the var */
if((stat=nc_open(file_name,0,&ncid))) usage(stat);
if((stat=nc_inq_varid(ncid,var_name,&varid))) usage(stat);
if((stat=nc_inq_var(ncid,varid,NULL,&vtype,&rank,dimids,NULL))) usage(stat);
if(rank == 0) usage(NC_EDIMSIZE);
if((stat=nc_inq_var_chunking(ncid,varid,&storage,chunklens))) usage(stat);
if(storage != NC_CHUNKED) usage(NC_EBADCHUNK);
chunkprod = 1;
for(i=0;i<rank;i++) {
if((stat=nc_inq_dimlen(ncid,dimids[i],&dimlens[i]))) usage(stat);
chunkcounts[i] = ceildiv(dimlens[i],chunklens[i]);
chunkprod *= chunklens[i];
}
if((stat=nc_close(ncid))) usage(stat);
if ((fileid = H5Fopen(file_name, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((grpid = H5Gopen(fileid, "/", H5P_DEFAULT)) < 0) usage(NC_EHDFERR);
if ((datasetid = H5Dopen1(grpid, var_name)) < 0) usage(NC_EHDFERR);
if((odom = odom_new(rank,chunkcounts,dimlens))==NULL) usage(NC_ENOMEM);
if((chunkdata = calloc(sizeof(int),chunkprod))==NULL) usage(NC_ENOMEM);
while(odom_more(odom)) {
setoffset(odom,chunklens,offset);
#ifdef DEBUG
fprintf(stderr,"(");
for(i=0;i<rank;i++)
fprintf(stderr,"%s%lu",(i > 0 ? "," : ""),(unsigned long)offset[i]);
fprintf(stderr,")\n");
fflush(stderr);
#endif
memset(chunkdata,0,sizeof(int)*chunkprod);
#if 1
if(H5Dread_chunk(datasetid, dxpl_id, offset, &filter_mask, chunkdata) < 0) abort();
for(r=0;r<rank;r++)
printf("[%lu/%lu]",(unsigned long)odom->index[r],(unsigned long)offset[r]);
printf(" =");
for(r=0;r<chunkprod;r++)
printf(" %02d", chunkdata[r]);
printf("\n");
fflush(stdout);
#endif
odom_next(odom);
}
/* Close up. */
if (H5Dclose(datasetid) < 0) abort();
if (H5Gclose(grpid) < 0) abort();
if (H5Fclose(fileid) < 0) abort();
/* Cleanup */
free(chunkdata);
odom_free(odom);
return 0;
}
Odometer*
odom_new(size_t rank, const size_t* stop, const size_t* max)
{
int i;
Odometer* odom = NULL;
if((odom = calloc(1,sizeof(Odometer))) == NULL)
return NULL;
odom->rank = rank;
for(i=0;i<rank;i++) {
odom->stop[i] = stop[i];
odom->max[i] = max[i];
odom->index[i] = 0;
}
return odom;
}
void
odom_free(Odometer* odom)
{
if(odom) free(odom);
}
int
odom_more(Odometer* odom)
{
return (odom->index[0] < odom->stop[0]);
}
int
odom_next(Odometer* odom)
{
size_t i;
for(i=odom->rank-1;i>=0;i--) {
odom->index[i]++;
if(odom->index[i] < odom->stop[i]) break;
if(i == 0) return 0; /* leave the 0th entry if it overflows */
odom->index[i] = 0; /* reset this position */
}
return 1;
}
/* Get the value of the odometer */
size_t*
odom_indices(Odometer* odom)
{
return odom->index;
}
size_t
odom_offset(Odometer* odom)
{
size_t offset;
int i;
offset = 0;
for(i=0;i<odom->rank;i++) {
offset *= odom->max[i];
offset += odom->index[i];
}
return offset;
}
void
setoffset(Odometer* odom, size_t* chunksizes, hsize_t* offset)
{
int i;
for(i=0;i<odom->rank;i++)
offset[i] = odom->index[i] * chunksizes[i];
}

8
ncgen/ncgen.1

@ -9,7 +9,6 @@ ncgen
\%[\-\fIformat_code\fP]
\%[\-1|3|4|5|6|7]
\%[\-b]
\%[\-B \fIbuffersize\fP]
\%[\-c]
\%[\-d]
\%[\-D \fIdebuglevel\fP]
@ -24,6 +23,7 @@ ncgen
\%[\-N \fIdatasetname\fP]
\%[\-o \fInetcdf_filename\fP]
\%[\-P]
.\%[\-W \fImaxwholevarsize\fP]
\%[\-x]
.hy
.ft
@ -69,8 +69,6 @@ Create a (binary) netCDF file. If the \fB-o\fP option is absent, a
default file name will be constructed from the basename of the CDL
file, with any suffix replaced by the `.nc' extension. If a
file already exists with the specified name, it will be overwritten.
.IP "\fB-B\fP \fRbuffersize\fP\fP"
Specify the internal iterator buffer size.
.IP "\fB-c\fP"
Generate
.B C
@ -205,6 +203,10 @@ seekable.
.IP "\fB-P\fP"
Use NC_DISKLESS mode to create the file totally in memory
before persisting it to disk.
.IP "\fB-W\fP \fRmaxwholevarsize\fP"
Set wholevarsizem where if total number of elements is less than maxwholevarsize
then updata a variable using a single nc_put_var. Requires
that the variable has no unlimited dimensions.
.IP "\fB-x\fP"
Don't initialize data with fill values. This can speed up creation of
large netCDF files greatly, but later attempts to read unwritten data

4
ncgen/ncgen.h

@ -127,7 +127,7 @@ extern struct Kvalues legalkinds[];
typedef struct Specialdata {
int flags;
Datalist* _Fillvalue; /* This is a per-type ; points to the _FillValue attribute node */
int _Storage; /* NC_CHUNKED | NC_CONTIGUOUS*/
int _Storage; /* NC_CHUNKED | NC_CONTIGUOUS | NC_COMPACT*/
size_t* _ChunkSizes; /* NULL => defaults*/
int nchunks; /* |_Chunksize| ; 0 => not specified*/
int _Fletcher32; /* 1=>fletcher32*/
@ -194,7 +194,7 @@ typedef struct Typeinfo {
typedef struct Varinfo {
int nattributes; /* |attributes|*/
List* attributes; /* List<Symbol*>*/
Specialdata* special;
Specialdata special;
} Varinfo;
typedef struct Groupinfo {

13
ncgen/ncgen.y

@ -108,7 +108,6 @@ List* gattdefs; /* global attributes only*/
List* xattdefs; /* unknown attributes*/
List* typdefs;
List* vardefs;
List* condefs; /* non-dimension constants used in type defs*/
List* tmp;
/* Forward */
@ -522,9 +521,6 @@ vardecl: typeref varlist
sym->typ.basetype = $1;
addtogroup(sym);
listpush(vardefs,(void*)sym);
sym->var.special = ecalloc(sizeof(Specialdata));
if(sym->var.special == NULL)
derror("out of memory");
}
}
listsetlength(stack,stackbase);/* remove stack nodes*/
@ -941,7 +937,6 @@ parse_init(void)
xattdefs = listnew();
typdefs = listnew();
vardefs = listnew();
condefs = listnew();
tmp = listnew();
/* Create the primitive types */
for(i=NC_NAT+1;i<=NC_STRING;i++) {
@ -978,6 +973,7 @@ install(const char *sname)
sp->lineno = lineno;
sp->location = currentgroup();
sp->container = currentgroup();
sp->var.special._Storage = NC_CONTIGUOUS;
listpush(symlist,sp);
return sp;
}
@ -1267,12 +1263,7 @@ makespecial(int tag, Symbol* vsym, Symbol* tsym, void* data, int isconst)
} else {
Specialdata* special;
/* Set up special info */
if(vsym->var.special == NULL) {
vsym->var.special = ecalloc(sizeof(Specialdata));
if(vsym->var.special == NULL)
derror("Out of memory");
}
special = vsym->var.special;
special = &vsym->var.special;
if(tag == _FILLVALUE_FLAG) {
/* fillvalue must be a single value*/
if(!isconst && datalistlen(list) != 1)

868
ncgen/ncgenl.c

File diff suppressed because it is too large Load Diff

561
ncgen/ncgeny.c

File diff suppressed because it is too large Load Diff

2
ncgen/ncgeny.h

@ -106,7 +106,7 @@ extern int ncgdebug;
union YYSTYPE
{
#line 152 "ncgen.y" /* yacc.c:1909 */
#line 151 "ncgen.y" /* yacc.c:1909 */
Symbol* sym;
unsigned long size; /* allow for zero size to indicate e.g. UNLIMITED*/

296
ncgen/semantics.c

@ -10,6 +10,9 @@
#include "ncoffsets.h"
#include "netcdf_aux.h"
#define floordiv(x,y) ((x) / (y))
#define ceildiv(x,y) (((x) % (y)) == 0 ? ((x) / (y)) : (((x) / (y)) + 1))
/* Forward*/
static void filltypecodes(void);
static void processenums(void);
@ -22,6 +25,7 @@ static void processunlimiteddims(void);
static void processeconstrefs(void);
static void processeconstrefsR(Symbol*,Datalist*);
static void processroot(void);
static void processvardata(void);
static void computefqns(void);
static void fixeconstref(Symbol*,NCConstant* con);
@ -33,13 +37,6 @@ static void computefqns(void);
static Symbol* uniquetreelocate(Symbol* refsym, Symbol* root);
static char* createfilename(void);
#if 0
static Symbol* locateenumtype(Symbol* econst, Symbol* group, NCConstant*);
static List* findecmatches(char* ident);
static List* ecsearchgrp(Symbol* grp, List* candidates);
static Symbol* checkeconst(Symbol* en, const char* refname);
#endif
List* vlenconstants; /* List<Constant*>;*/
/* ptr to vlen instances across all datalists*/
@ -67,6 +64,8 @@ processsemantics(void)
processeconstrefs();
/* Compute the unlimited dimension sizes */
processunlimiteddims();
/* Rebuild var datalists to show dim levels */
processvardata();
/* check internal consistency*/
checkconsistency();
}
@ -404,9 +403,6 @@ static void
processenums(void)
{
unsigned long i,j;
#if 0 /* Unused? */
List* enumids = listnew();
#endif
for(i=0;i<listlength(typdefs);i++) {
Symbol* sym = (Symbol*)listget(typdefs,i);
ASSERT(sym->objectclass == NC_TYPE);
@ -414,9 +410,6 @@ processenums(void)
for(j=0;j<listlength(sym->subnodes);j++) {
Symbol* esym = (Symbol*)listget(sym->subnodes,j);
ASSERT(esym->subclass == NC_ECONST);
#if 0 /* Unused? */
listpush(enumids,(void*)esym);
#endif
}
}
/* Convert enum values to match enum type*/
@ -458,8 +451,8 @@ processeconstrefs(void)
Symbol* var = (Symbol*)listget(vardefs,i);
if(var->data != NULL && listlength(var->data) > 0)
processeconstrefsR(var,var->data);
if(var->var.special->_Fillvalue != NULL)
processeconstrefsR(var,var->var.special->_Fillvalue);
if(var->var.special._Fillvalue != NULL)
processeconstrefsR(var,var->var.special._Fillvalue);
}
}
@ -531,149 +524,6 @@ fixeconstref(Symbol* avsym, NCConstant* con)
semerror(con->lineno,"Undefined enum or enum constant reference: %s",refsym->name);
}
#if 0
/* If we have an enum-valued group attribute, then we need to do
extra work to find the containing enum type
*/
static Symbol*
locateenumtype(Symbol* refsym, Symbol* parent, NCConstant* con)
{
Symbol* match = NULL;
List* grpmatches;
/* Locate all possible matching enum constant definitions */
List* candidates = findecmatches(refsym->name);
if(candidates == NULL) {
semerror(con->lineno,"Undefined enum or enum constant reference: %s",refsym->name);
return NULL;
}
/* One hopes that 99% of the time, the match is unique */
if(listlength(candidates) == 1) {
match = listget(candidates,0);
goto done;
}
/* If this ref has a specified group prefix, then find that group
and search only within it for matches to the candidates */
if(refsym->is_prefixed && refsym->prefix != NULL) {
parent = lookupgroup(refsym->prefix);
if(parent == NULL) {
semerror(con->lineno,"Undefined group reference: ",fullname(refsym));
goto done;
}
/* Search this group only for matches */
grpmatches = ecsearchgrp(parent,candidates);
switch (listlength(grpmatches)) {
case 0:
semerror(con->lineno,"Undefined enum or enum constant reference: ",refsym->name);
listfree(grpmatches);
goto done;
case 1:
break;
default:
semerror(con->lineno,"Ambiguous enum constant reference: %s", fullname(refsym));
}
match = listget(grpmatches,0);
listfree(grpmatches);
goto done;
}
/* Sigh, we have to search up the tree to see if any of our candidates are there */
assert(parent == NULL || parent->objectclass == NC_GRP);
while(parent != NULL && match == NULL) {
grpmatches = ecsearchgrp(parent,candidates);
switch (listlength(grpmatches)) {
case 0: break;
case 1: match = listget(grpmatches,0); break;
default:
semerror(con->lineno,"Ambiguous enum constant reference: %s", fullname(refsym));
match = listget(grpmatches,0);
break;
}
listfree(grpmatches);
}
if(match != NULL) goto done;
/* Not unique and not in the parent tree, so complains and pick the first candidate */
semerror(con->lineno,"Ambiguous enum constant reference: %s", fullname(refsym));
match = (Symbol*)listget(candidates,0);
done:
listfree(candidates);
return match;
}
/*
Locate enums whose name is a prefix of ident
and contains the suffix as an enum const
and capture that enum constant.
*/
static List*
findecmatches(char* ident)
{
List* matches = listnew();
int i;
for(i=0;i<listlength(typdefs);i++) {
int len;
Symbol* ec;
Symbol* en = (Symbol*)listget(typdefs,i);
if(en->subclass != NC_ENUM)
continue;
/* First, assume that the ident is the econst name only */
ec = checkeconst(en,ident);
if(ec != NULL)
listpush(matches,ec);
/* Second, do the prefix check */
len = strlen(en->name);
if(strncmp(ident,en->name,len) == 0) {
Symbol *ec;
/* Find the matching ec constant, if any */
if(*(ident+len) != '.') continue;
ec = checkeconst(en,ident+len+1); /* +1 for the dot */
if(ec != NULL)
listpush(matches,ec);
}
}
if(listlength(matches) == 0) {
listfree(matches);
matches = NULL;
}
return matches;
}
static List*
ecsearchgrp(Symbol* grp, List* candidates)
{
List* matches = listnew();
int i,j;
/* do the intersection of grp subnodes and candidates */
for(i=0;i<listlength(grp->subnodes);i++) {
Symbol* sub= (Symbol*)listget(grp->subnodes,i);
if(sub->subclass != NC_ENUM)
continue;
for(j=0;j<listlength(candidates);j++) {
Symbol* ec = (Symbol*)listget(candidates,j);
if(ec->container == sub)
listpush(matches,ec);
}
}
if(listlength(matches) == 0) {
listfree(matches);
matches = NULL;
}
return matches;
}
static Symbol*
checkeconst(Symbol* en, const char* refname)
{
int i;
for(i=0;i<listlength(en->subnodes);i++) {
Symbol* ec = (Symbol*)listget(en->subnodes,i);
if(strcmp(ec->name,refname) == 0)
return ec;
}
return NULL;
}
#endif
/* Compute type sizes and compound offsets*/
void
computesize(Symbol* tsym)
@ -832,9 +682,9 @@ processattributes(void)
/* Generate a default fill value */
asym->data = getfiller(asym->typ.basetype);
}
if(asym->att.var->var.special->_Fillvalue != NULL)
reclaimdatalist(asym->att.var->var.special->_Fillvalue);
asym->att.var->var.special->_Fillvalue = clonedatalist(asym->data);
if(asym->att.var->var.special._Fillvalue != NULL)
reclaimdatalist(asym->att.var->var.special._Fillvalue);
asym->att.var->var.special._Fillvalue = clonedatalist(asym->data);
} else if(asym->typ.basetype == NULL) {
inferattributetype(asym);
}
@ -1206,7 +1056,10 @@ thisunlim->name,
length += con->value.stringv.len;
break;
case NC_COMPOUND:
semwarn(datalistline(data),"Expected character constant, found {...}");
if(con->subtype == NC_DIM)
semwarn(datalistline(data),"Expected character constant, found {...}");
else
semwarn(datalistline(data),"Expected character constant, found (...)");
break;
default:
semwarn(datalistline(data),"Illegal character constant: %d",con->nctype);
@ -1314,3 +1167,122 @@ createfilename(void)
}
return strdup(filename);
}
#if 1
/* Recursive helper for processevardata */
static NCConstant*
processvardataR(Symbol* vsym, Dimset* dimset, Datalist* data, int dimindex)
{
int rank;
size_t offset;
Datalist* newlist = NULL;
NCConstant* result;
size_t datalen;
Symbol* dim;
rank = dimset->ndims;
dim = dimset->dimsyms[dimindex];
if(rank == 0) {/* scalar */
ASSERT((datalistlen(data) == 1));
/* return clone of this data */
newlist = clonedatalist(data);
goto done;
}
/* four cases to consider: (dimindex==rank-1 vs dimindex < rank-1) X (unlimited vs fixedsize)*/
if(dimindex == (rank-1)) {/* Stop recursion here */
if(dim->dim.isunlimited) {
/* return clone of this data */
newlist = clonedatalist(data);
} else { /* !unlimited */
/* return clone of this data */
newlist = clonedatalist(data);
}
} else {/* dimindex < rank-1 */
NCConstant* datacon;
Datalist* actual;
if(dim->dim.isunlimited && dimindex > 0) {
/* Should have a sequence of {..} representing the unlimited in next dimension
so, unbpack compound */
ASSERT(datalistlen(data) == 1);
datacon = datalistith(data,0);
actual = compoundfor(datacon);
} else
actual = data;
/* fall through */
{
newlist = builddatalist(0);
datalen = datalistlen(actual);
/* So we have a block of dims starting here */
int nextunlim = findunlimited(dimset,dimindex+1);
size_t dimblock;
int i;
/* compute the size of the subblocks */
for(dimblock=1,i=dimindex+1;i<nextunlim;i++)
dimblock *= dimset->dimsyms[i]->dim.declsize;
/* Divide this datalist into dimblock size sublists; last may be short and process each */
for(offset=0;;offset+=dimblock) {
size_t blocksize;
Datalist* subset;
NCConstant* newcon;
blocksize = (offset < datalen ? dimblock : (datalen - offset));
subset = builddatasubset(actual,offset,blocksize);
/* Construct a datalist to hold processed subset */
newcon = processvardataR(vsym,dimset,subset,dimindex+1);
dlappend(newlist,newcon);
reclaimdatalist(subset);
if((offset+blocksize) >= datalen) break; /* done */
}
}
}
done:
result = list2const(newlist);
setsubtype(result,NC_DIM);
return result;
}
/* listify n-dimensional data lists */
static void
processvardata(void)
{
int i;
for(i=0;i<listlength(vardefs);i++) {
Symbol* vsym = (Symbol*)listget(vardefs,i);
NCConstant* con;
if(vsym->data == NULL) continue;
/* Let char typed vars be handled by genchararray */
if(vsym->typ.basetype->typ.typecode == NC_CHAR) continue;
con = processvardataR(vsym,&vsym->typ.dimset,vsym->data,0);
reclaimdatalist(vsym->data);
ASSERT((islistconst(con)));
vsym->data = compoundfor(con);
clearconstant(con);
freeconst(con);
}
}
/* Convert char strings to 'x''... form */
Datalist*
explode(NCConstant* con)
{
int i;
char* p;
size_t len;
Datalist* chars;
ASSERT((con->nctype == NC_STRING));
len = con->value.stringv.len;
chars = builddatalist(len);
p = con->value.stringv.stringv;
fprintf(stderr,"p[%d]=|%s|\n",con->value.stringv.len,p);
for(i=0;i<len;i++,p++) {
NCConstant* chcon = nullconst();
chcon->nctype = NC_CHAR;
chcon->value.charv = *p;
dlappend(chars,chcon);
}
fprintf(stderr,"|chars|=%d\n",(int)datalistlen(chars));
return chars;
}
#endif

40
ncgen/util.c

@ -99,19 +99,8 @@ tztrim(
return;
}
#if 0
/* Assume bytebuffer contains pointers to char**/
void
reclaimattptrs(void* buf, long count)
{
int i;
char** ptrs = (char**)buf;
for(i=0;i<count;i++) {efree((void*)ptrs[i]);}
}
#endif
static void
freeSpecialdata(Specialdata* data)
clearSpecialdata(Specialdata* data)
{
if(data == NULL) return;
reclaimdatalist(data->_Fillvalue);
@ -126,16 +115,15 @@ freeSpecialdata(Specialdata* data)
}
efree(data->_Filters);
}
efree(data);
}
void
freeSymbol(Symbol* sym)
{
/* recurse first */
if(sym == NULL) return;
switch (sym->objectclass) {
case NC_VAR:
freeSpecialdata(sym->var.special);
clearSpecialdata(&sym->var.special);
listfree(sym->var.attributes);
break;
case NC_TYPE:
@ -554,7 +542,17 @@ reclaimSymbols(void)
void
cleanup()
{
reclaimSymbols();
reclaimSymbols();
listfree(symlist);
listfree(grpdefs);
listfree(dimdefs);
listfree(attdefs);
listfree(gattdefs);
listfree(xattdefs);
listfree(typdefs);
listfree(vardefs);
filldatalist->readonly = 0;
freedatalist(filldatalist);
}
/* compute the total n-dimensional size as 1 long array;
@ -589,18 +587,19 @@ extern int H5Eprint1(FILE * stream);
#endif
void
check_err(const int stat, const int line, const char* file)
check_err(const int stat, const int line, const char* file, const char* func)
{
check_err2(stat,-1,line,file);
check_err2(stat,-1,line,file,func);
}
void check_err2(const int stat, const int cdlline, const int line, const char* file) {
void check_err2(const int stat, const int cdlline, const int line, const char* file, const char* func)
{
if (stat != NC_NOERR) {
if(cdlline >= 0)
fprintf(stderr, "ncgen: cdl line %d; %s\n", cdlline, nc_strerror(stat));
else
fprintf(stderr, "ncgen: %s\n", nc_strerror(stat));
fprintf(stderr, "\t(%s:%d)\n", file,line);
fprintf(stderr, "\t(%s:%s:%d)\n", file,func,line);
#ifdef USE_HDF5
H5Eprint1(stderr);
#endif
@ -668,3 +667,4 @@ kind_string(int kind)
}
return NULL;
}

4
ncgen/util.h

@ -58,8 +58,8 @@ extern int countunlimited(Dimset* dimset);
extern unsigned char* makebytestring(char* s, size_t* lenp);
extern int getpadding(int offset, int alignment);
extern void check_err(const int stat, const int line, const char* file);
extern void check_err2(const int stat, const int cdlline, const int line, const char* file);
extern void check_err(const int stat, const int line, const char* file, const char* func);
extern void check_err2(const int stat, const int cdlline, const int line, const char* file, const char* func);
extern const char* kind_string(int kind);
extern int getrootid(int grpid);

3
oc2/oc.c

@ -5,6 +5,9 @@
#include <stdlib.h>
#include <assert.h>
#include <string.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include "netcdf.h"
#include "ocinternal.h"

13
oc2/occurlfunctions.c

@ -20,16 +20,6 @@
/* Define some .rc file entries of interest*/
#define NETRCFILETAG "HTTP.NETRC"
/* Check return value */
#define CHECK(state,flag,value) {if(check(state,flag,(void*)value) != OC_NOERR) {goto done;}}
static OCerror
check(OCstate* state, int flag, void* value)
{
OCerror stat = ocset_curlopt(state,flag,value);
return stat;
}
/*
Set a specific curl flag; primary wrapper for curl_easy_setopt
*/
@ -44,6 +34,9 @@ ocset_curlopt(OCstate* state, int flag, void* value)
return stat;
}
/* Check return value */
#define CHECK(state,flag,value) {if(ocset_curlopt(state,flag,(void*)value) != OC_NOERR) {goto done;}}
/*
Update a specific flag from state
*/

3
oc2/ocinternal.c

@ -4,6 +4,9 @@
#include "config.h"
#include <stdio.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif

3
oc2/ocread.c

@ -8,6 +8,9 @@
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif