2017-03-09 08:01:10 +08:00
|
|
|
/*********************************************************************
|
2018-12-07 05:24:28 +08:00
|
|
|
* Copyright 2018, UCAR/Unidata
|
2017-03-09 08:01:10 +08:00
|
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
|
|
*********************************************************************/
|
|
|
|
#include "config.h"
|
|
|
|
#include <stdarg.h>
|
2023-11-27 19:36:03 +08:00
|
|
|
#include <stddef.h>
|
2017-03-09 08:01:10 +08:00
|
|
|
#include <stdio.h>
|
|
|
|
|
|
|
|
#include "d4includes.h"
|
|
|
|
#include "ncdispatch.h"
|
2018-11-16 01:00:38 +08:00
|
|
|
#include "netcdf_aux.h"
|
2017-03-09 08:01:10 +08:00
|
|
|
|
Improve performance of the nc_reclaim_data and nc_copy_data functions.
re: Issue https://github.com/Unidata/netcdf-c/issues/2685
re: PR https://github.com/Unidata/netcdf-c/pull/2179
As noted in PR https://github.com/Unidata/netcdf-c/pull/2179,
the old code did not allow for reclaiming instances of types,
nor for properly copying them. That PR provided new functions
capable of reclaiming/copying instances of arbitrary types.
However, as noted by Issue https://github.com/Unidata/netcdf-c/issues/2685, using these
most general functions resulted in a significant performance
degradation, even for common cases.
This PR attempts to mitigate the cost of using the general
reclaim/copy functions in two ways.
First, the previous functions operating at the top level by
using ncid and typeid arguments. These functions were augmented
with equivalent versions that used the netcdf-c library internal
data structures to allow direct access to needed information.
These new functions are used internally to the library.
The second mitigation involves optimizing the internal functions
by providing early tests for common cases. This avoids
unnecessary recursive function calls.
The overall result is a significant improvement in speed by a
factor of roughly twenty -- your mileage may vary. These
optimized functions are still not as fast as the original (more
limited) functions, but they are getting close. Additional optimizations are
possible. But the cost is a significant "uglification" of the
code that I deemed a step too far, at least for now.
## Misc. Changes
1. Added a test case to check the proper reclamation/copy of complex types.
2. Found and fixed some places where nc_reclaim/copy should have been used.
3. Replaced, in the netcdf-c library, (almost all) occurrences of nc_reclaim_copy with calls to NC_reclaim/copy. This plus the optimizations is the primary speed-up mechanism.
4. In DAP4, the metadata is held in a substrate in-memory file; this required some changes so that the reclaim/copy code accessed that substrate dispatcher rather than the DAP4 dispatcher.
5. Re-factored and isolated the code that computes if a type is (transitively) variable-sized or not.
6. Clean up the reclamation code in ncgen; adding the use of nc_reclaim exposed some memory problems.
2023-05-21 07:11:25 +08:00
|
|
|
extern NC* NCD4_get_substrate(NC* nc);
|
|
|
|
|
2017-03-09 08:01:10 +08:00
|
|
|
#ifdef D4CATCH
|
|
|
|
/* Place breakpoint here to catch errors close to where they occur*/
|
|
|
|
int
|
2022-11-14 04:15:11 +08:00
|
|
|
d4breakpoint(int err) {
|
|
|
|
return err;
|
|
|
|
}
|
2017-03-09 08:01:10 +08:00
|
|
|
|
|
|
|
int
|
|
|
|
d4throw(int err)
|
|
|
|
{
|
|
|
|
if(err == 0) return err;
|
|
|
|
return d4breakpoint(err);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
int
|
|
|
|
d4panic(const char* fmt, ...)
|
|
|
|
{
|
|
|
|
va_list args;
|
|
|
|
if(fmt != NULL) {
|
|
|
|
va_start(args, fmt);
|
|
|
|
vfprintf(stderr, fmt, args);
|
|
|
|
fprintf(stderr, "\n" );
|
|
|
|
va_end( args );
|
|
|
|
} else {
|
|
|
|
fprintf(stderr, "panic" );
|
|
|
|
}
|
|
|
|
fprintf(stderr, "\n" );
|
|
|
|
fflush(stderr);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
const char*
|
|
|
|
NCD4_sortname(NCD4sort sort)
|
|
|
|
{
|
|
|
|
switch (sort) {
|
|
|
|
case NCD4_NULL: return "NCD4_NULL";
|
|
|
|
case NCD4_ATTR: return "NCD4_ATTR";
|
|
|
|
case NCD4_ATTRSET: return "NCD4_ATTRSET";
|
|
|
|
case NCD4_XML: return "NCD4_XML";
|
|
|
|
case NCD4_DIM: return "NCD4_DIM";
|
|
|
|
case NCD4_GROUP: return "NCD4_GROUP";
|
|
|
|
case NCD4_TYPE: return "NCD4_TYPE";
|
|
|
|
case NCD4_VAR: return "NCD4_VAR";
|
|
|
|
case NCD4_ECONST: return "NCD4_ECONST";
|
|
|
|
default: break;
|
|
|
|
}
|
2022-11-14 04:15:11 +08:00
|
|
|
return "sort.unknown";
|
2017-03-09 08:01:10 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
const char*
|
|
|
|
NCD4_subsortname(nc_type subsort)
|
|
|
|
{
|
|
|
|
switch (subsort) {
|
|
|
|
case NC_NAT: return "NC_NAT";
|
|
|
|
case NC_BYTE: return "NC_BYTE";
|
|
|
|
case NC_CHAR: return "NC_CHAR";
|
|
|
|
case NC_SHORT: return "NC_SHORT";
|
|
|
|
case NC_INT: return "NC_INT";
|
|
|
|
case NC_FLOAT: return "NC_FLOAT";
|
|
|
|
case NC_DOUBLE: return "NC_DOUBLE";
|
|
|
|
case NC_UBYTE: return "NC_UBYTE";
|
|
|
|
case NC_USHORT: return "NC_USHORT";
|
|
|
|
case NC_UINT: return "NC_UINT";
|
|
|
|
case NC_INT64: return "NC_INT64";
|
|
|
|
case NC_UINT64: return "NC_UINT64";
|
|
|
|
case NC_STRING: return "NC_STRING";
|
2022-11-14 04:15:11 +08:00
|
|
|
case NC_VLEN: return "NC_SEQ";
|
2017-03-09 08:01:10 +08:00
|
|
|
case NC_OPAQUE: return "NC_OPAQUE";
|
|
|
|
case NC_ENUM: return "NC_ENUM";
|
2022-11-14 04:15:11 +08:00
|
|
|
case NC_COMPOUND: return "NC_STRUCT";
|
2017-03-09 08:01:10 +08:00
|
|
|
default: break;
|
|
|
|
}
|
2022-11-14 04:15:11 +08:00
|
|
|
return "subsort.unknown";
|
2017-03-09 08:01:10 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
For debugging purposes, it is desirable to fake an nccopy
|
|
|
|
bv inserting the data into the substrate and then writing it out.
|
|
|
|
*/
|
|
|
|
|
|
|
|
int
|
|
|
|
NCD4_debugcopy(NCD4INFO* info)
|
|
|
|
{
|
2023-11-27 19:36:03 +08:00
|
|
|
size_t i;
|
|
|
|
int ret = NC_NOERR;
|
2023-10-09 09:59:28 +08:00
|
|
|
NCD4meta* meta = info->dmrmetadata;
|
2017-03-09 08:01:10 +08:00
|
|
|
NClist* topvars = nclistnew();
|
|
|
|
NC* ncp = info->controller;
|
2021-01-15 12:39:08 +08:00
|
|
|
void* memory = NULL;
|
2017-03-09 08:01:10 +08:00
|
|
|
|
|
|
|
/* Walk each top level variable, read all of it and write it to the substrate */
|
|
|
|
if((ret=NCD4_getToplevelVars(meta, NULL, topvars)))
|
|
|
|
goto done;
|
|
|
|
/* Read from the dap data by going thru the dap4 interface */
|
|
|
|
for(i=0;i<nclistlength(topvars);i++) {
|
|
|
|
NCD4node* var = nclistget(topvars,i);
|
|
|
|
NCD4node* type = var->basetype;
|
|
|
|
NCD4node* grp = NCD4_groupFor(var);
|
|
|
|
int grpid = grp->meta.id;
|
|
|
|
int varid = var->meta.id;
|
|
|
|
d4size_t varsize;
|
2018-11-16 01:00:38 +08:00
|
|
|
size_t dimprod = NCD4_dimproduct(var);
|
2017-03-09 08:01:10 +08:00
|
|
|
|
2018-11-16 01:00:38 +08:00
|
|
|
varsize = type->meta.memsize * dimprod;
|
2017-03-09 08:01:10 +08:00
|
|
|
memory = d4alloc(varsize);
|
|
|
|
if(memory == NULL)
|
|
|
|
{ret = NC_ENOMEM; goto done;}
|
|
|
|
{
|
|
|
|
/* We need to read via NCD4 */
|
|
|
|
int d4gid = makedap4id(ncp,grpid);
|
|
|
|
if((ret=nc_get_var(d4gid,varid,memory)))
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
/* Now, turn around and write it to the substrate.
|
|
|
|
WARNING: we have to specify the shape ourselves
|
|
|
|
because, if unlimited is involved then there is
|
|
|
|
potentially a difference between the substrate unlimited
|
|
|
|
size and the dap4 data specified size. In fact,
|
|
|
|
the substrate will always be zero unless debugcopy is used.
|
|
|
|
*/
|
|
|
|
{
|
|
|
|
size_t edges[NC_MAX_VAR_DIMS];
|
2023-11-27 19:36:03 +08:00
|
|
|
size_t d;
|
2017-03-09 08:01:10 +08:00
|
|
|
for(d=0;d<nclistlength(var->dims);d++) {
|
|
|
|
NCD4node* dim = (NCD4node*)nclistget(var->dims,d);
|
|
|
|
edges[d] = (size_t)dim->dim.size;
|
|
|
|
}
|
2019-03-31 04:06:20 +08:00
|
|
|
if((ret=nc_put_vara(grpid,varid,NC_coord_zero,edges,memory)))
|
2017-03-09 08:01:10 +08:00
|
|
|
goto done;
|
|
|
|
}
|
Improve performance of the nc_reclaim_data and nc_copy_data functions.
re: Issue https://github.com/Unidata/netcdf-c/issues/2685
re: PR https://github.com/Unidata/netcdf-c/pull/2179
As noted in PR https://github.com/Unidata/netcdf-c/pull/2179,
the old code did not allow for reclaiming instances of types,
nor for properly copying them. That PR provided new functions
capable of reclaiming/copying instances of arbitrary types.
However, as noted by Issue https://github.com/Unidata/netcdf-c/issues/2685, using these
most general functions resulted in a significant performance
degradation, even for common cases.
This PR attempts to mitigate the cost of using the general
reclaim/copy functions in two ways.
First, the previous functions operating at the top level by
using ncid and typeid arguments. These functions were augmented
with equivalent versions that used the netcdf-c library internal
data structures to allow direct access to needed information.
These new functions are used internally to the library.
The second mitigation involves optimizing the internal functions
by providing early tests for common cases. This avoids
unnecessary recursive function calls.
The overall result is a significant improvement in speed by a
factor of roughly twenty -- your mileage may vary. These
optimized functions are still not as fast as the original (more
limited) functions, but they are getting close. Additional optimizations are
possible. But the cost is a significant "uglification" of the
code that I deemed a step too far, at least for now.
## Misc. Changes
1. Added a test case to check the proper reclamation/copy of complex types.
2. Found and fixed some places where nc_reclaim/copy should have been used.
3. Replaced, in the netcdf-c library, (almost all) occurrences of nc_reclaim_copy with calls to NC_reclaim/copy. This plus the optimizations is the primary speed-up mechanism.
4. In DAP4, the metadata is held in a substrate in-memory file; this required some changes so that the reclaim/copy code accessed that substrate dispatcher rather than the DAP4 dispatcher.
5. Re-factored and isolated the code that computes if a type is (transitively) variable-sized or not.
6. Clean up the reclamation code in ncgen; adding the use of nc_reclaim exposed some memory problems.
2023-05-21 07:11:25 +08:00
|
|
|
if((ret=NC_reclaim_data(NCD4_get_substrate(ncp),type->meta.id,memory,dimprod)))
|
2018-11-16 01:00:38 +08:00
|
|
|
goto done;
|
2021-01-15 12:39:08 +08:00
|
|
|
nullfree(memory); memory = NULL;
|
2017-03-09 08:01:10 +08:00
|
|
|
}
|
|
|
|
done:
|
2021-01-15 12:39:08 +08:00
|
|
|
nullfree(memory);
|
|
|
|
nclistfree(topvars);
|
2017-03-09 08:01:10 +08:00
|
|
|
if(ret != NC_NOERR) {
|
|
|
|
fprintf(stderr,"debugcopy: %d %s\n",ret,nc_strerror(ret));
|
|
|
|
}
|
|
|
|
return THROW(ret);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Provide a string printer that can be called from gdb */
|
|
|
|
void
|
|
|
|
NCD4_printstring(const char* s)
|
|
|
|
{
|
|
|
|
fprintf(stderr,"%s\n",s);
|
|
|
|
fflush(stderr);
|
|
|
|
}
|