mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-11-27 07:30:33 +08:00
fb40a72b45
re: Issue https://github.com/Unidata/netcdf-c/issues/2685 re: PR https://github.com/Unidata/netcdf-c/pull/2179 As noted in PR https://github.com/Unidata/netcdf-c/pull/2179, the old code did not allow for reclaiming instances of types, nor for properly copying them. That PR provided new functions capable of reclaiming/copying instances of arbitrary types. However, as noted by Issue https://github.com/Unidata/netcdf-c/issues/2685, using these most general functions resulted in a significant performance degradation, even for common cases. This PR attempts to mitigate the cost of using the general reclaim/copy functions in two ways. First, the previous functions operating at the top level by using ncid and typeid arguments. These functions were augmented with equivalent versions that used the netcdf-c library internal data structures to allow direct access to needed information. These new functions are used internally to the library. The second mitigation involves optimizing the internal functions by providing early tests for common cases. This avoids unnecessary recursive function calls. The overall result is a significant improvement in speed by a factor of roughly twenty -- your mileage may vary. These optimized functions are still not as fast as the original (more limited) functions, but they are getting close. Additional optimizations are possible. But the cost is a significant "uglification" of the code that I deemed a step too far, at least for now. ## Misc. Changes 1. Added a test case to check the proper reclamation/copy of complex types. 2. Found and fixed some places where nc_reclaim/copy should have been used. 3. Replaced, in the netcdf-c library, (almost all) occurrences of nc_reclaim_copy with calls to NC_reclaim/copy. This plus the optimizations is the primary speed-up mechanism. 4. In DAP4, the metadata is held in a substrate in-memory file; this required some changes so that the reclaim/copy code accessed that substrate dispatcher rather than the DAP4 dispatcher. 5. Re-factored and isolated the code that computes if a type is (transitively) variable-sized or not. 6. Clean up the reclamation code in ncgen; adding the use of nc_reclaim exposed some memory problems.
105 lines
4.0 KiB
C
105 lines
4.0 KiB
C
/*********************************************************************
|
|
* Copyright 2018, UCAR/Unidata
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
* $Id$
|
|
* $Header$
|
|
*********************************************************************/
|
|
|
|
/*
|
|
* In order to use any of the netcdf_XXX.h files, it is necessary
|
|
* to include netcdf.h followed by any netcdf_XXX.h files.
|
|
* Various things (like EXTERNL) are defined in netcdf.h
|
|
* to make them available for use by the netcdf_XXX.h files.
|
|
*/
|
|
|
|
#ifndef NCAUX_H
|
|
#define NCAUX_H
|
|
|
|
#define NCAUX_ALIGN_C 0
|
|
#define NCAUX_ALIGN_UNIFORM 1
|
|
|
|
#if defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
|
|
/**
|
|
Reclaim a vector of instances of arbitrary type.
|
|
Intended for use with e.g. nc_get_vara or the input to e.g. nc_put_vara.
|
|
This recursively walks the top-level instances to
|
|
reclaim any nested data such as vlen or strings or such.
|
|
|
|
Assumes it is passed a pointer to count instances of xtype.
|
|
Reclaims any nested data.
|
|
WARNING: ncaux_reclaim_data does not reclaim the top-level memory
|
|
because we do not know how it was allocated.
|
|
However ncaux_reclaim_data_all does reclaim top-level memory.
|
|
|
|
WARNING: all data blocks below the top-level (e.g. string instances)
|
|
will be reclaimed, so do not call if there is any static data in the instance.
|
|
|
|
Should work for any netcdf format.
|
|
|
|
WARNING: deprecated in favor the corresponding function in netcdf.h.
|
|
These are just wrappers for nc_reclaim_data and
|
|
nc_reclaim_data_all and nc_copy_data and nc_copy_data_all and
|
|
are here for back compatibilty.
|
|
*/
|
|
|
|
EXTERNL int ncaux_reclaim_data(int ncid, int xtype, void* memory, size_t count);
|
|
EXTERNL int ncaux_reclaim_data_all(int ncid, int xtype, void* memory, size_t count);
|
|
EXTERNL int ncaux_copy_data(int ncid, int xtype, void* memory, size_t count, void* copy);
|
|
EXTERNL int ncaux_copy_data_all(int ncid, int xtype, void* memory, size_t count, void** copyp);
|
|
|
|
EXTERNL int ncaux_dump_data(int ncid, nc_type xtypeid, void* memory, size_t count, char** buf);
|
|
|
|
|
|
EXTERNL int ncaux_inq_any_type(int ncid, nc_type typeid, char *name, size_t *size, nc_type *basetypep, size_t *nfieldsp, int *classp);
|
|
|
|
/**************************************************/
|
|
/* Capture the id and parameters for a filter
|
|
using the HDF5 unsigned int format
|
|
*/
|
|
typedef struct NC_H5_Filterspec {
|
|
unsigned int filterid; /**< ID for arbitrary filter. */
|
|
size_t nparams; /**< nparams for arbitrary filter. */
|
|
unsigned int* params; /**< Params for arbitrary filter. */
|
|
} NC_H5_Filterspec;
|
|
|
|
EXTERNL int ncaux_h5filterspec_parse(const char* txt, unsigned int* idp, size_t* nparamsp, unsigned int** paramsp);
|
|
EXTERNL int ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, struct NC_H5_Filterspec*** vectorp);
|
|
EXTERNL int ncaux_h5filterspec_parse_parameter(const char* txt, size_t* nuiparamsp, unsigned int* uiparams);
|
|
EXTERNL void ncaux_h5filterspec_free(struct NC_H5_Filterspec* f);
|
|
EXTERNL void ncaux_h5filterspec_fix8(unsigned char* mem, int decode);
|
|
|
|
/**************************************************/
|
|
/* Wrappers to export selected functions from libnetcdf */
|
|
|
|
EXTERNL int ncaux_readfile(const char* filename, size_t* sizep, void** content);
|
|
EXTERNL int ncaux_writefile(const char* filename, size_t size, void* content);
|
|
|
|
/**************************************************/
|
|
|
|
/* Takes any type */
|
|
EXTERNL int ncaux_type_alignment(int xtype, int ncid, size_t* alignp);
|
|
EXTERNL int ncaux_class_alignment(int ncclass, size_t* alignp);
|
|
|
|
/**************************************************/
|
|
/* Takes type classes only */
|
|
|
|
/* Build compound types and properly handle offset and alignment */
|
|
|
|
EXTERNL int ncaux_class_alignment(int ncclass, size_t* alignp);
|
|
EXTERNL int ncaux_begin_compound(int ncid, const char *name, int alignmode, void** tag);
|
|
EXTERNL int ncaux_end_compound(void* tag, nc_type* xtypeid);
|
|
EXTERNL int ncaux_abort_compound(void* tag);
|
|
EXTERNL int ncaux_add_field(void* tag, const char *name, nc_type field_type,
|
|
int ndims, const int* dimsizes);
|
|
|
|
#if defined(__cplusplus)
|
|
}
|
|
#endif
|
|
|
|
#endif /*NCAUX_H*/
|
|
|