mirror of
https://github.com/Unidata/netcdf-c.git
synced 2024-12-27 08:49:16 +08:00
2afbdbd18f
The XArray implementation that uses Zarr for storage provides a mechanism to simulate named dimensions. It does this by adding a per-variable attribute called _ARRAY_DIMENSIONS. This attribute contains a list of names to be matched against the shape values of the variable. In effect a named dimension is created with the name _ARRAY_DIMENSIONS(i) and length shape(i) for all i in range 0..rank(variable). Both read and write support is provided. This XArray support is only invoked if the mode value of "xarray" is defined. So for example, as in this URL. ```` https://s3.us-west-1.amazonaws.com/bucket/dataset#mode=nczarr,xarray,s3 ```` Note that the "xarray" mode flag also implies mode flag "zarr", so the above is equivalent to this URL. ```` https://s3.us-west-1.amazonaws.com/bucket/dataset#mode=nczarr,zarr,xarray,s3 ```` The primary change to implement this was to unify the handling of dimension references in libnczarr/zsync. A test for this and other pure-zarr features was added as nczarr_test/run_purezarr.sh Other changes: * Make sure distcheck leaves no files around. * Change the special attribute flag DIMSCALEFLAG to HIDDENATTRFLAG to support the xarray attribute. * Annotate the zmap implementations with feature flags such as WRITEONCE (for zip files).
296 lines
7.5 KiB
C
296 lines
7.5 KiB
C
/*********************************************************************
|
|
* Copyright 1993, UCAR/Unidata
|
|
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
|
|
*********************************************************************/
|
|
|
|
#include "zincludes.h"
|
|
|
|
/* Forward */
|
|
static int zclose_group(NC_GRP_INFO_T*);
|
|
static int zclose_gatts(NC_GRP_INFO_T*);
|
|
static int zclose_vars(NC_GRP_INFO_T*);
|
|
static int zclose_dims(NC_GRP_INFO_T*);
|
|
static int zclose_types(NC_GRP_INFO_T*);
|
|
static int zclose_type(NC_TYPE_INFO_T* type);
|
|
static int zwrite_vars(NC_GRP_INFO_T *grp);
|
|
|
|
/**************************************************/
|
|
|
|
/**
|
|
* @internal This function will recurse through an open ZARR file and
|
|
* release resources. All ZARR annotations reclaimed
|
|
*
|
|
* @param file Pointer to ZARR file info struct.
|
|
* @param abort True if this is an abort.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @return ::NC_ENCZARR could not close the file.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
int
|
|
ncz_close_file(NC_FILE_INFO_T* file, int abort)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NCZ_FILE_INFO_T* zinfo = NULL;
|
|
|
|
ZTRACE(2,"file=%s abort=%d",file->hdr.name,abort);
|
|
|
|
if(!abort) {
|
|
/* Flush | create all chunks for all vars */
|
|
if((stat=zwrite_vars(file->root_grp))) goto done;
|
|
}
|
|
|
|
/* Internal close to reclaim zarr annotations */
|
|
if((stat = zclose_group(file->root_grp)))
|
|
goto done;
|
|
|
|
zinfo = file->format_file_info;
|
|
|
|
if((stat = nczmap_close(zinfo->map,(abort && zinfo->created)?1:0)))
|
|
goto done;
|
|
NCZ_freestringvec(0,zinfo->envv_controls);
|
|
NC_authfree(zinfo->auth);
|
|
nullfree(zinfo);
|
|
|
|
done:
|
|
return ZUNTRACE(stat);
|
|
}
|
|
|
|
/**************************************************/
|
|
/**
|
|
* @internal Recursively free zarr annotations for a group (and everything
|
|
* it contains).
|
|
*
|
|
* @param grp Pointer to group info struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_group(NC_GRP_INFO_T *grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NCZ_GRP_INFO_T* zgrp;
|
|
int i;
|
|
|
|
assert(grp && grp->format_grp_info != NULL);
|
|
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
|
|
|
|
/* Recursively call this function for each child, if any, stopping
|
|
* if there is an error. */
|
|
for(i=0; i<ncindexsize(grp->children); i++) {
|
|
if ((stat = zclose_group((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
|
|
goto done;
|
|
}
|
|
|
|
/* Close resources associated with global attributes. */
|
|
if ((stat = zclose_gatts(grp)))
|
|
goto done;
|
|
|
|
/* Close resources associated with vars. */
|
|
if ((stat = zclose_vars(grp)))
|
|
goto done;
|
|
|
|
/* Close resources associated with dims. */
|
|
if ((stat = zclose_dims(grp)))
|
|
goto done;
|
|
|
|
/* Close resources associated with types. */
|
|
if ((stat = zclose_types(grp)))
|
|
goto done;
|
|
|
|
/* Close the zgroup. */
|
|
zgrp = grp->format_grp_info;
|
|
LOG((4, "%s: closing group %s", __func__, grp->hdr.name));
|
|
nullfree(zgrp);
|
|
grp->format_grp_info = NULL; /* avoid memory errors */
|
|
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Close resources for global atts in a group.
|
|
*
|
|
* @param grp Pointer to group info struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_gatts(NC_GRP_INFO_T* grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NC_ATT_INFO_T *att;
|
|
int a;
|
|
for(a = 0; a < ncindexsize(grp->att); a++) {
|
|
NCZ_ATT_INFO_T* zatt = NULL;
|
|
att = (NC_ATT_INFO_T* )ncindexith(grp->att, a);
|
|
assert(att && att->format_att_info != NULL);
|
|
zatt = att->format_att_info;
|
|
nullfree(zatt);
|
|
att->format_att_info = NULL; /* avoid memory errors */
|
|
}
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Close resources for vars in a group.
|
|
*
|
|
* @param grp Pointer to group info struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_vars(NC_GRP_INFO_T* grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NC_VAR_INFO_T* var;
|
|
NCZ_VAR_INFO_T* zvar;
|
|
NC_ATT_INFO_T* att;
|
|
int a, i;
|
|
|
|
for(i = 0; i < ncindexsize(grp->vars); i++) {
|
|
var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
|
|
assert(var && var->format_var_info);
|
|
zvar = var->format_var_info;;
|
|
for(a = 0; a < ncindexsize(var->att); a++) {
|
|
NCZ_ATT_INFO_T* zatt;
|
|
att = (NC_ATT_INFO_T*)ncindexith(var->att, a);
|
|
assert(att && att->format_att_info);
|
|
zatt = att->format_att_info;
|
|
nullfree(zatt);
|
|
att->format_att_info = NULL; /* avoid memory errors */
|
|
}
|
|
/* Reclaim filters */
|
|
if(var->filters != NULL) {
|
|
(void)NCZ_filter_freelist(var);
|
|
}
|
|
var->filters = NULL;
|
|
/* Reclaim the type */
|
|
(void)zclose_type(var->type_info);
|
|
NCZ_free_chunk_cache(zvar->cache);
|
|
/* reclaim xarray */
|
|
nclistfreeall(zvar->xarray);
|
|
nullfree(zvar);
|
|
var->format_var_info = NULL; /* avoid memory errors */
|
|
}
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Close resources for dims in a group.
|
|
*
|
|
* @param grp Pointer to group info struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_dims(NC_GRP_INFO_T* grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NC_DIM_INFO_T* dim;
|
|
int i;
|
|
|
|
for(i = 0; i < ncindexsize(grp->dim); i++) {
|
|
NCZ_DIM_INFO_T* zdim;
|
|
dim = (NC_DIM_INFO_T*)ncindexith(grp->dim, i);
|
|
assert(dim && dim->format_dim_info);
|
|
zdim = dim->format_dim_info;
|
|
nullfree(zdim);
|
|
dim->format_dim_info = NULL; /* avoid memory errors */
|
|
}
|
|
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Close resources for a single type. Set values to
|
|
* 0 after closing types. Because of type reference counters, these
|
|
* closes can be called multiple times.
|
|
*
|
|
* @param type Pointer to type struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_type(NC_TYPE_INFO_T* type)
|
|
{
|
|
int stat = NC_NOERR;
|
|
NCZ_TYPE_INFO_T* ztype;
|
|
|
|
assert(type && type->format_type_info != NULL);
|
|
/* Get Zarr-specific type info. */
|
|
ztype = type->format_type_info;
|
|
nullfree(ztype);
|
|
type->format_type_info = NULL; /* avoid memory errors */
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Close resources for types in a group. Set values to
|
|
* 0 after closing types. Because of type reference counters, these
|
|
* closes can be called multiple times.
|
|
* Warning: note that atomic types are not covered here; this
|
|
* is only for user-defined types.
|
|
*
|
|
* @param grp Pointer to group info struct.
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zclose_types(NC_GRP_INFO_T* grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
int i;
|
|
NC_TYPE_INFO_T* type;
|
|
|
|
for(i = 0; i < ncindexsize(grp->type); i++)
|
|
{
|
|
type = (NC_TYPE_INFO_T*)ncindexith(grp->type, i);
|
|
if((stat = zclose_type(type))) goto done;
|
|
}
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
/**
|
|
* @internal Recursively flush/create all data for all vars.
|
|
*
|
|
* @param grp Pointer to group info struct whose vars need to be written
|
|
*
|
|
* @return ::NC_NOERR No error.
|
|
* @author Dennis Heimbigner
|
|
*/
|
|
static int
|
|
zwrite_vars(NC_GRP_INFO_T *grp)
|
|
{
|
|
int stat = NC_NOERR;
|
|
int i;
|
|
|
|
assert(grp && grp->format_grp_info != NULL);
|
|
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
|
|
|
|
/* Write all vars for this group breadth first */
|
|
for(i = 0; i < ncindexsize(grp->vars); i++) {
|
|
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
|
|
if((stat = ncz_write_var(var))) goto done;
|
|
}
|
|
|
|
/* Recursively call this function for each child group, if any, stopping
|
|
* if there is an error. */
|
|
for(i=0; i<ncindexsize(grp->children); i++) {
|
|
if ((stat = zwrite_vars((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
|
|
goto done;
|
|
}
|
|
|
|
done:
|
|
return stat;
|
|
}
|
|
|
|
|