netcdf-c/libnczarr/zclose.c
Dennis Heimbigner 2afbdbd18f Add support for the XArray Zarr _ARRAY_DIMENSIONS attribute
The XArray implementation that uses Zarr for storage
provides a mechanism to simulate named dimensions.
It does this by adding a per-variable attribute called
_ARRAY_DIMENSIONS. This attribute contains a list of names
to be matched against the shape values of the variable.
In effect a named dimension is created with the name
_ARRAY_DIMENSIONS(i) and length shape(i) for all i
in range 0..rank(variable).
Both read and write support is provided.

This XArray support is only invoked if the mode value
of "xarray" is defined. So for example, as in this URL.
````
https://s3.us-west-1.amazonaws.com/bucket/dataset#mode=nczarr,xarray,s3
````
Note that the "xarray" mode flag also implies mode flag "zarr", so the above
is equivalent to this URL.
````
https://s3.us-west-1.amazonaws.com/bucket/dataset#mode=nczarr,zarr,xarray,s3
````

The primary change to implement this was to unify the handling
of dimension references in libnczarr/zsync.

A test for this and other pure-zarr features was added as
nczarr_test/run_purezarr.sh

Other changes:
* Make sure distcheck leaves no files around.
* Change the special attribute flag DIMSCALEFLAG to HIDDENATTRFLAG
  to support the xarray attribute.
* Annotate the zmap implementations with feature flags such as
  WRITEONCE (for zip files).
2021-02-24 13:46:11 -07:00

296 lines
7.5 KiB
C

/*********************************************************************
* Copyright 1993, UCAR/Unidata
* See netcdf/COPYRIGHT file for copying and redistribution conditions.
*********************************************************************/
#include "zincludes.h"
/* Forward */
static int zclose_group(NC_GRP_INFO_T*);
static int zclose_gatts(NC_GRP_INFO_T*);
static int zclose_vars(NC_GRP_INFO_T*);
static int zclose_dims(NC_GRP_INFO_T*);
static int zclose_types(NC_GRP_INFO_T*);
static int zclose_type(NC_TYPE_INFO_T* type);
static int zwrite_vars(NC_GRP_INFO_T *grp);
/**************************************************/
/**
* @internal This function will recurse through an open ZARR file and
* release resources. All ZARR annotations reclaimed
*
* @param file Pointer to ZARR file info struct.
* @param abort True if this is an abort.
*
* @return ::NC_NOERR No error.
* @return ::NC_ENCZARR could not close the file.
* @author Dennis Heimbigner
*/
int
ncz_close_file(NC_FILE_INFO_T* file, int abort)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
ZTRACE(2,"file=%s abort=%d",file->hdr.name,abort);
if(!abort) {
/* Flush | create all chunks for all vars */
if((stat=zwrite_vars(file->root_grp))) goto done;
}
/* Internal close to reclaim zarr annotations */
if((stat = zclose_group(file->root_grp)))
goto done;
zinfo = file->format_file_info;
if((stat = nczmap_close(zinfo->map,(abort && zinfo->created)?1:0)))
goto done;
NCZ_freestringvec(0,zinfo->envv_controls);
NC_authfree(zinfo->auth);
nullfree(zinfo);
done:
return ZUNTRACE(stat);
}
/**************************************************/
/**
* @internal Recursively free zarr annotations for a group (and everything
* it contains).
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_group(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
NCZ_GRP_INFO_T* zgrp;
int i;
assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
/* Recursively call this function for each child, if any, stopping
* if there is an error. */
for(i=0; i<ncindexsize(grp->children); i++) {
if ((stat = zclose_group((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
goto done;
}
/* Close resources associated with global attributes. */
if ((stat = zclose_gatts(grp)))
goto done;
/* Close resources associated with vars. */
if ((stat = zclose_vars(grp)))
goto done;
/* Close resources associated with dims. */
if ((stat = zclose_dims(grp)))
goto done;
/* Close resources associated with types. */
if ((stat = zclose_types(grp)))
goto done;
/* Close the zgroup. */
zgrp = grp->format_grp_info;
LOG((4, "%s: closing group %s", __func__, grp->hdr.name));
nullfree(zgrp);
grp->format_grp_info = NULL; /* avoid memory errors */
done:
return stat;
}
/**
* @internal Close resources for global atts in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_gatts(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_ATT_INFO_T *att;
int a;
for(a = 0; a < ncindexsize(grp->att); a++) {
NCZ_ATT_INFO_T* zatt = NULL;
att = (NC_ATT_INFO_T* )ncindexith(grp->att, a);
assert(att && att->format_att_info != NULL);
zatt = att->format_att_info;
nullfree(zatt);
att->format_att_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for vars in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_vars(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_VAR_INFO_T* var;
NCZ_VAR_INFO_T* zvar;
NC_ATT_INFO_T* att;
int a, i;
for(i = 0; i < ncindexsize(grp->vars); i++) {
var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
assert(var && var->format_var_info);
zvar = var->format_var_info;;
for(a = 0; a < ncindexsize(var->att); a++) {
NCZ_ATT_INFO_T* zatt;
att = (NC_ATT_INFO_T*)ncindexith(var->att, a);
assert(att && att->format_att_info);
zatt = att->format_att_info;
nullfree(zatt);
att->format_att_info = NULL; /* avoid memory errors */
}
/* Reclaim filters */
if(var->filters != NULL) {
(void)NCZ_filter_freelist(var);
}
var->filters = NULL;
/* Reclaim the type */
(void)zclose_type(var->type_info);
NCZ_free_chunk_cache(zvar->cache);
/* reclaim xarray */
nclistfreeall(zvar->xarray);
nullfree(zvar);
var->format_var_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for dims in a group.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_dims(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
NC_DIM_INFO_T* dim;
int i;
for(i = 0; i < ncindexsize(grp->dim); i++) {
NCZ_DIM_INFO_T* zdim;
dim = (NC_DIM_INFO_T*)ncindexith(grp->dim, i);
assert(dim && dim->format_dim_info);
zdim = dim->format_dim_info;
nullfree(zdim);
dim->format_dim_info = NULL; /* avoid memory errors */
}
return stat;
}
/**
* @internal Close resources for a single type. Set values to
* 0 after closing types. Because of type reference counters, these
* closes can be called multiple times.
*
* @param type Pointer to type struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_type(NC_TYPE_INFO_T* type)
{
int stat = NC_NOERR;
NCZ_TYPE_INFO_T* ztype;
assert(type && type->format_type_info != NULL);
/* Get Zarr-specific type info. */
ztype = type->format_type_info;
nullfree(ztype);
type->format_type_info = NULL; /* avoid memory errors */
return stat;
}
/**
* @internal Close resources for types in a group. Set values to
* 0 after closing types. Because of type reference counters, these
* closes can be called multiple times.
* Warning: note that atomic types are not covered here; this
* is only for user-defined types.
*
* @param grp Pointer to group info struct.
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zclose_types(NC_GRP_INFO_T* grp)
{
int stat = NC_NOERR;
int i;
NC_TYPE_INFO_T* type;
for(i = 0; i < ncindexsize(grp->type); i++)
{
type = (NC_TYPE_INFO_T*)ncindexith(grp->type, i);
if((stat = zclose_type(type))) goto done;
}
done:
return stat;
}
/**
* @internal Recursively flush/create all data for all vars.
*
* @param grp Pointer to group info struct whose vars need to be written
*
* @return ::NC_NOERR No error.
* @author Dennis Heimbigner
*/
static int
zwrite_vars(NC_GRP_INFO_T *grp)
{
int stat = NC_NOERR;
int i;
assert(grp && grp->format_grp_info != NULL);
LOG((3, "%s: grp->name %s", __func__, grp->hdr.name));
/* Write all vars for this group breadth first */
for(i = 0; i < ncindexsize(grp->vars); i++) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
if((stat = ncz_write_var(var))) goto done;
}
/* Recursively call this function for each child group, if any, stopping
* if there is an error. */
for(i=0; i<ncindexsize(grp->children); i++) {
if ((stat = zwrite_vars((NC_GRP_INFO_T*)ncindexith(grp->children,i))))
goto done;
}
done:
return stat;
}