mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-23 16:59:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2066 from DennisHeimbigner/zarrfixes.dmh

Browse Source 
Fix a number of bugs in the nczarr code.
This commit is contained in:
Ward Fisher 2021-08-12 16:06:06 -06:00 committed by GitHub
commit 1279681d36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 293 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
RELEASE_NOTES.md
View File

@ -7,11 +7,12 @@ This file contains a high-level description of this package's evolution. Release
## 4.8.1 - TBD
* [Enhancement] Support windows network paths (e.g. \\svc\...). See [Github #2065](https://github.com/Unidata/netcdf-c/issues/2065).
* [Enhancement] Convert to a new representation of the NCZarr meta-data extensions: version 2. Read-only backward compatibility is provided. See [Github #2032](https://github.com/Unidata/netcdf-c/issues/2032).
* [Bug Fix] Fix dimension_separator bug in libnczarr. See [Github #2035](https://github.com/Unidata/netcdf-c/issues/2035).
* [Bug Fix] Fix bugs in libdap4. See [Github #2005](https://github.com/Unidata/netcdf-c/issues/2005).
* [Bug Fix] Store NCZarr fillvalue as a singleton instead of a 1-element array. See [Github #2017](https://github.com/Unidata/netcdf-c/issues/2017).
* [Bug Fix] Fix multiple bugs in libnczarr. See [Github #2066](https://github.com/Unidata/netcdf-c/pull/2066).
* [Enhancement] Support windows network paths (e.g. \\svc\...). See [Github #2065](https://github.com/Unidata/netcdf-c/pull/2065).
* [Enhancement] Convert to a new representation of the NCZarr meta-data extensions: version 2. Read-only backward compatibility is provided. See [Github #2032](https://github.com/Unidata/netcdf-c/pull/2032).
* [Bug Fix] Fix dimension_separator bug in libnczarr. See [Github #2035](https://github.com/Unidata/netcdf-c/pull/2035).
* [Bug Fix] Fix bugs in libdap4. See [Github #2005](https://github.com/Unidata/netcdf-c/pull/2005).
* [Bug Fix] Store NCZarr fillvalue as a singleton instead of a 1-element array. See [Github #2017](https://github.com/Unidata/netcdf-c/pull/2017).
* [Bug Fixes] The netcdf-c library was incorrectly determining the scope of dimension; similar to the type scope problem. See [Github #2012](https://github.com/Unidata/netcdf-c/pull/2012) for more information.
* [Bug Fix] Re-enable DAP2 authorization testing. See [Github #2011](https://github.com/Unidata/netcdf-c/issues/2011).
* [Bug Fix] Fix bug with windows version of mkstemp that causes failure to create more than 26 temp files. See [Github #1998](https://github.com/Unidata/netcdf-c/pull/1998).

4
libnczarr/zs3sdk.cpp
View File

@ -78,7 +78,9 @@ NCZ_s3sdkcreateconfig(const char* host, const char* region, void** configp)
if(region) config->region = region;
if(host) config->endpointOverride = host;
config->enableEndpointDiscovery = true;
config->followRedirects = true;
#if 0
config->followRedirects = Aws::Client::FollowRedirectsPolicy::ALWAYS;
#endif
if(configp) * configp = config;
return ZUNTRACE(stat);
}

106
libnczarr/zsync.c
View File

@ -116,6 +116,7 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
int i,stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = NULL;
char version[1024];
int purezarr = 0;
NCZMAP* map = NULL;
char* fullpath = NULL;
char* key = NULL;
@ -132,47 +133,50 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
zinfo = file->format_file_info;
map = zinfo->map;
purezarr = (zinfo->controls.flags & FLAG_PUREZARR)?1:0;
/* Construct grp key */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
/* Create dimensions dict */
if((stat = ncz_collect_dims(file,grp,&jdims))) goto done;
if(!purezarr) {
/* Create dimensions dict */
if((stat = ncz_collect_dims(file,grp,&jdims))) goto done;
/* Create vars list */
if((stat = NCJnew(NCJ_ARRAY,&jvars)))
goto done;
for(i=0; i<ncindexsize(grp->vars); i++) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
if((stat = NCJaddstring(jvars,NCJ_STRING,var->hdr.name))) goto done;
/* Create vars list */
if((stat = NCJnew(NCJ_ARRAY,&jvars)))
goto done;
for(i=0; i<ncindexsize(grp->vars); i++) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
if((stat = NCJaddstring(jvars,NCJ_STRING,var->hdr.name))) goto done;
}
/* Create subgroups list */
if((stat = NCJnew(NCJ_ARRAY,&jsubgrps)))
goto done;
for(i=0; i<ncindexsize(grp->children); i++) {
NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
if((stat = NCJaddstring(jsubgrps,NCJ_STRING,g->hdr.name))) goto done;
}
/* Create the "_NCZARR_GROUP" dict */
if((stat = NCJnew(NCJ_DICT,&json)))
goto done;
/* Insert the various dicts and arrays */
if((stat = NCJinsert(json,"dims",jdims))) goto done;
jdims = NULL; /* avoid memory problems */
if((stat = NCJinsert(json,"vars",jvars))) goto done;
jvars = NULL; /* avoid memory problems */
if((stat = NCJinsert(json,"groups",jsubgrps))) goto done;
jsubgrps = NULL; /* avoid memory problems */
}
/* Create subgroups list */
if((stat = NCJnew(NCJ_ARRAY,&jsubgrps)))
goto done;
for(i=0; i<ncindexsize(grp->children); i++) {
NC_GRP_INFO_T* g = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
if((stat = NCJaddstring(jsubgrps,NCJ_STRING,g->hdr.name))) goto done;
}
/* Create the "_NCZARR_GROUP" dict */
if((stat = NCJnew(NCJ_DICT,&json)))
goto done;
/* Insert the various dicts and arrays */
if((stat = NCJinsert(json,"dims",jdims))) goto done;
jdims = NULL; /* avoid memory problems */
if((stat = NCJinsert(json,"vars",jvars))) goto done;
jvars = NULL; /* avoid memory problems */
if((stat = NCJinsert(json,"groups",jsubgrps))) goto done;
jsubgrps = NULL; /* avoid memory problems */
/* build ZGROUP contents */
if((stat = NCJnew(NCJ_DICT,&jgroup)))
goto done;
snprintf(version,sizeof(version),"%d",zinfo->zarr.zarr_version);
if((stat = NCJaddstring(jgroup,NCJ_STRING,"zarr_format"))) goto done;
if((stat = NCJaddstring(jgroup,NCJ_INT,version))) goto done;
if(grp->parent == NULL) { /* Root group */
if(!purezarr && grp->parent == NULL) { /* Root group */
snprintf(version,sizeof(version),"%lu.%lu.%lu",
zinfo->zarr.nczarr_version.major,
zinfo->zarr.nczarr_version.minor,
@ -185,9 +189,11 @@ ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
jsuper = NULL;
}
/* Insert the "_NCZARR_GROUP" dict */
if((stat = NCJinsert(jgroup,NCZ_V2_GROUP,json))) goto done;
json = NULL;
if(!purezarr) {
/* Insert the "_NCZARR_GROUP" dict */
if((stat = NCJinsert(jgroup,NCZ_V2_GROUP,json))) goto done;
json = NULL;
}
/* build ZGROUP path */
if((stat = nczm_concat(fullpath,ZGROUP,&key)))
@ -329,7 +335,9 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
jtmp = NULL;
/* fill_value key */
if(!var->no_fill) {
if(var->no_fill) {
if((stat=NCJnew(NCJ_NULL,&jfill))) goto done;
} else {/*!var->no_fill*/
int fillsort;
int atomictype = var->type_info->hdr.id;
/* A scalar value providing the default value to use for uninitialized
@ -354,9 +362,9 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var)
jfill = jtmp;
jtmp = NULL;
}
if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done;
jfill = NULL;
}
if((stat = NCJinsert(jvar,"fill_value",jfill))) goto done;
jfill = NULL;
/* order key */
if((stat = NCJaddstring(jvar,NCJ_STRING,"order"))) goto done;
@ -577,6 +585,7 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
char* content = NULL;
char* dimpath = NULL;
int isxarray = 0;
int isrootgroup = 0;
LOG((3, "%s", __func__));
@ -585,6 +594,12 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
if(zinfo->controls.flags & FLAG_XARRAYDIMS) isxarray = 1;
if(container->sort == NCVAR) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
if(var->container && var->container->parent == NULL)
isrootgroup = 1;
}
if(!isxarray && ncindexsize(attlist) == 0)
goto done; /* do nothing */
@ -611,7 +626,7 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
}
/* Construct container path */
if(NCJsort(container) == NCGRP)
if(container->sort == NCGRP)
stat = NCZ_grpkey((NC_GRP_INFO_T*)container,&fullpath);
else
stat = NCZ_varkey((NC_VAR_INFO_T*)container,&fullpath);
@ -622,8 +637,8 @@ ncz_sync_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCindex* attlist)
if((stat = ncz_jsonize_atts(attlist,&jatts)))
goto done;
if(NCJsort(container) == NCVAR) {
if(isxarray) {
if(container->sort == NCVAR) {
if(isrootgroup && isxarray) {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Insert the XARRAY _ARRAY_ATTRIBUTE attribute */
if((stat = NCJnew(NCJ_ARRAY,&jdimrefs)))
@ -740,7 +755,7 @@ load_jatts(NCZMAP* map, NC_OBJ* container, int nczarrv1, NCjson** jattrsp, NClis
/* alway return (possibly empty) list of types */
atypes = nclistnew();
if(NCJsort(container) == NCGRP) {
if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
@ -1184,7 +1199,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
zinfo = file->format_file_info;
map = zinfo->map;
if(NCJsort(container) == NCGRP)
if(container->sort == NCGRP)
attlist = ((NC_GRP_INFO_T*)container)->att;
else
attlist = ((NC_VAR_INFO_T*)container)->att;
@ -1210,7 +1225,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
if(ra != NULL) {
/* case 1: name = _NCProperties, grp=root, varid==NC_GLOBAL, flags & READONLYFLAG */
if(strcmp(NCJstring(key),NCPROPS)==0
&& NCJsort(container) == NCGRP
&& container->sort == NCGRP
&& file->root_grp == (NC_GRP_INFO_T*)container) {
/* Setup provenance */
if(NCJsort(value) != NCJ_STRING)
@ -1220,7 +1235,7 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
}
/* case 2: name = _ARRAY_DIMENSIONS, sort==NCVAR, flags & HIDDENATTRFLAG */
if(strcmp(NCJstring(key),NC_XARRAY_DIMS)==0
&& NCJsort(container) == NCVAR
&& container->sort == NCVAR
&& (ra->flags & HIDDENATTRFLAG)) {
/* store for later */
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)((NC_VAR_INFO_T*)container)->format_var_info;
@ -1249,13 +1264,13 @@ ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container)
}
}
/* If we have not read a _FillValue, then go ahead and create it */
if(fillvalueatt == NULL && NCJsort(container) == NCVAR) {
if(fillvalueatt == NULL && container->sort == NCVAR) {
if((stat = ncz_create_fillvalue((NC_VAR_INFO_T*)container)))
goto done;
}
/* Remember that we have read the atts for this var or group. */
if(NCJsort(container) == NCVAR)
if(container->sort == NCVAR)
((NC_VAR_INFO_T*)container)->atts_read = 1;
else
((NC_GRP_INFO_T*)container)->atts_read = 1;
@ -1399,6 +1414,8 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
if((stat = ncz_gettype(file,grp,vtype,&var->type_info)))
goto done;
} else {stat = NC_EBADTYPE; goto done;}
if(endianness == NC_ENDIAN_NATIVE)
endianness = zinfo->native_endianness;
if(endianness == NC_ENDIAN_LITTLE || endianness == NC_ENDIAN_BIG) {
var->endianness = endianness;
} else {stat = NC_EBADTYPE; goto done;}
@ -1466,7 +1483,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* fill_value */
{
if((stat = NCJdictget(jvar,"fill_value",&jvalue))) goto done;
if(jvalue == NULL)
if(jvalue == NULL || NCJsort(jvalue) == NCJ_NULL)
var->no_fill = 1;
else {
size_t fvlen;
@ -1511,6 +1528,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* Extract the NCZ_V2_ARRAY dict */
if((stat = NCJdictget(jvar,NCZ_V2_ARRAY,&jncvar))) goto done;
}
if(jncvar == NULL) {stat = NC_ENCZARR; goto done;}
assert((NCJsort(jncvar) == NCJ_DICT));
/* Extract storage flag */
if((stat = NCJdictget(jncvar,"storage",&jvalue)))

1
libnczarr/zutil.c
View File

@ -514,6 +514,7 @@ ncz_dtype2typeinfo(const char* dtype, nc_type* nctypep, int* endianp)
switch (dtype[0]) {
case '<': endianness = NC_ENDIAN_LITTLE; break;
case '>': endianness = NC_ENDIAN_BIG; break;
case '|': endianness = NC_ENDIAN_NATIVE; break;
default: goto zerr;
}
/* Decode the type length */

3
nczarr_test/CMakeLists.txt
View File

@ -78,7 +78,7 @@ IF(ENABLE_TESTS)
add_sh_test(nczarr_test run_ut_misc)
add_sh_test(nczarr_test run_ut_chunk)
IF(USE_HDF5)
add_sh_test(nczarr_test run_nccopyz)
# add_sh_test(nczarr_test run_nccopyz)
add_sh_test(nczarr_test run_fillonlyz)
ENDIF()
add_sh_test(nczarr_test run_ncgen4)
@ -90,6 +90,7 @@ IF(ENABLE_TESTS)
add_sh_test(nczarr_test run_purezarr)
add_sh_test(nczarr_test run_interop)
add_sh_test(nczarr_test run_misc)
add_sh_test(nczarr_test run_nczarr_fill)
if(ENABLE_NCZARR_S3)
add_sh_test(nczarr_test run_s3_cleanup)

9
nczarr_test/Makefile.am
View File

@ -47,7 +47,7 @@ if BUILD_UTILITIES
TESTS += run_ncgen4.sh
if USE_HDF5
TESTS += run_nccopyz.sh
#TESTS += run_nccopyz.sh
TESTS += run_fillonlyz.sh
endif
@ -58,6 +58,7 @@ TESTS += run_chunkcases.sh
TESTS += run_purezarr.sh
TESTS += run_interop.sh
TESTS += run_misc.sh
TESTS += run_nczarr_fill.sh
endif
@ -108,7 +109,7 @@ ncdumpchunks_SOURCES = ncdumpchunks.c
EXTRA_DIST = CMakeLists.txt \
run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \
run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \
run_purezarr.sh run_interop.sh run_misc.sh run_newformat.sh
run_purezarr.sh run_interop.sh run_misc.sh run_newformat.sh run_nczarr_fill.sh
EXTRA_DIST += \
ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \
@ -125,7 +126,9 @@ ref_misc1.cdl ref_misc1.dmp ref_misc2.cdl \
ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \
ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl \
ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes.zip ref_quotes.cdl \
ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl
ref_oldformat.cdl ref_oldformat.zip ref_newformatpure.cdl \
ref_groups.h5 ref_byte.zarr.zip ref_byte_fill_value_null.zarr.zip \
ref_groups_regular.cdl ref_byte.cdl ref_byte_fill_value_null.cdl
CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc

34
nczarr_test/ref_byte.cdl Normal file
View File

@ -0,0 +1,34 @@
netcdf ref_byte {
dimensions:
.zdim_20 = 20 ;
variables:
ubyte byte(.zdim_20, .zdim_20) ;
byte:_Storage = "chunked" ;
byte:_ChunkSizes = 20, 20 ;
// global attributes:
:_Format = "netCDF-4" ;
data:
byte =
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ;
}

BIN
nczarr_test/ref_byte.zarr.zip Normal file
View File

Binary file not shown.

55
nczarr_test/ref_byte_fill_value_null.cdl Normal file
View File

@ -0,0 +1,55 @@
netcdf ref_byte_fill_value_null {
dimensions:
.zdim_20 = 20 ;
variables:
ubyte byte(.zdim_20, .zdim_20) ;
byte:_Storage = "chunked" ;
byte:_ChunkSizes = 20, 20 ;
byte:_NoFill = "true" ;
// global attributes:
:_Format = "netCDF-4" ;
data:
byte =
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255 ;
}

BIN
nczarr_test/ref_byte_fill_value_null.zarr.zip Executable file
View File

Binary file not shown.

BIN
nczarr_test/ref_groups.h5 Executable file
View File

Binary file not shown.

45
nczarr_test/ref_groups_regular.cdl Normal file
View File

@ -0,0 +1,45 @@
netcdf tmp_groups_regular {
dimensions:
.zdim_3 = 3 ;
.zdim_2 = 2 ;
.zdim_10 = 10 ;
// global attributes:
:_Format = "netCDF-4" ;
group: MyGroup {
variables:
int dset1(.zdim_3, .zdim_3) ;
dset1:_Storage = "chunked" ;
dset1:_ChunkSizes = 3, 3 ;
dset1:_NoFill = "true" ;
// group attributes:
data:
dset1 =
1, 2, 3,
1, 2, 3,
1, 2, 3 ;
group: Group_A {
variables:
int dset2(.zdim_2, .zdim_10) ;
dset2:_Storage = "chunked" ;
dset2:_ChunkSizes = 2, 10 ;
dset2:_NoFill = "true" ;
// group attributes:
data:
dset2 =
1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ;
} // group Group_A
group: Group_B {
// group attributes:
} // group Group_B
} // group MyGroup
}

14
nczarr_test/run_misc.sh
View File

@ -54,11 +54,15 @@ diff -wb ${abs_srcdir}/ref_misc2.cdl tmp_misc2_$zext.cdl
}
testcase1 file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase1 zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase1 s3; fi
testcase2 file
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase2 zip; fi
if test "x$FEATURE_S3TESTS" = xyes ; then testcase2 s3; fi
exit 0
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then
testcase1 zip
testcase2 zip
fi
if test "x$FEATURE_S3TESTS" = xyes ; then
testcase1 s3
testcase2 s3
fi
exit 0

59
nczarr_test/run_nczarr_fill.sh Executable file
View File

@ -0,0 +1,59 @@
#!/bin/sh
if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh
. "$srcdir/test_nczarr.sh"
set -e
echo "*** Test: Github issues #2063, #2062, #2059"
testcase2059() {
zext=$1
echo "*** Test: Github issue #2059"
fileargs tmp_groups_regular "mode=zarr,$zext"
deletemap $zext $file
${NCCOPY} ${srcdir}/ref_groups.h5 "$fileurl"
rm -f tmp.cdl
${NCDUMP} -s -n tmp_groups_regular "$fileurl" > tmp.cdl
sclean tmp.cdl tmp_groups_regular_$zext.cdl
diff -wb ${srcdir}/ref_groups_regular.cdl tmp_groups_regular_$zext.cdl
}
testcase2062() {
zext=$1
echo "*** Test: Github issue #2062"
rm -fr ref_byte.zarr
unzip ${srcdir}/ref_byte.zarr.zip
rm -fr tmp.cdl
${NCDUMP} -s "file://ref_byte.zarr#mode=zarr,$zext" > tmp.cdl
sclean tmp.cdl tmp_byte_$zext.cdl
diff -wb ${srcdir}/ref_byte.cdl tmp_byte_$zext.cdl
rm -fr ref_byte.zarr
}
testcase2063() {
zext=$1
echo "*** Test: Github issue #2063"
rm -fr ref_byte_fill_value_null.zarr
unzip ${srcdir}/ref_byte_fill_value_null.zarr.zip
rm -fr tmp.cdl
${NCDUMP} -s "file://ref_byte_fill_value_null.zarr#mode=zarr,$zext" > tmp.cdl
sclean tmp.cdl tmp_byte_fill_value_null_$zext.cdl
diff -wb ${srcdir}/ref_byte_fill_value_null.cdl tmp_byte_fill_value_null_$zext.cdl
rm -fr ref_byte_fill_value_null.zarr
}
#testcase2059 file
#testcase2062 file
testcase2063 file
exit 0
if test "x$FEATURE_NCZARR_ZIP" = xyes ; then
testcase2059 zip
fi
if test "x$FEATURE_S3TESTS" = xyes ; then
testcase2059 s3
fi
exit 0

11
nczarr_test/test_nczarr.sh
View File

@ -126,6 +126,17 @@ for t in ${TESTS} ; do
done
}
# Function to remove selected -s attributes from file;
# These attributes might be platform dependent
sclean() {
cat $1 \
| sed -e '/:_IsNetcdf4/d' \
| sed -e '/:_Endianness/d' \
| sed -e '/_NCProperties/d' \
| sed -e '/_SuperblockVersion/d' \
| cat > $2
}
# Make sure execdir and srcdir absolute paths are available
WD=`pwd`
cd $srcdir ; abs_srcdir=`pwd` ; cd $WD