mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-03-07 17:08:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2574 from DennisHeimbigner/loop.dmh

Browse Source 
Fix infinite loop in file inferencing
This commit is contained in:
Ward Fisher 2023-01-10 15:14:56 -08:00 committed by GitHub
commit 2a6c41caab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 177 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/run_tests_osx.yml vendored
View File

@ -7,7 +7,7 @@
name: Run macOS-based netCDF Tests
on: [pull_request, workflow_dispatch]
on: [pull_request,workflow_dispatch]
jobs:

6
.github/workflows/run_tests_ubuntu.yml vendored
View File

@ -4,7 +4,7 @@
name: Run Ubuntu/Linux netCDF Tests
on: [pull_request, workflow_dispatch]
on: [pull_request,workflow_dispatch]
jobs:
@ -42,7 +42,7 @@ jobs:
wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
tar -jxf hdf-4.2.15.tar.bz2
pushd hdf-4.2.15
./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib
./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-hdf4-xdr
make -j
make install -j
popd
@ -89,7 +89,7 @@ jobs:
wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
tar -jxf hdf-4.2.15.tar.bz2
pushd hdf-4.2.15
CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel
CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel --enable-hdf4-xdr
make -j
make install -j
popd

2
.github/workflows/run_tests_win_cygwin.yml vendored
View File

@ -1,6 +1,6 @@
name: Run Cygwin-based tests
on: [pull_request, workflow_dispatch]
on: [pull_request,workflow_dispatch]
env:
SHELLOPTS: igncr

2
.github/workflows/run_tests_win_mingw.yml vendored
View File

@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests
env:
CPPFLAGS: "-D_BSD_SOURCE"
on: [pull_request, workflow_dispatch]
on: [pull_request,workflow_dispatch]
jobs:

23
libdispatch/dfile.c
View File

@ -1839,19 +1839,16 @@ NC_create(const char *path0, int cmode, size_t initialsz,
TRACE(nc_create);
if(path0 == NULL)
return NC_EINVAL;
{stat = NC_EINVAL; goto done;}
/* Check mode flag for sanity. */
if ((stat = check_create_mode(cmode)))
return stat;
if ((stat = check_create_mode(cmode))) goto done;
/* Initialize the library. The available dispatch tables
* will depend on how netCDF was built
* (with/without netCDF-4, DAP, CDMREMOTE). */
if(!NC_initialized)
{
if ((stat = nc_initialize()))
return stat;
if(!NC_initialized) {
if ((stat = nc_initialize())) goto done;
}
{
@ -1863,10 +1860,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
memset(&model,0,sizeof(model));
newpath = NULL;
if((stat = NC_infermodel(path,&cmode,1,useparallel,NULL,&model,&newpath))) {
nullfree(newpath);
goto done;
}
if((stat = NC_infermodel(path,&cmode,1,useparallel,NULL,&model,&newpath))) goto done;
if(newpath) {
nullfree(path);
path = newpath;
@ -1918,7 +1912,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
dispatcher = NC3_dispatch_table;
break;
default:
return NC_ENOTNC;
{stat = NC_ENOTNC; goto done;}
}
/* Create the NC* instance and insert its dispatcher and model */
@ -1937,6 +1931,7 @@ NC_create(const char *path0, int cmode, size_t initialsz,
}
done:
nullfree(path);
nullfree(newpath);
return stat;
}
@ -1980,12 +1975,12 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp,
TRACE(nc_open);
if(!NC_initialized) {
stat = nc_initialize();
if(stat) return stat;
if(stat) goto done;
}
/* Check inputs. */
if (!path0)
return NC_EINVAL;
{stat = NC_EINVAL; goto done;}
/* Capture the inmemory related flags */
mmap = ((omode & NC_MMAP) == NC_MMAP);

203
libdispatch/dinfermodel.c
View File

@ -143,7 +143,15 @@ static const struct MACRODEF {
{NULL,NULL,{NULL}}
};
/* Mode inferences: if mode contains key, then add the inference and infer again */
/*
Mode inferences: if mode contains key value, then add the inferred value;
Warning: be careful how this list is constructed to avoid infinite inferences.
In order to (mostly) avoid that consequence, any attempt to
infer a value that is already present will be ignored.
This effectively means that the inference graph
must be a DAG and may not have cycles.
You have been warned.
*/
static const struct MODEINFER {
char* key;
char* inference;
@ -151,6 +159,7 @@ static const struct MODEINFER {
{"zarr","nczarr"},
{"xarray","zarr"},
{"noxarray","nczarr"},
{"noxarray","zarr"},
{NULL,NULL}
};
@ -202,6 +211,7 @@ static int processmacros(NClist** fraglistp);
static char* envvlist2string(NClist* pairs, const char*);
static void set_default_mode(int* cmodep);
static int parseonchar(const char* s, int ch, NClist* segments);
static int mergelist(NClist** valuesp);
static int openmagic(struct MagicFile* file);
static int readmagic(struct MagicFile* file, long pos, char* magic);
@ -217,8 +227,9 @@ static int parsepair(const char* pair, char** keyp, char** valuep);
static NClist* parsemode(const char* modeval);
static const char* getmodekey(const NClist* envv);
static int replacemode(NClist* envv, const char* newval);
static int inferone(const char* mode, NClist* newmodes);
static void infernext(NClist* current, NClist* next);
static int negateone(const char* mode, NClist* modes);
static void cleanstringlist(NClist* strs, int caseinsensitive);
/*
If the path looks like a URL, then parse it, reformat it.
@ -416,28 +427,6 @@ envvlist2string(NClist* envv, const char* delim)
return result;
}
/* Convert a list into a comma'd string */
static char*
list2string(NClist* list)
{
int i;
NCbytes* buf = NULL;
char* result = NULL;
if(list == NULL || nclistlength(list)==0) return strdup("");
buf = ncbytesnew();
for(i=0;i<nclistlength(list);i++) {
const char* m = nclistget(list,i);
if(m == NULL || strlen(m) == 0) continue;
if(i > 0) ncbytescat(buf,",");
ncbytescat(buf,m);
}
result = ncbytesextract(buf);
ncbytesfree(buf);
if(result == NULL) result = strdup("");
return result;
}
/* Given a mode= argument, fill in the impl */
static int
processmodearg(const char* arg, NCmodel* model)
@ -504,9 +493,10 @@ processinferences(NClist* fraglenv)
{
int stat = NC_NOERR;
const char* modeval = NULL;
NClist* modes = NULL;
NClist* newmodes = nclistnew();
int i,inferred = 0;
NClist* currentmodes = NULL;
NClist* nextmodes = nclistnew();
int i;
char* newmodeval = NULL;
if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done;
@ -515,22 +505,53 @@ processinferences(NClist* fraglenv)
if((modeval = getmodekey(fraglenv))==NULL) goto done;
/* Get the mode as list */
modes = parsemode(modeval);
currentmodes = parsemode(modeval);
/* Repeatedly walk the mode list until no more new positive inferences */
do {
for(i=0;i<nclistlength(modes);i++) {
const char* mode = nclistget(modes,i);
inferred = inferone(mode,newmodes);
nclistpush(newmodes,strdup(mode)); /* keep key */
if(!inferred) nclistpush(newmodes,strdup(mode));
#ifdef DEBUG
printlist(currentmodes,"processinferences: initial mode list");
#endif
/* Do what amounts to breadth first inferencing down the inference DAG. */
for(;;) {
NClist* tmp = NULL;
/* Compute the next set of inferred modes */
#ifdef DEBUG
printlist(currentmodes,"processinferences: current mode list");
#endif
infernext(currentmodes,nextmodes);
#ifdef DEBUG
printlist(nextmodes,"processinferences: next mode list");
#endif
/* move current modes into list of newmodes */
for(i=0;i<nclistlength(currentmodes);i++) {
nclistpush(newmodes,nclistget(currentmodes,i));
}
} while(inferred);
nclistsetlength(currentmodes,0); /* clear current mode list */
if(nclistlength(nextmodes) == 0) break; /* nothing more to do */
#ifdef DEBUG
printlist(newmodes,"processinferences: new mode list");
#endif
/* Swap current and next */
tmp = currentmodes;
currentmodes = nextmodes;
nextmodes = tmp;
tmp = NULL;
}
/* cleanup any unused elements in currenmodes */
nclistclearall(currentmodes);
/* Ensure no duplicates */
cleanstringlist(newmodes,1);
#ifdef DEBUG
printlist(newmodes,"processinferences: final inferred mode list");
#endif
/* Remove negative inferences */
for(i=0;i<nclistlength(modes);i++) {
const char* mode = nclistget(modes,i);
inferred = negateone(mode,newmodes);
for(i=0;i<nclistlength(newmodes);i++) {
const char* mode = nclistget(newmodes,i);
negateone(mode,newmodes);
}
/* Store new mode value */
@ -541,11 +562,13 @@ processinferences(NClist* fraglenv)
done:
nullfree(newmodeval);
nclistfreeall(modes);
nclistfreeall(newmodes);
nclistfreeall(currentmodes);
nclistfreeall(nextmodes);
return check(stat);
}
static int
negateone(const char* mode, NClist* newmodes)
{
@ -568,23 +591,28 @@ negateone(const char* mode, NClist* newmodes)
return changed;
}
static int
inferone(const char* mode, NClist* newmodes)
static void
infernext(NClist* current, NClist* next)
{
const struct MODEINFER* tests = modeinferences;
int changed = 0;
for(;tests->key;tests++) {
if(strcasecmp(tests->key,mode)==0) {
/* Append the inferred mode; dups removed later */
nclistpush(newmodes,strdup(tests->inference));
changed = 1;
int i;
for(i=0;i<nclistlength(current);i++) {
const struct MODEINFER* tests = NULL;
const char* cur = nclistget(current,i);
for(tests=modeinferences;tests->key;tests++) {
if(strcasecmp(tests->key,cur)==0) {
/* Append the inferred mode unless dup */
if(!nclistmatch(next,tests->inference,1))
nclistpush(next,strdup(tests->inference));
}
}
}
return changed;
}
/*
Given a list of strings, remove nulls and duplicates
*/
static int
mergekey(NClist** valuesp)
mergelist(NClist** valuesp)
{
int i,j;
int stat = NC_NOERR;
@ -686,12 +714,12 @@ cleanfragments(NClist** fraglenvp)
/* collect all unique keys */
collectallkeys(fraglenv,allkeys);
/* Collect all values for same key across all fragments */
/* Collect all values for same key across all fragment pairs */
for(i=0;i<nclistlength(allkeys);i++) {
key = nclistget(allkeys,i);
collectvaluesbykey(fraglenv,key,tmp);
/* merge the key values, remove duplicate */
if((stat=mergekey(&tmp))) goto done;
if((stat=mergelist(&tmp))) goto done;
/* Construct key,value pair and insert into newlist */
key = strdup(key);
nclistpush(newlist,key);
@ -923,7 +951,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
}
} else {/* Not URL */
if(*newpathp) *newpathp = NULL;
if(newpathp) *newpathp = NULL;
}
/* Phase 8: mode inference from mode flags */
@ -1101,6 +1129,71 @@ parsemode(const char* modeval)
return modes;
}
/* Convert a list into a comma'd string */
static char*
list2string(NClist* list)
{
int i;
NCbytes* buf = NULL;
char* result = NULL;
if(list == NULL || nclistlength(list)==0) return strdup("");
buf = ncbytesnew();
for(i=0;i<nclistlength(list);i++) {
const char* m = nclistget(list,i);
if(m == NULL || strlen(m) == 0) continue;
if(i > 0) ncbytescat(buf,",");
ncbytescat(buf,m);
}
result = ncbytesextract(buf);
ncbytesfree(buf);
if(result == NULL) result = strdup("");
return result;
}
#if 0
/* Given a comma separated string, remove duplicates; mostly used to cleanup mode list */
static char*
cleancommalist(const char* commalist, int caseinsensitive)
{
NClist* tmp = nclistnew();
char* newlist = NULL;
if(commalist == NULL || strlen(commalist)==0) return nulldup(commalist);
(void)parseonchar(commalist,',',tmp);/* split on commas */
cleanstringlist(tmp,caseinsensitive);
newlist = list2string(tmp);
nclistfreeall(tmp);
return newlist;
}
#endif
/* Given a list of strings, remove nulls and duplicated */
static void
cleanstringlist(NClist* strs, int caseinsensitive)
{
int i,j;
if(nclistlength(strs) == 0) return;
/* Remove nulls */
for(i=nclistlength(strs)-1;i>=0;i--) {
if(nclistget(strs,i)==NULL) nclistremove(strs,i);
}
/* Remove duplicates*/
for(i=0;i<nclistlength(strs);i++) {
const char* value = nclistget(strs,i);
/* look ahead for duplicates */
for(j=nclistlength(strs)-1;j>i;j--) {
int match;
const char* candidate = nclistget(strs,j);
if(caseinsensitive)
match = (strcasecmp(value,candidate) == 0);
else
match = (strcmp(value,candidate) == 0);
if(match) {char* dup = nclistremove(strs,j); nullfree(dup);}
}
}
}
/**************************************************/
/**
* @internal Given an existing file, figure out its format and return
@ -1502,8 +1595,10 @@ printlist(NClist* list, const char* tag)
{
int i;
fprintf(stderr,"%s:",tag);
for(i=0;i<nclistlength(list);i++)
for(i=0;i<nclistlength(list);i++) {
fprintf(stderr," %s",(char*)nclistget(list,i));
fprintf(stderr,"[%p]",(char*)nclistget(list,i));
}
fprintf(stderr,"\n");
dbgflush();
}

4
libdispatch/ncbytes.c
View File

@ -122,9 +122,7 @@ ncbytesappend(NCbytes* bb, char elem)
int
ncbytescat(NCbytes* bb, const char* s)
{
if(s == NULL) {
return 1;
}
if(s == NULL) return 1;
ncbytesappendn(bb,(void*)s,strlen(s)+1); /* include trailing null*/
/* back up over the trailing null*/
if(bb->length == 0) return ncbytesfail();

4
libdispatch/nclist.c
View File

@ -183,6 +183,7 @@ nclistremove(NClist* l, size_t i)
return elem;
}
/* Match on == */
int
nclistcontains(NClist* l, void* elem)
{
@ -193,7 +194,7 @@ nclistcontains(NClist* l, void* elem)
return 0;
}
/* Return 1/0 */
/* Match on str(case)cmp */
int
nclistmatch(NClist* l, const char* elem, int casesensitive)
{
@ -230,7 +231,6 @@ nclistelemremove(NClist* l, void* elem)
return found;
}
/* Extends nclist to include a unique operator
which remove duplicate values; NULL values removed
return value is always 1.

11
libnczarr/zsync.c
View File

@ -1429,6 +1429,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
char* varpath = NULL;
char* key = NULL;
NCZ_FILE_INFO_T* zinfo = NULL;
NC_VAR_INFO_T* var = NULL;
NCZ_VAR_INFO_T* zvar = NULL;
NCZMAP* map = NULL;
NCjson* jvar = NULL;
@ -1460,7 +1461,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* Load each var in turn */
for(i = 0; i < nclistlength(varnames); i++) {
NC_VAR_INFO_T* var;
const char* varname = nclistget(varnames,i);
if((stat = nc4_var_list_add2(grp, varname, &var)))
goto done;
@ -1477,10 +1477,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* Indicate we do not have quantizer yet */
var->quantize_mode = -1;
/* Set filter list */
assert(var->filters == NULL);
var->filters = (void*)nclistnew();
/* Construct var path */
if((stat = NCZ_varkey(var,&varpath)))
goto done;
@ -1697,9 +1693,9 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
object MUST contain a "id" key identifying the codec to be used. */
/* Do filters key before compressor key so final filter chain is in correct order */
{
#ifdef ENABLE_NCZARR_FILTERS
if(var->filters == NULL) var->filters = (void*)nclistnew();
if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew();
#ifdef ENABLE_NCZARR_FILTERS
{ int k;
chainindex = 0; /* track location of filter in the chain */
if((stat = NCZ_filter_initialize())) goto done;
@ -1722,8 +1718,8 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
/* From V2 Spec: A JSON object identifying the primary compression codec and providing
configuration parameters, or ``null`` if no compressor is to be used. */
{
if(var->filters == NULL) var->filters = (void*)nclistnew();
#ifdef ENABLE_NCZARR_FILTERS
if(var->filters == NULL) var->filters = (void*)nclistnew();
if((stat = NCZ_filter_initialize())) goto done;
if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done;
if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) {
@ -1752,6 +1748,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
nullfree(shapes); shapes = NULL;
if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;}
NCJreclaim(jvar); jvar = NULL;
var = NULL;
}
done:

6
libnczarr/zvar.c
View File

@ -391,9 +391,11 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims,
var->meta_read = NC_TRUE;
var->atts_read = NC_TRUE;
#ifdef ENABLE_NCZARR_FILTERS
/* Set the filter list */
assert(var->filters == NULL);
var->filters = (void*)nclistnew();
#endif
/* Point to the type, and increment its ref. count */
var->type_info = type;
@ -558,10 +560,12 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
/* Can't turn on parallel and deflate/fletcher32/szip/shuffle
* before HDF5 1.10.3. */
#ifdef ENABLE_NCZARR_FILTERS
#ifndef HDF5_SUPPORTS_PAR_FILTERS
if (h5->parallel == NC_TRUE)
if (nclistlength(((NClist*)var->filters)) > 0 || fletcher32 || shuffle)
{retval = NC_EINVAL; goto done;}
#endif
#endif
/* If the HDF5 dataset has already been created, then it is too
@ -628,8 +632,10 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1,
* no filters in use for this data. */
if (storage != NC_CHUNKED)
{
#ifdef NCZARR_FILTERS
if (nclistlength(((NClist*)var->filters)) > 0)
{retval = NC_EINVAL; goto done;}
#endif
for (d = 0; d < var->ndims; d++)
if (var->dim[d]->unlimited)
{retval = NC_EINVAL; goto done;}