From f37fe57cebe207977380dde6973fcf422ac619cc Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Fri, 24 Apr 2020 09:25:21 -0500 Subject: [PATCH 01/21] added check for DAOS file --- libdispatch/dinfermodel.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index d4e144f65..41332eab5 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -490,6 +490,27 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void int omode = *omodep; NClist* modeargs = nclistnew(); +#if H5_VERSION_GE(1,12,0) + + hid_t fapl_id; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; + // printf("testing if %s accessible \n", path); + if( H5Fis_accessible(path, fapl_id) == 1 ) { + + hid_t H5VL_id = H5VLget_connector_id(fapl_id); + if( H5VL_id != H5VL_NATIVE && H5VL_id != H5I_INVALID_HID) { + printf("DAOS OBJECT is %s accessible \n", path); + model->impl = NC_FORMATX_NC4; + model->format = NC_FORMAT_NETCDF4; + if (H5Pclose(fapl_id) < 0) goto done; + goto done; + } + } + + if (H5Pclose(fapl_id) < 0) goto done; + +#endif + /* Phase 1: Reformat the uri to canonical form; store canonical form into newpath. Return the "mode=" list in modeargs */ if((stat = processuri(path, &uri, &newpath, modeargs))) goto done; From 8f2d3d58c6c206d0471460a3627886d7da6a9331 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Fri, 24 Apr 2020 09:42:07 -0500 Subject: [PATCH 02/21] added check for DAOS file --- libdispatch/dinfermodel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 41332eab5..edcafc72f 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -27,6 +27,7 @@ #ifdef ENABLE_BYTERANGE #include "nchttp.h" #endif +#include "hdf5.h" #undef DEBUG From 32ff909df1ad77843fa02fd5a23b5a766eabe9db Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 22 Jul 2020 16:00:41 +0000 Subject: [PATCH 03/21] Merged from master --- libdispatch/dinfermodel.c | 58 +++++++++++++++++++++++++++++---------- libhdf5/hdf5open.c | 34 +++++++++++++++++++++-- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 1a337b087..b7b0b7e45 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -7,7 +7,7 @@ * Copyright 2018 University Corporation for Atmospheric * Research/Unidata. See COPYRIGHT file for more info. */ - +#include #include "config.h" #include #ifdef HAVE_UNISTD_H @@ -699,29 +699,57 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void const char* modeval = NULL; #if H5_VERSION_GE(1,12,0) - hid_t fapl_id; - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; - // printf("testing if %s accessible \n", path); - if( H5Fis_accessible(path, fapl_id) == 1 ) { +#if 1 +/* if( H5VLis_connector_registered_by_name("daos") > 0) { */ + hid_t fapl_id; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; + H5Pset_fapl_sec2(fapl_id); +#if 0 + hid_t file_id = H5Fopen(path, H5F_ACC_RDONLY, fapl_id); + if( file_id < 0) { + printf("failed to open file\n"); + } else { + printf("opened file \n"); + } +#endif + int rank; + int size; + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + MPI_Comm_size(MPI_COMM_WORLD,&size); + printf("NETCDF RANK %d %d\n",rank,size); - hid_t H5VL_id = H5VLget_connector_id(fapl_id); - if( H5VL_id != H5VL_NATIVE && H5VL_id != H5I_INVALID_HID) { - printf("DAOS OBJECT is %s accessible \n", path); - model->impl = NC_FORMATX_NC4; - model->format = NC_FORMAT_NETCDF4; - if (H5Pclose(fapl_id) < 0) goto done; - goto done; + htri_t accessible; + accessible = H5Fis_accessible(path, fapl_id); + printf("IS H5Fis_accessible %d \n",accessible); + if(accessible) { + int POSIX=access(path, F_OK); + printf("POSIX %d \n", POSIX); + if( POSIX != 0 ) { + printf("\033[36;01m H5Fis_accessible %s IS accessible \033[0m \n", path); + + hid_t vol_id; + herr_t status = H5Pget_vol_id ( fapl_id, &vol_id ); + hid_t vol_id_by_name = H5VLget_connector_id_by_name ( "daos" ); + if( vol_id == vol_id_by_name ) { + printf(" \033[37;01m %s IS A DAOS OBJECT \033[0m \n", path); + model->impl = NC_FORMATX_NC4; + model->format = NC_FORMAT_NETCDF4; + if (H5Pclose(fapl_id) < 0) goto done; + goto done; + } } - } - if (H5Pclose(fapl_id) < 0) goto done; + } + if (H5Pclose(fapl_id) < 0) goto done; +// } #endif + printf("Done with check\n"); /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments */ if((stat = processuri(path, &uri, fraglenv))) goto done; - + if(uri != NULL) { #ifdef DEBUG printlist(fraglenv,"processuri"); diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 06b5adbae..75150620b 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -864,12 +864,19 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) * information may be difficult to resolve here, if, for example, a * dataset of user-defined type is encountered before the * definition of that type. */ + printf("AA rec_read_metadata \n"); if ((retval = rec_read_metadata(nc4_info->root_grp))) BAIL(retval); + printf("Check for classic model attribute. \n"); + /* Check for classic model attribute. */ if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) BAIL(retval); + + printf("AFTER Check for classic model attribute. \n"); + + if (is_classic) nc4_info->cmode |= NC_CLASSIC_MODEL; @@ -891,7 +898,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Close the property list. */ if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); - + printf("DONE\n"); return NC_NOERR; exit: @@ -2406,8 +2413,13 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, BAIL(NC_EHDFERR); /* Is this a dimscale? */ + + printf("H5DSis_scale \n"); + if ((is_scale = H5DSis_scale(datasetid)) < 0) BAIL(NC_EHDFERR); + printf("AFTER H5DSis_scale \n"); + if (is_scale) { hsize_t dims[H5S_MAX_RANK]; @@ -2492,6 +2504,8 @@ static int read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, void *_op_data) { + + printf("inside read_hdf5_obj\n"); /* Pointer to user data for callback */ user_data_t *udata = (user_data_t *)_op_data; hdf5_obj_info_t oinfo; /* Pointer to info for object */ @@ -2503,21 +2517,25 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, /* Get info about the object.*/ #if H5_VERSION_GE(1,12,0) + printf("H5Oget_info \n"); if (H5Oget_info3(oinfo.oid, &oinfo.statbuf, H5O_INFO_BASIC) < 0) BAIL(H5_ITER_ERROR); #else if (H5Gget_objinfo(oinfo.oid, ".", 1, &oinfo.statbuf) < 0) BAIL(H5_ITER_ERROR); #endif + printf("End H5Oget_info \n"); strncpy(oinfo.oname, name, NC_MAX_NAME); + + printf("NAME %s %d\n", name, oinfo.statbuf.type); /* Add object to list, for later */ switch(oinfo.statbuf.type) { case H5G_GROUP: LOG((3, "found group %s", oinfo.oname)); - + printf("found group %s\n", oinfo.oname); /* Defer descending into child group immediately, so that the * types in the current group can be processed and be ready for * use by vars in the child group(s). */ @@ -2527,12 +2545,14 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, case H5G_DATASET: LOG((3, "found dataset %s", oinfo.oname)); - + printf("found dataset %s\n", oinfo.oname); /* Learn all about this dataset, which may be a dimscale * (i.e. dimension metadata), or real data. */ if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, &oinfo.statbuf))) { + + printf("failed read_dataset \n"); /* Allow NC_EBADTYPID to transparently skip over datasets * which have a datatype that netCDF-4 doesn't understand * (currently), but break out of iteration for other @@ -2544,6 +2564,8 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, } /* Close the object */ + + printf("H5Oclose \n"); if (H5Oclose(oinfo.oid) < 0) BAIL(H5_ITER_ERROR); break; @@ -2647,6 +2669,7 @@ rec_read_metadata(NC_GRP_INFO_T *grp) if (H5Pget_link_creation_order(pid, &crt_order_flags) < 0) BAIL(NC_EHDFERR); + /* Set the iteration index to use. */ if (crt_order_flags & H5P_CRT_ORDER_TRACKED) iter_index = H5_INDEX_CRT_ORDER; @@ -2666,10 +2689,15 @@ rec_read_metadata(NC_GRP_INFO_T *grp) * passed as a parameter to the callback function * read_hdf5_obj(). (I have also tried H5Oiterate(), but it is much * slower iterating over the same file - Ed.) */ + + printf("H5Literate, %ld %d %d \n", hdf5_grp->hdf_grpid, iter_index, idx); + if (H5Literate(hdf5_grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, read_hdf5_obj, (void *)&udata) < 0) BAIL(NC_EHDFERR); + printf("after H5Literate \n"); + /* Process the child groups found. (Deferred until now, so that the * types in the current group get processed and are available for * vars in the child group(s).) */ From 269c306c1d6505a76f3cc088b31d44947c6fb979 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 22 Jul 2020 18:01:26 +0000 Subject: [PATCH 04/21] update --- libdispatch/dinfermodel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index b7b0b7e45..39f2ab464 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -741,6 +741,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void } if (H5Pclose(fapl_id) < 0) goto done; // } +#endif #endif printf("Done with check\n"); From ea4cfa45628f7edfbcc5f108eb045f17a16fd305 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Fri, 25 Sep 2020 21:39:16 +0000 Subject: [PATCH 05/21] removed used of DAOS VOL APIs --- libdispatch/dinfermodel.c | 58 +++++++++++++++------------------------ libhdf5/hdf5open.c | 25 ----------------- libhdf5/nc4info.c | 2 +- 3 files changed, 23 insertions(+), 62 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 39f2ab464..22446d74a 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -699,51 +699,37 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void const char* modeval = NULL; #if H5_VERSION_GE(1,12,0) -#if 1 -/* if( H5VLis_connector_registered_by_name("daos") > 0) { */ - hid_t fapl_id; - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; - H5Pset_fapl_sec2(fapl_id); -#if 0 - hid_t file_id = H5Fopen(path, H5F_ACC_RDONLY, fapl_id); - if( file_id < 0) { - printf("failed to open file\n"); - } else { - printf("opened file \n"); - } -#endif - int rank; - int size; - MPI_Comm_rank(MPI_COMM_WORLD,&rank); - MPI_Comm_size(MPI_COMM_WORLD,&size); - printf("NETCDF RANK %d %d\n",rank,size); - - htri_t accessible; - accessible = H5Fis_accessible(path, fapl_id); - printf("IS H5Fis_accessible %d \n",accessible); - if(accessible) { - int POSIX=access(path, F_OK); - printf("POSIX %d \n", POSIX); - if( POSIX != 0 ) { - printf("\033[36;01m H5Fis_accessible %s IS accessible \033[0m \n", path); + hid_t fapl_id; + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; + H5Pset_fapl_sec2(fapl_id); + + htri_t accessible; + accessible = H5Fis_accessible(path, fapl_id); + printf("IS H5Fis_accessible %d \n",accessible); + if(accessible) { + int rc=0; + FILE *fp; + char *cmd; + cmd = (char*)malloc((strlen(path)+28)*sizeof(char)); + strcpy(cmd, "getfattr "); + strcat(cmd, path); + strcat(cmd, " | grep -c '.daos'"); - hid_t vol_id; - herr_t status = H5Pget_vol_id ( fapl_id, &vol_id ); - hid_t vol_id_by_name = H5VLget_connector_id_by_name ( "daos" ); - if( vol_id == vol_id_by_name ) { + if((fp = popen(cmd, "r")) != NULL) { + fscanf(fp, "%d", &rc); + pclose(fp); + } + free(cmd); + if(rc == 1) { printf(" \033[37;01m %s IS A DAOS OBJECT \033[0m \n", path); model->impl = NC_FORMATX_NC4; model->format = NC_FORMAT_NETCDF4; if (H5Pclose(fapl_id) < 0) goto done; goto done; } - } - } + } if (H5Pclose(fapl_id) < 0) goto done; -// } #endif -#endif - printf("Done with check\n"); /* Phase 1: 1. convert special protocols to http|https diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 75150620b..ed25fde04 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -864,19 +864,13 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) * information may be difficult to resolve here, if, for example, a * dataset of user-defined type is encountered before the * definition of that type. */ - printf("AA rec_read_metadata \n"); if ((retval = rec_read_metadata(nc4_info->root_grp))) BAIL(retval); - printf("Check for classic model attribute. \n"); - /* Check for classic model attribute. */ if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) BAIL(retval); - printf("AFTER Check for classic model attribute. \n"); - - if (is_classic) nc4_info->cmode |= NC_CLASSIC_MODEL; @@ -898,7 +892,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Close the property list. */ if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); - printf("DONE\n"); return NC_NOERR; exit: @@ -2413,12 +2406,8 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, BAIL(NC_EHDFERR); /* Is this a dimscale? */ - - printf("H5DSis_scale \n"); - if ((is_scale = H5DSis_scale(datasetid)) < 0) BAIL(NC_EHDFERR); - printf("AFTER H5DSis_scale \n"); if (is_scale) { @@ -2505,7 +2494,6 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, void *_op_data) { - printf("inside read_hdf5_obj\n"); /* Pointer to user data for callback */ user_data_t *udata = (user_data_t *)_op_data; hdf5_obj_info_t oinfo; /* Pointer to info for object */ @@ -2517,25 +2505,20 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, /* Get info about the object.*/ #if H5_VERSION_GE(1,12,0) - printf("H5Oget_info \n"); if (H5Oget_info3(oinfo.oid, &oinfo.statbuf, H5O_INFO_BASIC) < 0) BAIL(H5_ITER_ERROR); #else if (H5Gget_objinfo(oinfo.oid, ".", 1, &oinfo.statbuf) < 0) BAIL(H5_ITER_ERROR); #endif - printf("End H5Oget_info \n"); strncpy(oinfo.oname, name, NC_MAX_NAME); - - printf("NAME %s %d\n", name, oinfo.statbuf.type); /* Add object to list, for later */ switch(oinfo.statbuf.type) { case H5G_GROUP: LOG((3, "found group %s", oinfo.oname)); - printf("found group %s\n", oinfo.oname); /* Defer descending into child group immediately, so that the * types in the current group can be processed and be ready for * use by vars in the child group(s). */ @@ -2545,14 +2528,12 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, case H5G_DATASET: LOG((3, "found dataset %s", oinfo.oname)); - printf("found dataset %s\n", oinfo.oname); /* Learn all about this dataset, which may be a dimscale * (i.e. dimension metadata), or real data. */ if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, &oinfo.statbuf))) { - printf("failed read_dataset \n"); /* Allow NC_EBADTYPID to transparently skip over datasets * which have a datatype that netCDF-4 doesn't understand * (currently), but break out of iteration for other @@ -2564,8 +2545,6 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, } /* Close the object */ - - printf("H5Oclose \n"); if (H5Oclose(oinfo.oid) < 0) BAIL(H5_ITER_ERROR); break; @@ -2690,14 +2669,10 @@ rec_read_metadata(NC_GRP_INFO_T *grp) * read_hdf5_obj(). (I have also tried H5Oiterate(), but it is much * slower iterating over the same file - Ed.) */ - printf("H5Literate, %ld %d %d \n", hdf5_grp->hdf_grpid, iter_index, idx); - if (H5Literate(hdf5_grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, read_hdf5_obj, (void *)&udata) < 0) BAIL(NC_EHDFERR); - printf("after H5Literate \n"); - /* Process the child groups found. (Deferred until now, so that the * types in the current group get processed and are available for * vars in the child group(s).) */ diff --git a/libhdf5/nc4info.c b/libhdf5/nc4info.c index a0cb25f2d..742ec4640 100644 --- a/libhdf5/nc4info.c +++ b/libhdf5/nc4info.c @@ -258,7 +258,7 @@ NC4_read_ncproperties(NC_FILE_INFO_T* h5, char** propstring) } /* NCPROPS Attribute exists, make sure it is legitimate */ - attid = H5Aopen_name(hdf5grpid, NCPROPS); + attid = H5Aopen_by_name(hdf5grpid, ".", NCPROPS, H5P_DEFAULT, H5P_DEFAULT); assert(attid > 0); aspace = H5Aget_space(attid); atype = H5Aget_type(attid); From deff6a5927ea2b719141135fcc0294a0c685a8cb Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 28 Sep 2020 16:23:04 +0000 Subject: [PATCH 06/21] debugging --- libdispatch/dinfermodel.c | 8 ++++---- libhdf5/hdf5dim.c | 14 +++++++------- libsrc4/ncindex.c | 1 + 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 22446d74a..d41de5fc5 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -697,7 +697,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void NClist* modeargs = nclistnew(); char* sfrag = NULL; const char* modeval = NULL; - +#if 0 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; @@ -705,8 +705,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void htri_t accessible; accessible = H5Fis_accessible(path, fapl_id); - printf("IS H5Fis_accessible %d \n",accessible); - if(accessible) { + printf("%s IS H5Fis_accessible %d \n",path,accessible); + if(accessible > 0) { int rc=0; FILE *fp; char *cmd; @@ -730,7 +730,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void } if (H5Pclose(fapl_id) < 0) goto done; #endif - +#endif /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index fd93bf9b2..bdb013ecb 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -50,7 +50,8 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) LOG((2, "%s: ncid 0x%x name %s len %d", __func__, ncid, name, (int)len)); - + printf("%s: ncid 0x%x name %s len %d \n", __func__, ncid, name, + (int)len); /* Find our global metadata structure. */ if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) return retval; @@ -59,7 +60,6 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) /* If the file is read-only, return an error. */ if (h5->no_write) return NC_EPERM; - /* Check some stuff if strict nc3 rules are in effect. */ if (h5->cmode & NC_CLASSIC_MODEL) { @@ -76,7 +76,6 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) if (!(h5->flags & NC_INDEF)) return NC_ENOTINDEFINE; } - /* Make sure this is a valid netcdf name. */ if ((retval = nc4_check_name(name, norm_name))) return retval; @@ -86,19 +85,20 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) if (h5->cmode & NC_CLASSIC_MODEL) if(len > X_UINT_MAX) /* Backward compat */ return NC_EDIMSIZE; - + printf("HDF5 77\n"); /* Make sure the name is not already in use. */ dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); - if(dim != NULL) + if(dim != NULL){ + printf("ncindexlookup eerr HDF5 7x7\n"); return NC_ENAMEINUSE; - +} + printf("HDF5 88\n"); /* If it's not in define mode, enter define mode. Do this only * after checking all input data, so we only enter define mode if * input is good. */ if (!(h5->flags & NC_INDEF)) if ((retval = NC4_redef(ncid))) return retval; - /* Add a dimension to the list. The ID must come from the file * information, since dimids are visible in more than one group. */ if ((retval = nc4_dim_list_add(grp, norm_name, len, -1, &dim))) diff --git a/libsrc4/ncindex.c b/libsrc4/ncindex.c index 5b20d4d66..e69e520fc 100644 --- a/libsrc4/ncindex.c +++ b/libsrc4/ncindex.c @@ -59,6 +59,7 @@ ncindexlookup(NCindex* ncindex, const char* name) assert(ncindex->map != NULL); if(!NC_hashmapget(ncindex->map,(void*)name,strlen(name),&index)) return NULL; /* not present */ + printf("lkjdsf\n"); obj = (NC_OBJ*)nclistget(ncindex->list,(size_t)index); #else int i; From 770caff80a4d838f2aecd8ff1feb4dd5d77f6617 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 28 Dec 2020 19:37:10 +0000 Subject: [PATCH 07/21] added timers --- include/netcdf.h | 2 ++ libdispatch/dinfermodel.c | 2 +- libhdf5/hdf5create.c | 2 +- libhdf5/hdf5dim.c | 8 +++---- libhdf5/hdf5file.c | 46 ++++++++++++++++++++++++++++++++++++++- libhdf5/hdf5var.c | 8 ++++++- libsrc4/ncindex.c | 1 - 7 files changed, 60 insertions(+), 9 deletions(-) diff --git a/include/netcdf.h b/include/netcdf.h index 4395b30e8..6e4b84990 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -1971,6 +1971,8 @@ ncrecget(int ncid, long recnum, void **datap); EXTERNL int ncrecput(int ncid, long recnum, void *const *datap); +EXTERNL double dt_h5dwrite; +EXTERNL double dt_h5dread; /* This function may be called to force the library to initialize itself. It is not required, however. */ diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index d41de5fc5..42eb2769a 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -697,7 +697,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void NClist* modeargs = nclistnew(); char* sfrag = NULL; const char* modeval = NULL; -#if 0 +#if 1 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 27a99405c..60f8588bb 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -168,7 +168,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, #ifdef HAVE_H5PSET_LIBVER_BOUNDS #if H5_VERSION_GE(1,10,2) - if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0) + if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_V112) < 0) #else if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index bdb013ecb..49768d16e 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -50,8 +50,8 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) LOG((2, "%s: ncid 0x%x name %s len %d", __func__, ncid, name, (int)len)); - printf("%s: ncid 0x%x name %s len %d \n", __func__, ncid, name, - (int)len); + //printf("%s: ncid 0x%x name %s len %d \n", __func__, ncid, name, + // (int)len); /* Find our global metadata structure. */ if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) return retval; @@ -85,14 +85,14 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) if (h5->cmode & NC_CLASSIC_MODEL) if(len > X_UINT_MAX) /* Backward compat */ return NC_EDIMSIZE; - printf("HDF5 77\n"); + // printf("HDF5 77\n"); /* Make sure the name is not already in use. */ dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); if(dim != NULL){ printf("ncindexlookup eerr HDF5 7x7\n"); return NC_ENAMEINUSE; } - printf("HDF5 88\n"); + //printf("HDF5 88\n"); /* If it's not in define mode, enter define mode. Do this only * after checking all input data, so we only enter define mode if * input is good. */ diff --git a/libhdf5/hdf5file.c b/libhdf5/hdf5file.c index 131a28569..bdfada27c 100644 --- a/libhdf5/hdf5file.c +++ b/libhdf5/hdf5file.c @@ -14,6 +14,9 @@ #include "config.h" #include "hdf5internal.h" #include "ncrc.h" +#include +double dt_h5dwrite= 0.; +double dt_h5dread = 0.; extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); /* In nc4memcb.c */ @@ -199,6 +202,11 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) { NC_HDF5_FILE_INFO_T *hdf5_info; int retval; + int rank, nprocs, i; + double *rtimers=NULL; /* All timers from ranks */ + double min[2]; + double max[2]; + double mean[2]; assert(h5 && h5->root_grp && h5->format_file_info); LOG((3, "%s: h5->path %s abort %d", __func__, h5->controller->path, abort)); @@ -209,6 +217,42 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) #ifdef USE_PARALLEL4 /* Free the MPI Comm & Info objects, if we opened the file in * parallel. */ + MPI_Comm_rank(h5->comm, &rank); + MPI_Comm_size(h5->comm, &nprocs); + if(rank == 0) { + rtimers = (double *) malloc(nprocs*sizeof(double)); + mean[0] = 0.; + min[0] = dt_h5dwrite; + max[0] = dt_h5dwrite; + } + MPI_Gather(&dt_h5dwrite, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, 0, h5->comm); + if(rank == 0) { + for(i = 0; i < nprocs; i++) { + if(rtimers[i] > max[0]) max[0] = rtimers[i]; + if(rtimers[i] < min[0]) min[0] = rtimers[i]; + mean[0] += rtimers[i]; + } + mean[0] /= nprocs; + free(rtimers); + } + if(rank == 0) { + rtimers = (double *) malloc(nprocs*sizeof(double)); + mean[1] = 0.; + min[1] = dt_h5dread; + max[1] = dt_h5dread; + } + + MPI_Gather(&dt_h5dread, 1, MPI_DOUBLE, rtimers, 1, MPI_DOUBLE, 0, h5->comm); + if(rank == 0) { + for(i = 0; i < nprocs; i++) { + if(rtimers[i] > max[1]) max[1] = rtimers[i]; + if(rtimers[i] < min[1]) min[1] = rtimers[i]; + mean[1] += rtimers[i]; + } + mean[1] /= nprocs; + printf("%s %d %d %f %f %f %f %f %f\n","#wr", rank, nprocs, mean[0], min[0], max[0], mean[1], min[1], max[1]); + free(rtimers); + } if (h5->parallel) { if (h5->comm != MPI_COMM_NULL) @@ -229,7 +273,7 @@ nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio) dumpopenobjects(h5); return NC_EHDFERR; } - + /* If inmemory is used and user wants the final memory block, then capture and return the final memory block else free it */ if (h5->mem.inmemory) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 6311cf9f9..c507fd5c3 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1753,11 +1753,14 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Write the data. At last! */ LOG((4, "about to H5Dwrite datasetid 0x%x mem_spaceid 0x%x " "file_spaceid 0x%x", hdf5_var->hdf_datasetid, mem_spaceid, file_spaceid)); + double t1, t2; + t1 = MPI_Wtime(); if (H5Dwrite(hdf5_var->hdf_datasetid, ((NC_HDF5_TYPE_INFO_T *)var->type_info->format_type_info)->hdf_typeid, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); - + t2 = MPI_Wtime(); + dt_h5dwrite = dt_h5dwrite + (t2-t1); /* Remember that we have written to this var so that Fill Value * can't be set for it. */ if (!var->written_to) @@ -2043,10 +2046,13 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Read this hyperslab into memory. */ LOG((5, "About to H5Dread some data...")); + double t1 = MPI_Wtime(); if (H5Dread(hdf5_var->hdf_datasetid, ((NC_HDF5_TYPE_INFO_T *)var->type_info->format_type_info)->native_hdf_typeid, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); + double t2 = MPI_Wtime(); + dt_h5dread = dt_h5dread + (t2-t1); } /* endif ! no_read */ else { diff --git a/libsrc4/ncindex.c b/libsrc4/ncindex.c index e69e520fc..5b20d4d66 100644 --- a/libsrc4/ncindex.c +++ b/libsrc4/ncindex.c @@ -59,7 +59,6 @@ ncindexlookup(NCindex* ncindex, const char* name) assert(ncindex->map != NULL); if(!NC_hashmapget(ncindex->map,(void*)name,strlen(name),&index)) return NULL; /* not present */ - printf("lkjdsf\n"); obj = (NC_OBJ*)nclistget(ncindex->list,(size_t)index); #else int i; From 5254a8f1902d6460844f6e18cc18f529021ee4d8 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 09:46:50 -0500 Subject: [PATCH 08/21] removed timing and debugging --- include/netcdf.h | 2 -- libdispatch/dinfermodel.c | 8 +++++--- libhdf5/hdf5dim.c | 14 +++++++------- libhdf5/hdf5open.c | 9 +++------ libhdf5/hdf5var.c | 8 +------- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/include/netcdf.h b/include/netcdf.h index 6a07a2aa4..b6d434a78 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -1972,8 +1972,6 @@ ncrecget(int ncid, long recnum, void **datap); EXTERNL int ncrecput(int ncid, long recnum, void *const *datap); -EXTERNL double dt_h5dwrite; -EXTERNL double dt_h5dread; /* This function may be called to force the library to initialize itself. It is not required, however. */ diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 260beed3b..21c62d71c 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -778,7 +778,9 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void NClist* modeargs = nclistnew(); char* sfrag = NULL; const char* modeval = NULL; -#if 1 + + /* Check for a DAOS contianer */ +#ifdef USE_NETCDF4 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; @@ -786,7 +788,6 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void htri_t accessible; accessible = H5Fis_accessible(path, fapl_id); - printf("%s IS H5Fis_accessible %d \n",path,accessible); if(accessible > 0) { int rc=0; FILE *fp; @@ -802,7 +803,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void } free(cmd); if(rc == 1) { - printf(" \033[37;01m %s IS A DAOS OBJECT \033[0m \n", path); + /* Is a DAOS container */ model->impl = NC_FORMATX_NC4; model->format = NC_FORMAT_NETCDF4; if (H5Pclose(fapl_id) < 0) goto done; @@ -812,6 +813,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void if (H5Pclose(fapl_id) < 0) goto done; #endif #endif + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments diff --git a/libhdf5/hdf5dim.c b/libhdf5/hdf5dim.c index 056ede15e..45fd8c554 100644 --- a/libhdf5/hdf5dim.c +++ b/libhdf5/hdf5dim.c @@ -50,8 +50,7 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) LOG((2, "%s: ncid 0x%x name %s len %d", __func__, ncid, name, (int)len)); - //printf("%s: ncid 0x%x name %s len %d \n", __func__, ncid, name, - // (int)len); + /* Find our global metadata structure. */ if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5))) return retval; @@ -60,6 +59,7 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) /* If the file is read-only, return an error. */ if (h5->no_write) return NC_EPERM; + /* Check some stuff if strict nc3 rules are in effect. */ if (h5->cmode & NC_CLASSIC_MODEL) { @@ -76,6 +76,7 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) if (!(h5->flags & NC_INDEF)) return NC_ENOTINDEFINE; } + /* Make sure this is a valid netcdf name. */ if ((retval = nc4_check_name(name, norm_name))) return retval; @@ -85,20 +86,19 @@ HDF5_def_dim(int ncid, const char *name, size_t len, int *idp) if (h5->cmode & NC_CLASSIC_MODEL) if(len > X_UINT_MAX) /* Backward compat */ return NC_EDIMSIZE; - // printf("HDF5 77\n"); + /* Make sure the name is not already in use. */ dim = (NC_DIM_INFO_T*)ncindexlookup(grp->dim,norm_name); - if(dim != NULL){ - printf("ncindexlookup eerr HDF5 7x7\n"); + if(dim != NULL) return NC_ENAMEINUSE; -} - //printf("HDF5 88\n"); + /* If it's not in define mode, enter define mode. Do this only * after checking all input data, so we only enter define mode if * input is good. */ if (!(h5->flags & NC_INDEF)) if ((retval = NC4_redef(ncid))) return retval; + /* Add a dimension to the list. The ID must come from the file * information, since dimids are visible in more than one group. */ if ((retval = nc4_dim_list_add(grp, norm_name, len, -1, &dim))) diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index be5cf1611..ade593458 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -909,7 +909,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Check for classic model attribute. */ if ((retval = check_for_classic_model(nc4_info->root_grp, &is_classic))) BAIL(retval); - if (is_classic) nc4_info->cmode |= NC_CLASSIC_MODEL; @@ -931,6 +930,7 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) /* Close the property list. */ if (H5Pclose(fapl_id) < 0) BAIL(NC_EHDFERR); + return NC_NOERR; exit: @@ -2464,7 +2464,6 @@ read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid, const char *obj_name, /* Is this a dimscale? */ if ((is_scale = H5DSis_scale(datasetid)) < 0) BAIL(NC_EHDFERR); - if (is_scale) { hsize_t dims[H5S_MAX_RANK]; @@ -2549,7 +2548,6 @@ static int read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, void *_op_data) { - /* Pointer to user data for callback */ user_data_t *udata = (user_data_t *)_op_data; hdf5_obj_info_t oinfo; /* Pointer to info for object */ @@ -2575,6 +2573,7 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, { case H5G_GROUP: LOG((3, "found group %s", oinfo.oname)); + /* Defer descending into child group immediately, so that the * types in the current group can be processed and be ready for * use by vars in the child group(s). */ @@ -2584,12 +2583,12 @@ read_hdf5_obj(hid_t grpid, const char *name, const H5L_info_t *info, case H5G_DATASET: LOG((3, "found dataset %s", oinfo.oname)); + /* Learn all about this dataset, which may be a dimscale * (i.e. dimension metadata), or real data. */ if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, &oinfo.statbuf))) { - /* Allow NC_EBADTYPID to transparently skip over datasets * which have a datatype that netCDF-4 doesn't understand * (currently), but break out of iteration for other @@ -2704,7 +2703,6 @@ rec_read_metadata(NC_GRP_INFO_T *grp) if (H5Pget_link_creation_order(pid, &crt_order_flags) < 0) BAIL(NC_EHDFERR); - /* Set the iteration index to use. */ if (crt_order_flags & H5P_CRT_ORDER_TRACKED) iter_index = H5_INDEX_CRT_ORDER; @@ -2724,7 +2722,6 @@ rec_read_metadata(NC_GRP_INFO_T *grp) * passed as a parameter to the callback function * read_hdf5_obj(). (I have also tried H5Oiterate(), but it is much * slower iterating over the same file - Ed.) */ - if (H5Literate(hdf5_grp->hdf_grpid, iter_index, H5_ITER_INC, &idx, read_hdf5_obj, (void *)&udata) < 0) BAIL(NC_EHDFERR); diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index f7e5d33c6..45a218d44 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1591,14 +1591,11 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Write the data. At last! */ LOG((4, "about to H5Dwrite datasetid 0x%x mem_spaceid 0x%x " "file_spaceid 0x%x", hdf5_var->hdf_datasetid, mem_spaceid, file_spaceid)); - double t1, t2; - t1 = MPI_Wtime(); if (H5Dwrite(hdf5_var->hdf_datasetid, ((NC_HDF5_TYPE_INFO_T *)var->type_info->format_type_info)->hdf_typeid, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); - t2 = MPI_Wtime(); - dt_h5dwrite = dt_h5dwrite + (t2-t1); + /* Remember that we have written to this var so that Fill Value * can't be set for it. */ if (!var->written_to) @@ -1892,13 +1889,10 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, /* Read this hyperslab into memory. */ LOG((5, "About to H5Dread some data...")); - double t1 = MPI_Wtime(); if (H5Dread(hdf5_var->hdf_datasetid, ((NC_HDF5_TYPE_INFO_T *)var->type_info->format_type_info)->native_hdf_typeid, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); - double t2 = MPI_Wtime(); - dt_h5dread = dt_h5dread + (t2-t1); } /* endif ! no_read */ else { From 0bc25064ae48f3ea0323ddf8cfcfee1c98ec7167 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 09:48:33 -0500 Subject: [PATCH 09/21] removed timing and debugging --- libdispatch/dinfermodel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 21c62d71c..c2ffbf750 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -819,7 +819,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void 2. begin collecting fragments */ if((stat = processuri(path, &uri, fraglenv))) goto done; - + if(uri != NULL) { #ifdef DEBUG printlist(fraglenv,"processuri"); From f98eb86a001d8aa0a110561227bacb0de4a4d398 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 10:03:05 -0500 Subject: [PATCH 10/21] removed timing and debugging --- libdispatch/dinfermodel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index c2ffbf750..0d4ee57e7 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -779,7 +779,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void char* sfrag = NULL; const char* modeval = NULL; - /* Check for a DAOS contianer */ + /* Check for a DAOS container */ #ifdef USE_NETCDF4 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; From e768f1716b55fd52401a93f9d5bae0da96ea475e Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 10:20:20 -0500 Subject: [PATCH 11/21] ifdef around HDF5 --- libdispatch/dinfermodel.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 0d4ee57e7..a26738226 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -27,7 +27,10 @@ #ifdef ENABLE_BYTERANGE #include "nchttp.h" #endif + +#ifdef USE_NETCDF4 #include "hdf5.h" +#endif #undef DEBUG From ea5fa5d8b355c2f8068ae0559fd7ba0779d959b5 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 10:32:01 -0500 Subject: [PATCH 12/21] ifdef around HDF5 --- libdispatch/dinfermodel.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index a26738226..569f570de 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -27,10 +27,9 @@ #ifdef ENABLE_BYTERANGE #include "nchttp.h" #endif - -#ifdef USE_NETCDF4 -#include "hdf5.h" -#endif +#ifdef USE_HDF5 +#include +#endif /* USE_HDF5 */ #undef DEBUG From 68897c2f2e034faef34ccb09cbf4062773f0fadf Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 10:34:00 -0500 Subject: [PATCH 13/21] ifdef around HDF5 --- libdispatch/dinfermodel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 569f570de..da89a0201 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -7,7 +7,6 @@ * Copyright 2018 University Corporation for Atmospheric * Research/Unidata. See COPYRIGHT file for more info. */ -#include #include "config.h" #include #ifdef HAVE_UNISTD_H @@ -28,6 +27,7 @@ #include "nchttp.h" #endif #ifdef USE_HDF5 +#include #include #endif /* USE_HDF5 */ @@ -782,7 +782,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void const char* modeval = NULL; /* Check for a DAOS container */ -#ifdef USE_NETCDF4 +#ifdef USE_HDF5 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; From d44f3535bf58bb7094152fd4b3f1ab240cf5624c Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Wed, 16 Jun 2021 11:16:20 -0500 Subject: [PATCH 14/21] ifdef around HDF5 --- libdispatch/dinfermodel.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index da89a0201..ef4b33b07 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -26,10 +26,12 @@ #ifdef ENABLE_BYTERANGE #include "nchttp.h" #endif +#ifndef _WIN32 #ifdef USE_HDF5 #include #include #endif /* USE_HDF5 */ +#endif /* _WIN32 */ #undef DEBUG @@ -782,6 +784,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void const char* modeval = NULL; /* Check for a DAOS container */ +#ifndef _WIN32 #ifdef USE_HDF5 #if H5_VERSION_GE(1,12,0) hid_t fapl_id; @@ -814,6 +817,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void } if (H5Pclose(fapl_id) < 0) goto done; #endif +#endif #endif /* Phase 1: From e5973deda11cc3e33f85cbb02cf2ed9b51f13bb0 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 09:59:03 -0600 Subject: [PATCH 15/21] Merge in updated version of the file from Dennis. --- libdispatch/dinfermodel.c | 168 ++++++++++++++++++++++++++------------ 1 file changed, 114 insertions(+), 54 deletions(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index c11cd548f..a68e222d8 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -10,12 +10,21 @@ #include "config.h" #include #include +#include #ifdef HAVE_UNISTD_H #include #endif #ifdef HAVE_SYS_TYPES_H #include #endif +#ifndef _WIN32 +#ifdef USE_HDF5 +#include +#endif /* USE_HDF5 */ +#endif /* _WIN32 */ +#ifdef HAVE_SYS_XATTR_H +#include +#endif #include "ncdispatch.h" #include "ncpathmgr.h" @@ -33,12 +42,6 @@ #ifndef nulldup #define nulldup(x) ((x)?strdup(x):(x)) #endif -#ifndef _WIN32 -#ifdef USE_HDF5 -#include -#include -#endif /* USE_HDF5 */ -#endif /* _WIN32 */ #undef DEBUG @@ -234,6 +237,7 @@ static int replacemode(NClist* envv, const char* newval); static void infernext(NClist* current, NClist* next); static int negateone(const char* mode, NClist* modes); static void cleanstringlist(NClist* strs, int caseinsensitive); +static int isdaoscontainer(const char* path); /* If the path looks like a URL, then parse it, reformat it. @@ -851,45 +855,6 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void NClist* modeargs = nclistnew(); char* sfrag = NULL; const char* modeval = NULL; - - /* Check for a DAOS container */ -#ifndef _WIN32 -#ifdef USE_HDF5 -#if H5_VERSION_GE(1,12,0) - hid_t fapl_id; - if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto done; - H5Pset_fapl_sec2(fapl_id); - - htri_t accessible; - accessible = H5Fis_accessible(path, fapl_id); - if(accessible > 0) { - int rc=0; - FILE *fp; - char *cmd; - cmd = (char*)malloc((strlen(path)+28)*sizeof(char)); - strcpy(cmd, "getfattr "); - strcat(cmd, path); - strcat(cmd, " | grep -c '.daos'"); - - if((fp = popen(cmd, "r")) != NULL) { - fscanf(fp, "%d", &rc); - pclose(fp); - } - free(cmd); - if(rc == 1) { - /* Is a DAOS container */ - model->impl = NC_FORMATX_NC4; - model->format = NC_FORMAT_NETCDF4; - if (H5Pclose(fapl_id) < 0) goto done; - goto done; - } - } - if (H5Pclose(fapl_id) < 0) goto done; -#endif -#endif -#endif - - char* abspath = NULL; NClist* tmp = NULL; @@ -1011,15 +976,22 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void if((stat = NC_omodeinfer(useparallel,omode,model))) goto done; } - /* Phase 9: Infer from file content, if possible; - this has highest precedence, so it may override - previous decisions. Note that we do this last - because we need previously determined model info - to guess if this file is readable. - */ - if(!iscreate && isreadable(uri,model)) { - /* Ok, we need to try to read the file */ - if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done; + /* Phase 9: Special case for file stored in DAOS container */ + if(isdaoscontainer(path) == NC_NOERR) { + /* This is a DAOS container, so immediately assume it is HDF5. */ + model->impl = NC_FORMATX_NC_HDF5; + model->format = NC_FORMAT_NETCDF4; + } else { + /* Phase 10: Infer from file content, if possible; + this has highest precedence, so it may override + previous decisions. Note that we do this last + because we need previously determined model info + to guess if this file is readable. + */ + if(!iscreate && isreadable(uri,model)) { + /* Ok, we need to try to read the file */ + if((stat = check_file_type(path, omode, useparallel, params, model, uri))) goto done; + } } /* Need a decision */ @@ -1610,6 +1582,94 @@ done: return check(status); } +/* Return NC_NOERR if path is a DAOS container; return NC_EXXX otherwise */ +static int +isdaoscontainer(const char* path) +{ + int stat = NC_ENOTNC; /* default is that this is not a DAOS container */ +#ifndef _WIN32 +#ifdef USE_HDF5 +#if H5_VERSION_GE(1,12,0) + htri_t accessible; + hid_t fapl_id; + int rc; + /* Check for a DAOS container */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) {stat = NC_EHDFERR; goto done;} + H5Pset_fapl_sec2(fapl_id); + accessible = H5Fis_accessible(path, fapl_id); + H5Pclose(fapl_id); /* Ignore any error */ + rc = 0; + if(accessible > 0) { +#ifdef HAVE_SYS_XATTR_H + ssize_t xlen; +#ifdef __APPLE__ + xlen = listxattr(path, NULL, 0, 0); +#else + xlen = listxattr(path, NULL, 0); +#endif + if(xlen > 0) { + char* xlist = NULL; + char* xvalue = NULL; + char* p; + char* endp; + if((xlist = (char*)calloc(1,(size_t)xlen))==NULL) + {stat = NC_ENOMEM; goto done;} +#ifdef __APPLE__ + (void)listxattr(path, xlist, (size_t)xlen, 0); /* Get xattr names */ +#else + (void)listxattr(path, xlist, (size_t)xlen); /* Get xattr names */ +#endif + p = xlist; endp = p + xlen; /* delimit names */ + /* walk the list of xattr names */ + for(;p < endp;p += (strlen(p)+1)) { + /* The popen version looks for the string ".daos"; + It would be nice if we know whether that occurred + int the xattr's name or it value. + Oh well, we will do the general search */ + /* Look for '.daos' in the key */ + if(strstr(p,".daos") != NULL) {rc = 1; break;} /* success */ + /* Else get the p'th xattr's value size */ +#ifdef __APPLE__ + xlen = getxattr(path, p, NULL, 0, 0, 0); +#else + xlen = getxattr(path, p, NULL, 0); +#endif + if((xvalue = (char*)calloc(1,(size_t)xlen))==NULL) + {stat = NC_ENOMEM; goto done;} + /* Read the value */ +#ifdef __APPLE__ + (void)getxattr(path, p, xvalue, (size_t)xlen, 0, 0); +#else + (void)getxattr(path, p, xvalue, (size_t)xlen); +#endif +fprintf(stderr,"@@@ %s=|%s|\n",p,xvalue); + /* Look for '.daos' in the value */ + if(strstr(xvalue,".daos") != NULL) {rc = 1; break;} /* success */ + } + } +#else /*!HAVE_SYS_XATTR_H*/ + { + FILE *fp; + char cmd[4096]; + memset(cmd,0,sizeof(cmd)); + snprintf(cmd,sizeof(cmd),"getfattr %s | grep -c '.daos'",path); + if((fp = popen(cmd, "r")) != NULL) { + fscanf(fp, "%d", &rc); + pclose(fp); + } + } +#endif /*HAVE_SYS_XATTR_H*/ + } + /* Test for DAOS container */ + stat = (rc == 1 ? NC_NOERR : NC_ENOTNC); +done: +#endif +#endif +#endif + errno = 0; /* reset */ + return stat; +} + #ifdef DEBUG static void printmagic(const char* tag, char* magic, struct MagicFile* f) From 85e5d1137432bcea5b589d60b9bbb9d91537965a Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 11:00:59 -0600 Subject: [PATCH 16/21] Break cygwin build and run tests into two different steps to try to diagnose the timeout. --- .github/workflows/run_tests_win_cygwin.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 887a64ebb..6534400f5 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -78,10 +78,15 @@ jobs: if [ $(find /tmp/pretend-root/$(pwd) -type f | wc -l) -gt 0 ]; then exit 1; fi fi - - name: (Autotools) Build and run tests + - name: (Autotools) Build tests timeout-minutes: 30 run: | - make check -j8 SHELL=/bin/dash + make check -j 8 TESTS="" SHELL=/bin/dash + + - name: (Autotools) Run tests + timeout-minutes: 30 + run: | + make check -j 8 SHELL=/bin/dash build-and-test-cmake: name: Cygwin-based CMake tests From 15c2e25f73215e1364b4258090f51daef836e83b Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 11:03:39 -0600 Subject: [PATCH 17/21] Taking an additional stab by modifying parallelism. --- .github/workflows/run_tests_win_cygwin.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 6534400f5..8bf67af00 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -81,12 +81,12 @@ jobs: - name: (Autotools) Build tests timeout-minutes: 30 run: | - make check -j 8 TESTS="" SHELL=/bin/dash + make check -j$(nproc) TESTS="" SHELL=/bin/dash - name: (Autotools) Run tests timeout-minutes: 30 run: | - make check -j 8 SHELL=/bin/dash + make check -j$(nproc) SHELL=/bin/dash build-and-test-cmake: name: Cygwin-based CMake tests From 5f89cd0c10874de6fee9631d621a1b5ce2263b7e Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 12:05:49 -0600 Subject: [PATCH 18/21] Fix a hang in cygwin based tests on Windows. Add check for sys/xattr.h so it can be used when available. Added installation of a tool required for cygwin. --- .github/workflows/run_tests_win_cygwin.yml | 2 +- CMakeLists.txt | 1 + config.h.cmake.in | 3 +++ configure.ac | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 8bf67af00..463177097 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -41,7 +41,7 @@ jobs: libhdf4-devel zipinfo libxml2-devel perl zlib-devel libzstd-devel libbz2-devel libaec-devel libzip-devel libdeflate-devel gcc-core gcc-g++ libcurl-devel libiconv-devel - libssl-devel libcrypt-devel + libssl-devel libcrypt-devel attr libattr-devel - name: (Autotools) Run autoconf and friends run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d68f3219..a9319e323 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1151,6 +1151,7 @@ CHECK_INCLUDE_file("io.h" HAVE_IO_H) endif(MSVC) CHECK_INCLUDE_file("stdlib.h" HAVE_STDLIB_H) CHECK_INCLUDE_file("ctype.h" HAVE_CTYPE_H) +CHECK_INCLUDE_file("sys/xattr_h" HAVE_SYS_XATTR_H) CHECK_INCLUDE_file("stdarg.h" HAVE_STDARG_H) CHECK_INCLUDE_file("strings.h" HAVE_STRINGS_H) CHECK_INCLUDE_file("signal.h" HAVE_SIGNAL_H) diff --git a/config.h.cmake.in b/config.h.cmake.in index a38749a92..4193b071a 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -373,6 +373,9 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_CTYPE_H 1 +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_XATTR_H + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_STRINGS_H 1 diff --git a/configure.ac b/configure.ac index 61a456390..13ca293bd 100644 --- a/configure.ac +++ b/configure.ac @@ -1292,7 +1292,7 @@ AC_CHECK_HEADERS([sys/param.h]) AC_CHECK_HEADERS([libgen.h]) #AC_CHECK_HEADERS([locale.h]) AC_HEADER_STDC -AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h]) +AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h sys/xattr_h]) # Do sys/resource.h separately #AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0]) From 848e85171a4786e585e31c9bc03413b6941fcbb3 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 12:19:16 -0600 Subject: [PATCH 19/21] Mitigate extreme slowdown on systems without xattr.h or getfattr. Standard output was being spammed, resulting in a test that currently runs in around a minute to balloon out to almost 20. --- CMakeLists.txt | 5 +++++ config.h.cmake.in | 3 +++ configure.ac | 6 ++++++ libdispatch/dinfermodel.c | 16 +++++++++++----- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9319e323..891088789 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -87,6 +87,11 @@ if(UNAME) set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}") endif() +find_program(GETFATTR NAMES getfattr) +if(GETFATTR) + set(HAVE_GETFATTR TRUE) +endif() + # Define some Platforms if(osname MATCHES "CYGWIN.*") set(ISCYGWIN yes) diff --git a/config.h.cmake.in b/config.h.cmake.in index 4193b071a..fe7dabcdf 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -373,6 +373,9 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_CTYPE_H 1 +/* Define to 1 if you have the getfattr command line utility. */ +#cmakedefine HAVE_GETFATTR 1 + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_SYS_XATTR_H diff --git a/configure.ac b/configure.ac index 13ca293bd..4f1602450 100644 --- a/configure.ac +++ b/configure.ac @@ -82,6 +82,12 @@ AC_LANG_POP([C]) if test $have_no_strict_aliasing = no; then CFLAGS=$SAVE_CFLAGS fi + +## +# Check to see if we have getfattr +## +AC_CHECK_PROGS([HAVE_GETFATTR], [getfattr]) + ## # Some files need to exist in build directories # that do not correspond to their source directory, or diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index a68e222d8..a381ee008 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -1648,16 +1648,22 @@ fprintf(stderr,"@@@ %s=|%s|\n",p,xvalue); } } #else /*!HAVE_SYS_XATTR_H*/ + +#ifdef HAVE_GETFATTR { FILE *fp; char cmd[4096]; memset(cmd,0,sizeof(cmd)); - snprintf(cmd,sizeof(cmd),"getfattr %s | grep -c '.daos'",path); - if((fp = popen(cmd, "r")) != NULL) { - fscanf(fp, "%d", &rc); - pclose(fp); + snprintf(cmd,sizeof(cmd),"getfattr %s | grep -c '.daos'",path); + if((fp = popen(cmd, "r")) != NULL) { + fscanf(fp, "%d", &rc); + pclose(fp); } - } + } +#else /*!HAVE_GETFATTR*/ + /* We just can't test for DAOS container.*/ + stat = 0; +#endif /*HAVE_GETFATTR*/ #endif /*HAVE_SYS_XATTR_H*/ } /* Test for DAOS container */ From 9012c58edbcbc0662a9ea566d87d7bd2db25be6c Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 12:36:51 -0600 Subject: [PATCH 20/21] Fix check for getfattr in configure.ac --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4f1602450..aa380919a 100644 --- a/configure.ac +++ b/configure.ac @@ -87,7 +87,9 @@ fi # Check to see if we have getfattr ## AC_CHECK_PROGS([HAVE_GETFATTR], [getfattr]) - +if test -n $HAVE_GETFATTR; then + AC_DEFINE_UNQUOTED([HAVE_GETFATTR],[1],[getfattr is available]) +fi ## # Some files need to exist in build directories # that do not correspond to their source directory, or From b67745eb0ba003de3aaf9d7f70d933bf74a79585 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 16 Jul 2024 12:42:10 -0600 Subject: [PATCH 21/21] Correct the check. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index aa380919a..d223c19f8 100644 --- a/configure.ac +++ b/configure.ac @@ -87,7 +87,7 @@ fi # Check to see if we have getfattr ## AC_CHECK_PROGS([HAVE_GETFATTR], [getfattr]) -if test -n $HAVE_GETFATTR; then +if test -n "$HAVE_GETFATTR"; then AC_DEFINE_UNQUOTED([HAVE_GETFATTR],[1],[getfattr is available]) fi ##