From 2cd228bcd4dbf611f8df5243550a0deaa55f9fc9 Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Mon, 16 Sep 2019 11:28:18 -0600 Subject: [PATCH 1/3] porting changes from other PR --- include/nc.h | 1 + include/nc4internal.h | 1 + libdispatch/nclistmgr.c | 29 +++++++++++++++++++++ libsrc4/nc4internal.c | 56 +++++++++++++++++++++++++---------------- 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/include/nc.h b/include/nc.h index de6e0482d..c9be6e751 100644 --- a/include/nc.h +++ b/include/nc.h @@ -72,6 +72,7 @@ extern int add_to_NCList(NC*); extern void del_from_NCList(NC*);/* does not free object */ extern NC* find_in_NCList(int ext_ncid); extern NC* find_in_NCList_by_name(const char*); +extern int move_in_NCList(NC *ncp, int new_id); extern void free_NCList(void);/* reclaim whole list */ extern int count_NCList(void); /* return # of entries in NClist */ extern int iterate_NCList(int i,NC**); /* Walk from 0 ...; ERANGE return => stop */ diff --git a/include/nc4internal.h b/include/nc4internal.h index b4e07c7d3..3132c09ea 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -388,6 +388,7 @@ int nc4_file_list_add(int ncid, const char *path, int mode, int nc4_file_list_get(int ncid, char **path, int *mode, void **dispatchdata); int nc4_file_list_del(int ncid); +int nc4_file_change_ncid(int ncid, unsigned short new_ncid_index); int nc4_var_list_add(NC_GRP_INFO_T* grp, const char* name, int ndims, NC_VAR_INFO_T **var); int nc4_var_list_add2(NC_GRP_INFO_T* grp, const char* name, diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index e05b7a3a4..5506faca9 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -100,6 +100,35 @@ add_to_NCList(NC* ncp) return NC_NOERR; } +/** + * Move an NC in the nc_filelist. This is required by PIO. + * + * @param ncp Pointer to already-allocated and initialized NC struct. + * @param new_id New index in the nc_filelist for this file. + * + * @return ::NC_NOERR No error. + * @return ::NC_EINVAL Invalid input. + * @author Ed Hartnett + */ +int +move_in_NCList(NC *ncp, int new_id) +{ + /* If no files in list, error. */ + if (!nc_filelist) + return NC_EINVAL; + + /* If new slot is already taken, error. */ + if (nc_filelist[new_id]) + return NC_EINVAL; + + /* Move the file. */ + nc_filelist[ncp->ext_ncid >> ID_SHIFT] = NULL; + nc_filelist[new_id] = ncp; + ncp->ext_ncid = (new_id << ID_SHIFT); + + return NC_NOERR; +} + /** * Delete an NC struct from the list. This happens when the file is * closed. Relies on all memory in the NC being deallocated after this diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index ad4d3a70f..f396ef061 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -125,30 +125,42 @@ nc4_file_list_add(int ncid, const char *path, int mode, void **dispatchdata) return NC_NOERR; } -/* /\** */ -/* * @internal Change the ncid of an open file. This is needed for PIO */ -/* * integration. */ -/* * */ -/* * @param ncid The ncid of the file (aka ext_ncid). */ -/* * @param new_ncid The new ncid to use. */ -/* * */ -/* * @return ::NC_NOERR No error. */ -/* * @return ::NC_EBADID No NC struct with this ext_ncid. */ -/* * @return ::NC_ENOMEM Out of memory. */ -/* * @author Ed Hartnett */ -/* *\/ */ -/* int */ -/* nc4_file_change_ncid(int ncid, int new_ncid) */ -/* { */ -/* NC *nc; */ -/* int ret; */ +/** + * @internal Change the ncid of an open file. This is needed for PIO + * integration. + * + * @param ncid The ncid of the file (aka ext_ncid). + * @param new_ncid The new ncid index to use (i.e. the first two bytes + * of the ncid). + * + * @return ::NC_NOERR No error. + * @return ::NC_EBADID No NC struct with this ext_ncid. + * @return ::NC_ENOMEM Out of memory. + * @author Ed Hartnett + */ +int +nc4_file_change_ncid(int ncid, unsigned short new_ncid_index) +{ + NC *nc; + int ret; -/* /\* Find NC pointer for this file. *\/ */ -/* if ((ret = NC_check_id(ncid, &nc))) */ -/* return ret; */ + LOG((2, "%s: ncid %d new_ncid_index %d", __func__, ncid, new_ncid_index)); -/* return NC_NOERR; */ -/* } */ + /* Find NC pointer for this file. */ + if ((ret = NC_check_id(ncid, &nc))) + return ret; + + /* Move it in the list. It will faile if list spot is already + * occupied. */ + LOG((3, "moving nc->ext_ncid %d nc->ext_ncid >> ID_SHIFT %d", + nc->ext_ncid, nc->ext_ncid >> ID_SHIFT)); + if (move_in_NCList(nc, new_ncid_index)) + return NC_EIO; + LOG((3, "moved to new_ncid_index %d new nc->ext_ncid %d", new_ncid_index, + nc->ext_ncid)); + + return NC_NOERR; +} /** * @internal Get info about a file on the list of libsrc4 open From c296a3deb8b638d63ca9fab7a4fd1c4f237bfee6 Mon Sep 17 00:00:00 2001 From: edwardhartnett Date: Mon, 16 Sep 2019 11:38:48 -0600 Subject: [PATCH 2/3] porting tests --- unit_test/tst_nc4internal.c | 51 +++++++++++++++++++------------------ unit_test/tst_nclist.c | 31 ++++++++++++++++++++++ 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/unit_test/tst_nc4internal.c b/unit_test/tst_nc4internal.c index 323a47989..4fbca4849 100644 --- a/unit_test/tst_nc4internal.c +++ b/unit_test/tst_nc4internal.c @@ -265,36 +265,37 @@ main(int argc, char **argv) free_NC(ncp); } SUMMARIZE_ERR; - /* printf("Testing changing ncid..."); */ - /* { */ - /* NC *ncp, *ncp2; */ - /* int mode = 0; */ - /* NCmodel model; */ - /* int ret; */ + printf("Testing changing ncid..."); + { + NC *ncp; + NCmodel model; + NC_GRP_INFO_T *grp; + NC_FILE_INFO_T *h5; + int old_ncid; - /* /\* Create the NC* instance and insert its dispatcher and model. *\/ */ - /* if ((ret = new_NC(NULL, FILE_NAME, mode, &model, &ncp))) ERR; */ + /* Create the NC, add it to nc_filelist array, add and init + * NC_FILE_INFO_T. */ + if (new_NC(NC3_dispatch_table, FILE_NAME, 0, &model, &ncp)) ERR; + add_to_NCList(ncp); + if (nc4_file_list_add(ncp->ext_ncid, FILE_NAME, 0, NULL)) ERR; + if (nc4_find_nc_grp_h5(ncp->ext_ncid, NULL, &grp, &h5)) ERR; - /* /\* Add to list of known open files and define ext_ncid. *\/ */ - /* add_to_NCList(ncp); */ + /* Change the ncid. */ + old_ncid = ncp->ext_ncid; + if (nc4_file_change_ncid(ncp->ext_ncid, TEST_VAL_42)) ERR; - /* /\* Find it in the list. *\/ */ - /* if (!(ncp2 = find_in_NCList(ncp->ext_ncid))) ERR; */ - /* if (!(ncp2 = find_in_NCList_by_name(FILE_NAME))) ERR; */ - /* if ((ret = iterate_NCList(1, &ncp2))) ERR; */ - /* if (count_NCList() != 1) ERR; */ + /* Can't find old ncid. */ + if (nc4_find_nc_grp_h5(old_ncid, NULL, NULL, NULL) != NC_EBADID) ERR; - /* /\* Change the ncid. *\/ */ - /* if (nc4_file_change_ncid(ncp->ext_ncid, TEST_VAL_42)) ERR; */ + /* Delete it. */ + if (nc4_file_list_del(ncp->ext_ncid)) ERR; + del_from_NCList(ncp); /* Will free empty list. */ + free_NC(ncp); - /* /\* Delete it. *\/ */ - /* del_from_NCList(ncp); /\* Will free empty list. *\/ */ - /* free_NC(ncp); */ + /* Ensure it is no longer in list. */ + /* if (find_in_NCList(ncp->ext_ncid)) ERR; */ - /* /\* Ensure it is no longer in list. *\/ */ - /* /\* if (find_in_NCList(ncp->ext_ncid)) ERR; *\/ */ - - /* } */ - /* SUMMARIZE_ERR; */ + } + SUMMARIZE_ERR; FINAL_RESULTS; } diff --git a/unit_test/tst_nclist.c b/unit_test/tst_nclist.c index 0daae16a9..cdb56a9c5 100644 --- a/unit_test/tst_nclist.c +++ b/unit_test/tst_nclist.c @@ -77,6 +77,37 @@ main(int argc, char **argv) if (find_in_NCList(ncid)) ERR; } SUMMARIZE_ERR; + printf("Testing moving in NC list (needed for PIO)..."); + { + int ncid; + NC *ncp, *ncp2; + int mode = 0; + NCmodel model; + int ret; + + /* Create the NC* instance and add it to list. */ + if ((ret = new_NC(NULL, FILE_NAME, mode, &model, &ncp))) ERR; + add_to_NCList(ncp); + + /* Find it in the list. */ + if (!(ncp2 = find_in_NCList(ncp->ext_ncid))) ERR; + + /* Move it. */ + ncid = ncp->ext_ncid; + if (move_in_NCList(ncp, TEST_VAL_42)) ERR; + + /* Now we won't find old ncid in the list. */ + if (find_in_NCList(ncid)) ERR; + + /* Delete it. */ + ncid = ncp->ext_ncid; + del_from_NCList(ncp); /* Will free empty list. */ + free_NC(ncp); + + /* Ensure it is no longer in list. */ + if (find_in_NCList(ncid)) ERR; + } + SUMMARIZE_ERR; #ifdef LARGE_FILE_TESTS /* This test is slow, only run it on large file test builds. */ printf("Testing maxing out NC list..."); From 1d42cab572d8699a0ab4ee6c7fec96d551cb0845 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 20 Sep 2019 14:12:15 -0600 Subject: [PATCH 3/3] Updated Release Notes. --- RELEASE_NOTES.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index d627979bd..f2357704f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.7.2 - TBD +* [Enhancement] Added a function for changing the ncid of an open file. This function should only be used if you know what you are doing, and is meant to be used primarily with PIO integration. See [GitHub #1483](https://github.com/Unidata/netcdf-c/pull/1483) and [GitHub #1487](https://github.com/Unidata/netcdf-c/pull/1487) for more information. + ## 4.7.1 - August 27, 2019 * [Enhancement] Added unit_test directory, which contains unit tests for the libdispatch and libsrc4 code (and any other directories that want to put unit tests there). Use --disable-unit-tests to run without unit tests (ex. for code coverage analysis). See [GitHub #1458](https://github.com/Unidata/netcdf-c/issues/1458). @@ -26,7 +28,7 @@ read-only access to, for example, Amazon S3 objects and also Thredds Server datasets via the HTTPService access method. See [GitHub #1251](https://github.com/Unidata/netcdf-c/issues/1251). -* Update the license from the home-brewed NetCDF license to the standard 3-Clause BSD License. This change does not result in any new restrictions; it is merely the adoption of a standard, well-known and well-understood license in place of the historic NetCDF license written at Unidata. This is part of a broader push by Unidata to adopt modern, standardized licensing. +* Update the license from the home-brewed NetCDF license to the standard 3-Clause BSD License. This change does not result in any new restrictions; it is merely the adoption of a standard, well-known and well-understood license in place of the historic NetCDF license written at Unidata. This is part of a broader push by Unidata to adopt modern, standardized licensing. ## 4.6.3 - February 28, 2019 @@ -35,7 +37,7 @@ See [GitHub #1251](https://github.com/Unidata/netcdf-c/issues/1251). * Some fixes for rename, including fix for renumbering of varids after a rename (#1307), renaming var to dim without coordinate var. See [Github #1297](https://github.com/Unidata/netcdf-c/issues/1297). * Fix of NULL parameter causing segfaults in put_vars functions. See [Github #1265](https://github.com/Unidata/netcdf-c/issues/1265) for more information. * Fix of --enable-benchmark benchmark tests [Github #1211](https://github.com/Unidata/netcdf-c/issues/1211) -* Update the license from the home-brewed NetCDF license to the standard 3-Clause BSD License. This change does not result in any new restrictions; it is merely the adoption of a standard, well-known and well-understood license in place of the historic NetCDF license written at Unidata. This is part of a broader push by Unidata to adopt modern, standardized licensing. +* Update the license from the home-brewed NetCDF license to the standard 3-Clause BSD License. This change does not result in any new restrictions; it is merely the adoption of a standard, well-known and well-understood license in place of the historic NetCDF license written at Unidata. This is part of a broader push by Unidata to adopt modern, standardized licensing. * [BugFix] Corrected DAP-releated issues on big-endian machines. See [Github #1321](https://github.com/Unidata/netcdf-c/issues/1321), [Github #1302](https://github.com/Unidata/netcdf-c/issues/1302) for more information. * [BugFix][Enhancement] Various and sundry bugfixes and performance enhancements, thanks to \@edhartnett, \@gsjaardema, \@t-b, \@wkliao, and all of our other contributors. * [Enhancement] Extended `nccopy -F` syntax to support multiple variables with a single invocation. See [Github #1311](https://github.com/Unidata/netcdf-c/issues/1311) for more information.