From 739dfa2aa2127cea842a90a75a69f68aa41c34a1 Mon Sep 17 00:00:00 2001 From: Wei-keng Liao Date: Sun, 15 Jul 2018 16:19:21 -0500 Subject: [PATCH] PnetCDF does not support per-variable collective/independent data mode change; modify nc_var_par_access to ignore varid --- docs/tutorial.dox | 6 +++- examples/C/parallel_vara.c | 4 +-- libsrcp/ncpdispatch.c | 71 ++++++++++++++++++++++++++++++-------- nc_test/tst_parallel2.c | 5 +++ nc_test/tst_pnetcdf.c | 6 ++-- nc_test/tst_small.c | 6 ++-- nc_test/util.c | 10 ++---- 7 files changed, 74 insertions(+), 34 deletions(-) diff --git a/docs/tutorial.dox b/docs/tutorial.dox index 640204b4f..3d3287353 100644 --- a/docs/tutorial.dox +++ b/docs/tutorial.dox @@ -809,7 +809,11 @@ create/open a netCDF file with parallel access. Parallel file access is either collective (all processors must participate) or independent (any processor may access the data without -waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. To make writes to a variable collective, call nc_var_par_access(). +waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (ex. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent (the default) or collective. +To change from collective to independent mode or vis versa, call +nc_var_par_access() with argument 'access' set to either NC_INDEPENDENT or +NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid' is ignored, as +PnetCDF does not support per-variable collective/independent mode change. \page tutorial_ncids Numbering of NetCDF IDs diff --git a/examples/C/parallel_vara.c b/examples/C/parallel_vara.c index 4aa789c81..757d91769 100644 --- a/examples/C/parallel_vara.c +++ b/examples/C/parallel_vara.c @@ -150,7 +150,7 @@ int main(int argc, char** argv) err = nc_enddef(ncid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* now we are in data mode */ start[0] = 0; @@ -176,7 +176,7 @@ int main(int argc, char** argv) err = nc_inq_varid(ncid, "var", &varid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* each process reads its subarray from the file */ err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index d651abc4f..9e561057c 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -9,10 +9,14 @@ #include #include "nc.h" #include "ncdispatch.h" +#include "fbits.h" /* Must follow netcdf.h */ #include +#define NCP_MODE_DATA 0x0001 +#define NCP_MODE_INDEP 0x0002 + typedef struct NCP_INFO { /* pnetcdf_file will be true if the file is created/opened with the @@ -104,6 +108,9 @@ NCP_create(const char *path, int cmode, res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); + /* Default to independent access, like netCDF-4/HDF5 files. */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ done: return res; @@ -152,6 +159,9 @@ NCP_open(const char *path, int cmode, nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); if(nc5 == NULL) {res = NC_ENOMEM; goto done;} + /* file open automatically enters data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); @@ -160,7 +170,7 @@ NCP_open(const char *path, int cmode, /* Default to independent access, like netCDF-4/HDF5 files. */ if(!res) { res = ncmpi_begin_indep_data(nc->int_ncid); - nc5->pnetcdf_access_mode = NC_INDEPENDENT; + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); } done: return res; @@ -170,8 +180,14 @@ static int NCP_redef(int ncid) { NC* nc; + NCP_INFO* nc5; + int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; + + nc5 = NCP_DATA(nc); + fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + return ncmpi_redef(nc->int_ncid); } @@ -203,7 +219,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t #endif if(!status) { - if (nc5->pnetcdf_access_mode == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) status = ncmpi_begin_indep_data(nc->int_ncid); } return status; @@ -611,7 +628,7 @@ NCP_get_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -705,7 +722,7 @@ NCP_put_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -801,7 +818,7 @@ NCP_get_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -897,7 +914,7 @@ NCP_put_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -995,7 +1012,7 @@ NCP_get_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1093,7 +1110,7 @@ NCP_put_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1198,19 +1215,43 @@ NCP_var_par_access(int ncid, int varid, int par_access) if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE) return NC_EINVAL; +#ifdef _DO_NOT_IGNORE_VARID_ + if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */ + return NC_EINVAL; +#endif + status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; nc5 = NCP_DATA(nc); assert(nc5); - if(par_access == nc5->pnetcdf_access_mode) - return NC_NOERR; - nc5->pnetcdf_access_mode = par_access; - if (par_access == NC_INDEPENDENT) - return ncmpi_begin_indep_data(nc->int_ncid); - else - return ncmpi_end_indep_data(nc->int_ncid); + /* if currently in data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) { + if (par_access == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) + return NC_NOERR; + else { /* currently in collective data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_begin_indep_data(nc->int_ncid); + } + } + else { /* want to enter collective data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_end_indep_data(nc->int_ncid); + } + else + return NC_NOERR; + } + } + else { /* currently in define mode */ + if (par_access == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + else + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + } + return NC_NOERR; } #ifdef USE_NETCDF4 diff --git a/nc_test/tst_parallel2.c b/nc_test/tst_parallel2.c index 92c271d26..45f61a4bc 100644 --- a/nc_test/tst_parallel2.c +++ b/nc_test/tst_parallel2.c @@ -135,8 +135,13 @@ main(int argc, char **argv) sleep(mpi_rank); #endif /* USE_MPE */ +#ifdef USE_PNETCDF +/* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/ + if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR; +#else /* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/ if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR; +#endif if (!mpi_rank) start_time = MPI_Wtime(); diff --git a/nc_test/tst_pnetcdf.c b/nc_test/tst_pnetcdf.c index 812ab5dfd..bcf8d21d5 100644 --- a/nc_test/tst_pnetcdf.c +++ b/nc_test/tst_pnetcdf.c @@ -68,10 +68,8 @@ int main(int argc, char* argv[]) } if (nc_enddef(ncid)) ERR; - for (i=0; i