diff --git a/docs/tutorial.dox b/docs/tutorial.dox index 18536667c..bcd8c6a49 100644 --- a/docs/tutorial.dox +++ b/docs/tutorial.dox @@ -815,8 +815,11 @@ independent (any processor may access the data without waiting for others). All netCDF metadata writing operations are collective. That is, all creation of groups, types, variables, dimensions, or attributes. Data reads and writes (e.g. calls to nc_put_vara_int() and nc_get_vara_int()) may be independent, the -default) or collective. To make writes to a variable collective, call -nc_var_par_access(). +default) or collective. To change from collective to independent mode or vis +versa, call nc_var_par_access() with argument 'access' set to either +NC_INDEPENDENT or NC_COLLECTIVE. Note when using PnetCDF, the argument 'varid' +is ignored, as PnetCDF does not support per-variable collective/independent +mode change. \page tutorial_ncids Numbering of NetCDF IDs diff --git a/examples/C/parallel_vara.c b/examples/C/parallel_vara.c index 4aa789c81..757d91769 100644 --- a/examples/C/parallel_vara.c +++ b/examples/C/parallel_vara.c @@ -150,7 +150,7 @@ int main(int argc, char** argv) err = nc_enddef(ncid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* now we are in data mode */ start[0] = 0; @@ -176,7 +176,7 @@ int main(int argc, char** argv) err = nc_inq_varid(ncid, "var", &varid); ERR /* set to use MPI/PnetCDF collective I/O */ - err = nc_var_par_access(ncid, varid, NC_COLLECTIVE); ERR + err = nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE); ERR /* each process reads its subarray from the file */ err = nc_get_vara_int(ncid, varid, start, count, &buf[0][0]); ERR diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index cc5e8eea1..1005db6f4 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -9,10 +9,14 @@ #include #include "nc.h" #include "ncdispatch.h" +#include "fbits.h" /* Must follow netcdf.h */ #include +#define NCP_MODE_DATA 0x0001 +#define NCP_MODE_INDEP 0x0002 + typedef struct NCP_INFO { /* pnetcdf_access_mode keeps track of whether independpent or collective @@ -85,7 +89,12 @@ NCP_create(const char *path, int cmode, res = ncmpi_create(comm, path, cmode, info, &(nc->int_ncid)); - if (res != NC_NOERR) free(nc5); /* reclaim allocated space */ + + /* Default to independent access, like netCDF-4/HDF5 files. */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + + if(res && nc5 != NULL) free(nc5); /* reclaim allocated space */ + done: return res; } @@ -116,6 +125,9 @@ NCP_open(const char *path, int cmode, nc5 = (NCP_INFO*)calloc(1,sizeof(NCP_INFO)); if(nc5 == NULL) {res = NC_ENOMEM; goto done;} + /* file open automatically enters data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + /* Link nc5 and nc */ NCP_DATA_SET(nc,nc5); @@ -124,7 +136,7 @@ NCP_open(const char *path, int cmode, /* Default to independent access, like netCDF-4/HDF5 files. */ if (res == NC_NOERR) { res = ncmpi_begin_indep_data(nc->int_ncid); - nc5->pnetcdf_access_mode = NC_INDEPENDENT; + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); } done: return res; @@ -134,8 +146,14 @@ static int NCP_redef(int ncid) { NC* nc; + NCP_INFO* nc5; + int status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; + + nc5 = NCP_DATA(nc); + fClr(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + return ncmpi_redef(nc->int_ncid); } @@ -166,7 +184,8 @@ NCP__enddef(int ncid, size_t h_minfree, size_t v_align, size_t v_minfree, size_t #endif if(!status) { - if (nc5->pnetcdf_access_mode == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA); + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) status = ncmpi_begin_indep_data(nc->int_ncid); } return status; @@ -578,7 +597,7 @@ NCP_get_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -672,7 +691,7 @@ NCP_put_vara(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vara_schar(nc->int_ncid, varid, mpi_start, mpi_count, ip); break; @@ -768,7 +787,7 @@ NCP_get_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -864,7 +883,7 @@ NCP_put_vars(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_vars_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, ip); break; @@ -962,7 +981,7 @@ NCP_get_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status=ncmpi_get_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1060,7 +1079,7 @@ NCP_put_varm(int ncid, if (status) return status; } - if(nc5->pnetcdf_access_mode == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { switch(memtype) { case NC_BYTE: status = ncmpi_put_varm_schar(nc->int_ncid, varid, mpi_start, mpi_count, mpi_stride, mpi_imap, ip); break; @@ -1176,19 +1195,43 @@ NCP_var_par_access(int ncid, int varid, int par_access) if (par_access != NC_INDEPENDENT && par_access != NC_COLLECTIVE) return NC_EINVAL; +#ifdef _DO_NOT_IGNORE_VARID_ + if (varid != NC_GLOBAL) /* PnetCDF cannot do per-veriable mode change */ + return NC_EINVAL; +#endif + status = NC_check_id(ncid, &nc); if(status != NC_NOERR) return status; nc5 = NCP_DATA(nc); assert(nc5); - if(par_access == nc5->pnetcdf_access_mode) - return NC_NOERR; - nc5->pnetcdf_access_mode = par_access; - if (par_access == NC_INDEPENDENT) - return ncmpi_begin_indep_data(nc->int_ncid); - else - return ncmpi_end_indep_data(nc->int_ncid); + /* if currently in data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_DATA)) { + if (par_access == NC_INDEPENDENT) { + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) + return NC_NOERR; + else { /* currently in collective data mode */ + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_begin_indep_data(nc->int_ncid); + } + } + else { /* want to enter collective data mode */ + if (fIsSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP)) { + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + return ncmpi_end_indep_data(nc->int_ncid); + } + else + return NC_NOERR; + } + } + else { /* currently in define mode */ + if (par_access == NC_INDEPENDENT) + fSet(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + else + fClr(nc5->pnetcdf_access_mode, NCP_MODE_INDEP); + } + return NC_NOERR; } #ifdef USE_NETCDF4 diff --git a/nc_test/tst_parallel2.c b/nc_test/tst_parallel2.c index 92c271d26..45f61a4bc 100644 --- a/nc_test/tst_parallel2.c +++ b/nc_test/tst_parallel2.c @@ -135,8 +135,13 @@ main(int argc, char **argv) sleep(mpi_rank); #endif /* USE_MPE */ +#ifdef USE_PNETCDF +/* if (nc_var_par_access(ncid, NC_GLOBAL, NC_COLLECTIVE)) ERR;*/ + if (nc_var_par_access(ncid, NC_GLOBAL, NC_INDEPENDENT)) ERR; +#else /* if (nc_var_par_access(ncid, varid, NC_COLLECTIVE)) ERR;*/ if (nc_var_par_access(ncid, varid, NC_INDEPENDENT)) ERR; +#endif if (!mpi_rank) start_time = MPI_Wtime(); diff --git a/nc_test/tst_pnetcdf.c b/nc_test/tst_pnetcdf.c index 812ab5dfd..bcf8d21d5 100644 --- a/nc_test/tst_pnetcdf.c +++ b/nc_test/tst_pnetcdf.c @@ -68,10 +68,8 @@ int main(int argc, char* argv[]) } if (nc_enddef(ncid)) ERR; - for (i=0; i