mirror/netcdf-c
2
0
Fork 0
mirror of https://github.com/Unidata/netcdf-c.git synced 2025-02-23 16:59:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

merged master

Browse Source
This commit is contained in:
Ed Hartnett 2018-06-07 07:27:47 -06:00
commit 5112ef8c72
14 changed files with 550 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
libhdf5/hdf5file.c
View File

@ -695,48 +695,6 @@ dumpopenobjects(NC_HDF5_FILE_INFO_T* h5)
return;
}
#ifdef NC4NOTUSED
/**
* @internal Define the names of attributes to ignore added by the
* HDF5 dimension scale; these attached to variables. They cannot be
* modified thru the netcdf-4 API.
*/
static const char* NC_RESERVED_VARATT_LIST[] = {
NC_ATT_REFERENCE_LIST,
NC_ATT_CLASS,
NC_ATT_DIMENSION_LIST,
NC_ATT_NAME,
NC_ATT_COORDINATES,
NC_DIMID_ATT_NAME,
NULL
};
/**
* @internal Define the names of attributes to ignore because they are
* "hidden" global attributes. They can be read, but not modified thru
* the netcdf-4 API.
*/
static const char* NC_RESERVED_ATT_LIST[] = {
NC_ATT_FORMAT,
NC3_STRICT_ATT_NAME,
NCPROPS,
ISNETCDF4ATT,
SUPERBLOCKATT,
NULL
};
/**
* @internal Define the subset of the reserved list that is readable
* by name only
*/
static const char* NC_RESERVED_SPECIAL_LIST[] = {
ISNETCDF4ATT,
SUPERBLOCKATT,
NCPROPS,
NULL
};
#endif
size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE; /**< Default chunk cache size. */
size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS; /**< Default chunk cache number of elements. */
float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION; /**< Default chunk cache preemption. */
@ -1094,8 +1052,14 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, void* parameters,
/* Define mode gets turned on automatically on create. */
nc4_info->flags |= NC_INDEF;
NC4_get_fileinfo(nc4_info,&globalpropinfo);
NC4_put_propattr(nc4_info);
/* Get the HDF5 superblock and read and parse the special
* _NCProperties attribute. */
if ((retval = NC4_get_fileinfo(nc4_info, &globalpropinfo)))
BAIL(retval);
/* Write the _NCProperties attribute. */
if ((retval = NC4_put_propattr(nc4_info)))
BAIL(retval);
return NC_NOERR;
@ -2663,7 +2627,10 @@ nc4_open_file(const char *path, int mode, void* parameters, NC *nc)
if (H5Pclose(fapl_id) < 0)
BAIL(NC_EHDFERR);
NC4_get_fileinfo(nc4_info,NULL);
/* Get the HDF5 superblock and read and parse the special
* _NCProperties attribute. */
if ((retval = NC4_get_fileinfo(nc4_info, NULL)))
BAIL(retval);
return NC_NOERR;

66
libhdf5/nc4hdf.c
View File

@ -684,13 +684,8 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
#endif
int retval = NC_NOERR, range_error = 0, i, d2;
void *bufr = NULL;
#ifndef HDF5_CONVERT
int need_to_convert = 0;
size_t len = 1;
#endif
#ifdef HDF5_CONVERT
hid_t mem_typeid = 0;
#endif
/* Find our metadata for this file, group, and var. */
assert(nc);
@ -782,7 +777,6 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
BAIL(NC_EHDFERR);
}
#ifndef HDF5_CONVERT
/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if ((mem_nc_type != var->type_info->hdr.id || (var->type_info->hdr.id == NC_INT && is_long)) &&
@ -811,28 +805,12 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
BAIL(NC_ENOMEM);
}
else
#endif /* ifndef HDF5_CONVERT */
bufr = data;
#ifdef HDF5_CONVERT
/* Get the HDF type of the data in memory. */
if ((retval = nc4_get_hdf_typeid(h5, mem_nc_type, &mem_typeid,
var->type_info->endianness)))
BAIL(retval);
#endif
/* Create the data transfer property list. */
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
/* Apply the callback function which will detect range
* errors. Which one to call depends on the length of the
* destination buffer type. */
#ifdef HDF5_CONVERT
if (H5Pset_type_conv_cb(xfer_plistid, except_func, &range_error) < 0)
BAIL(NC_EHDFERR);
#endif
#ifdef USE_PARALLEL4
/* Set up parallel I/O, if needed. */
if ((retval = set_par_access(h5, var, xfer_plistid)))
@ -918,7 +896,6 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
}
}
#ifndef HDF5_CONVERT
/* Do we need to convert the data? */
if (need_to_convert)
{
@ -927,7 +904,6 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
(h5->cmode & NC_CLASSIC_MODEL), is_long, 0)))
BAIL(retval);
}
#endif
/* Write the data. At last! */
LOG((4, "about to H5Dwrite datasetid 0x%x mem_spaceid 0x%x "
@ -950,19 +926,13 @@ nc4_put_vara(NC *nc, int ncid, int varid, const size_t *startp,
range_error = 0;
exit:
#ifdef HDF5_CONVERT
if (mem_typeid > 0 && H5Tclose(mem_typeid) < 0)
BAIL2(NC_EHDFERR);
#endif
if (file_spaceid > 0 && H5Sclose(file_spaceid) < 0)
BAIL2(NC_EHDFERR);
if (mem_spaceid > 0 && H5Sclose(mem_spaceid) < 0)
BAIL2(NC_EHDFERR);
if (xfer_plistid && (H5Pclose(xfer_plistid) < 0))
BAIL2(NC_EPARINIT);
#ifndef HDF5_CONVERT
if (need_to_convert && bufr) free(bufr);
#endif
/* If there was an error return it, otherwise return any potential
range error value. If none, return NC_NOERR as usual.*/
@ -1018,13 +988,8 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
int fill_value_size[NC_MAX_VAR_DIMS];
int scalar = 0, retval = NC_NOERR, range_error = 0, i, d2;
void *bufr = NULL;
#ifdef HDF5_CONVERT
hid_t mem_typeid = 0;
#endif
#ifndef HDF5_CONVERT
int need_to_convert = 0;
size_t len = 1;
#endif
/* Find our metadata for this file, group, and var. */
assert(nc);
@ -1184,7 +1149,6 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
bufr = *(char **)data;
}
#ifndef HDF5_CONVERT
/* Are we going to convert any data? (No converting of compound or
* opaque types.) */
if ((mem_nc_type != var->type_info->hdr.id || (var->type_info->hdr.id == NC_INT && is_long)) &&
@ -1206,29 +1170,13 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
BAIL(NC_ENOMEM);
}
else
#endif /* ifndef HDF5_CONVERT */
if(!bufr)
bufr = data;
/* Get the HDF type of the data in memory. */
#ifdef HDF5_CONVERT
if ((retval = nc4_get_hdf_typeid(h5, mem_nc_type, &mem_typeid,
var->type_info->endianness)))
BAIL(retval);
#endif
/* Create the data transfer property list. */
if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0)
BAIL(NC_EHDFERR);
#ifdef HDF5_CONVERT
/* Apply the callback function which will detect range
* errors. Which one to call depends on the length of the
* destination buffer type. */
if (H5Pset_type_conv_cb(xfer_plistid, except_func, &range_error) < 0)
BAIL(NC_EHDFERR);
#endif
#ifdef USE_PARALLEL4
/* Set up parallel I/O, if needed. */
if ((retval = set_par_access(h5, var, xfer_plistid)))
@ -1241,12 +1189,7 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0)
BAIL(NC_EHDFERR);
#ifndef HDF5_CONVERT
/* Eventually the block below will go away. Right now it's
needed to support conversions between int/float, and range
checking converted data in the netcdf way. These features are
being added to HDF5 at the HDF5 World Hall of Coding right
now, by a staff of thousands of programming gnomes. */
/* Convert data if needed. */
if (need_to_convert)
{
if ((retval = nc4_convert_type(bufr, data, var->type_info->hdr.id, mem_nc_type,
@ -1262,7 +1205,6 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
range_error)
range_error = 0;
}
#endif
/* For strict netcdf-3 rules, ignore erange errors between UBYTE
* and BYTE types. */
@ -1357,10 +1299,6 @@ nc4_get_vara(NC *nc, int ncid, int varid, const size_t *startp,
}
exit:
#ifdef HDF5_CONVERT
if (mem_typeid > 0 && H5Tclose(mem_typeid) < 0)
BAIL2(NC_EHDFERR);
#endif
if (file_spaceid > 0)
{
if (H5Sclose(file_spaceid) < 0)
@ -1376,10 +1314,8 @@ exit:
if (H5Pclose(xfer_plistid) < 0)
BAIL2(NC_EHDFERR);
}
#ifndef HDF5_CONVERT
if (need_to_convert && bufr != NULL)
free(bufr);
#endif
if (xtend_size)
free(xtend_size);
if (fillvalue)

1
nc_test4/CMakeLists.txt
View File

@ -87,6 +87,7 @@ IF(TEST_PARALLEL4)
build_bin_test(tst_parallel)
build_bin_test(tst_parallel3)
build_bin_test(tst_parallel4)
build_bin_test(tst_parallel5)
build_bin_test(tst_nc4perf)
build_bin_test(tst_mode)
build_bin_test(tst_simplerw_coll_r)

26
nc_test4/Makefile.am
View File

@ -114,8 +114,8 @@ endif
# This are extra tests that will only be run if netcdf-4 is configured
# with --enable-parallel-tests.
if TEST_PARALLEL4
check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \
tst_parallel4 tst_nc4perf tst_mode tst_simplerw_coll_r
check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \
tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r
TESTS += run_par_test.sh
endif
@ -138,19 +138,15 @@ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \
ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl unfiltered.cdl \
ref_bzip2.c findplugin.in perftest.sh
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt \
radar_3d_compression.txt radar_2d_compression.txt \
radar_3d_chunking.txt tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl \
tst_*.nc tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl \
tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* \
usi_01.* thetau_01.* tst_*.nc tst_*.h5 tst_grp_rename.cdl \
tst_grp_rename.nc tst_grp_rename.dmp ref_grp_rename.cdl foo1.nc \
tst_interops2.h4 tst_h5_endians.nc tst_h4_lendian.h4 test.nc \
tst_atts_string_rewrite.nc tst_empty_vlen_unlim.nc \
tst_empty_vlen_lim.nc tst_parallel4_simplerw_coll.nc \
tst_fill_attr_vanish.nc tst_rehash.nc testszip.nc test.h5 \
szip_dump.cdl perftest.txt bigmeta.nc
CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt \
radar_3d_compression.txt radar_2d_compression.txt \
radar_3d_chunking.txt tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl \
tst_*.nc tst_floats2_*.cdl tst_ints2_*.cdl tst_shorts2_*.cdl \
tst_elena_*.cdl tst_simple*.cdl tst_chunks.cdl pr_A1.* tauu_A1.* \
usi_01.* thetau_01.* tst_*.h5 tst_grp_rename.cdl tst_grp_rename.dmp \
ref_grp_rename.cdl foo1.nc test.nc testszip.nc test.h5 szip_dump.cdl \
perftest.txt bigmeta.nc
DISTCLEANFILES = findplugin.sh

4
nc_test4/run_par_test.sh
View File

@ -28,6 +28,10 @@ mpiexec -n 8 ./tst_parallel4
#mpiexec -n 16 ./tst_parallel4
#mpiexec -n 32 ./tst_parallel4
#mpiexec -n 64 ./tst_parallel4
echo
echo "Testing collective writes with some 0 element writes..."
mpiexec -n 4 ./tst_parallel5
echo
echo "Parallel Performance Test for NASA"
mpiexec -n 4 ./tst_nc4perf

48
nc_test4/tst_enums.c
View File

@ -309,7 +309,55 @@ main(int argc, char **argv)
if (nc_insert_enum(ncid, typeid, "name", &ubyte_value)) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("*** testing enum ncdump...");
#define ENUM_NAME "cargo"
#define ENUM_VAR_NAME "in_the_hold_of_the_Irish_Rover"
#define NUM_ENUM_FIELDS 8
#define DIMSIZE 4
#define NDIMS1 1
{
int ncid, dimid, v1id, typeid;
int f;
size_t start[NDIMS1] = {0}, count[NDIMS1] = {1};
char field_name[NUM_ENUM_FIELDS][NC_MAX_NAME + 1] = {"bags of the best Sligo rags", "barrels of bones",
"bails of old nanny goats' tails", "barrels of stones",
"dogs", "hogs", "barrels of porter",
"sides of old blind horses hides"};
unsigned long long field_value[NUM_ENUM_FIELDS] = {1000000, 2000000, 3000000, 4000000,
5000000, 6000000, 7000000, 8000000};
unsigned long long data = 1000000, data_in;
/* Create a netcdf-4 file. */
/*nc_set_log_level(3);*/
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
/* Create a dimension. */
if (nc_def_dim(ncid, DIM_NAME, DIMSIZE, &dimid)) ERR;
/* Create an enum type. */
if (nc_def_enum(ncid, NC_UINT64, ENUM_NAME, &typeid)) ERR;
for (f = 0; f < NUM_ENUM_FIELDS; f++)
if (nc_insert_enum(ncid, typeid, field_name[f], &field_value[f])) ERR;
/* Create one var. */
if (nc_def_var(ncid, ENUM_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Write phoney data. */
if (nc_put_vara(ncid, v1id, start, count, &data)) ERR;
if (nc_sync(ncid)) ERR;
/* Read phoney data. */
if (nc_get_vara(ncid, v1id, start, count, &data_in)) ERR;
if (data_in != data) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}

74
nc_test4/tst_files.c
View File

@ -1,6 +1,6 @@
/* This is part of the netCDF package.
Copyright 2005 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
/* This is part of the netCDF package. Copyright 2005-2018 University
Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
conditions of use.
Test netcdf-4 file code.
Ed Hartnett
@ -10,9 +10,6 @@
#include <nc_tests.h>
#include "err_macros.h"
#ifdef IGNORE
extern NC_FILE_INFO_T *nc_file;
#endif
int test_redef(int format);
#define FILE_NAME "tst_files.nc"
@ -45,9 +42,6 @@ main(int argc, char **argv)
{
int ncid;
/* Make sure bad create mode causes failure. */
/*if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;*/
/* Create an empty file. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_close(ncid)) ERR;
@ -266,14 +260,12 @@ main(int argc, char **argv)
test_redef(NC_FORMAT_64BIT_OFFSET);
SUMMARIZE_ERR;
#ifdef USE_NETCDF4
printf("*** testing redef for netCDF-4 ...");
test_redef(NC_FORMAT_NETCDF4);
SUMMARIZE_ERR;
printf("*** testing redef for netCDF-4, with strict netCDF-3 rules...");
test_redef(NC_FORMAT_NETCDF4_CLASSIC);
SUMMARIZE_ERR;
#endif /* USE_NETCDF4 */
#ifdef ENABLE_CDF5
printf("*** testing redef for CDF5...");
@ -298,14 +290,11 @@ main(int argc, char **argv)
if (format != NC_FORMAT_64BIT_OFFSET) ERR;
if (nc_close(ncid)) ERR;
#ifdef USE_NETCDF4
/* Create a netcdf-4 file. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLOBBER, &ncid)) ERR;
if (nc_inq_format(ncid, &format)) ERR;
if (format != NC_FORMAT_NETCDF4) ERR;
if (nc_close(ncid)) ERR;
#endif /* USE_NETCDF4 */
}
SUMMARIZE_ERR;
printf("*** testing CLASSIC_MODEL flag with classic formats...");
@ -326,7 +315,6 @@ main(int argc, char **argv)
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
#ifdef USE_NETCDF4
printf("*** testing multiple open files...");
{
#define VAR_NAME "Captain_Kirk"
@ -370,7 +358,6 @@ main(int argc, char **argv)
if (nc_close(ncid[f])) ERR;
}
SUMMARIZE_ERR;
#endif /* USE_NETCDF4 */
FINAL_RESULTS;
}
@ -395,6 +382,14 @@ main(int argc, char **argv)
#define NEW_CACHE_SIZE 32000000
#define NEW_CACHE_NELEMS 2000
#define NEW_CACHE_PREEMPTION .75
#define NEW_CACHE_SIZE_2 16000000
#define NEW_CACHE_NELEMS_2 1000
#define NEW_CACHE_PREEMPTION_2 .50
/* These prototypes are needed because these functions, used by the
* Fortran API, are not prototyped in netcdf.h. */
int nc_get_chunk_cache_ints(int *sizep, int *nelemsp, int *preemptionp);
int nc_set_chunk_cache_ints(int size, int nelems, int preemption);
int
test_redef(int format)
@ -412,6 +407,8 @@ test_redef(int format)
short short_out = -999;
nc_type xtype_in;
size_t cache_size_in, cache_nelems_in;
int cache_size_int_in, cache_nelems_int_in;
int cache_preemption_int_in;
float cache_preemption_in;
int ret;
@ -436,6 +433,51 @@ test_redef(int format)
&cache_preemption_in)) ERR;
if (cache_size_in != NEW_CACHE_SIZE || cache_nelems_in != NEW_CACHE_NELEMS ||
cache_preemption_in != NEW_CACHE_PREEMPTION) ERR;
cache_size_in = 0;
if (nc_get_chunk_cache(&cache_size_in, NULL, NULL)) ERR;
if (cache_size_in != NEW_CACHE_SIZE) ERR;
cache_nelems_in = 0;
if (nc_get_chunk_cache(NULL, &cache_nelems_in, NULL)) ERR;
if (cache_nelems_in != NEW_CACHE_NELEMS) ERR;
cache_preemption_in = 0;
if (nc_get_chunk_cache(NULL, NULL, &cache_preemption_in)) ERR;
if (cache_preemption_in != NEW_CACHE_PREEMPTION) ERR;
/* Retrieve the chunk cache settings as integers, like the fortran API. */
if (nc_get_chunk_cache_ints(&cache_size_int_in, &cache_nelems_int_in,
&cache_preemption_int_in)) ERR;
if (cache_size_int_in != NEW_CACHE_SIZE || cache_nelems_int_in != NEW_CACHE_NELEMS ||
cache_preemption_int_in != (int)(NEW_CACHE_PREEMPTION * 100)) ERR;
if (nc_get_chunk_cache_ints(NULL, NULL, NULL)) ERR;
cache_size_int_in = 0;
if (nc_get_chunk_cache_ints(&cache_size_int_in, NULL, NULL)) ERR;
if (cache_size_int_in != NEW_CACHE_SIZE) ERR;
cache_nelems_int_in = 0;
if (nc_get_chunk_cache_ints(NULL, &cache_nelems_int_in, NULL)) ERR;
if (cache_nelems_int_in != NEW_CACHE_NELEMS) ERR;
cache_preemption_int_in = 0;
if (nc_get_chunk_cache_ints(NULL, NULL, &cache_preemption_int_in)) ERR;
if (cache_preemption_int_in != (int)(NEW_CACHE_PREEMPTION * 100)) ERR;
/* These won't work. */
if (nc_set_chunk_cache_ints(-1, NEW_CACHE_NELEMS_2,
(int)(NEW_CACHE_PREEMPTION_2 * 100)) != NC_EINVAL) ERR;
if (nc_set_chunk_cache_ints(NEW_CACHE_SIZE_2, 0,
(int)(NEW_CACHE_PREEMPTION_2 * 100)) != NC_EINVAL) ERR;
if (nc_set_chunk_cache_ints(NEW_CACHE_SIZE_2, NEW_CACHE_NELEMS_2,
-1) != NC_EINVAL) ERR;
if (nc_set_chunk_cache_ints(NEW_CACHE_SIZE_2, NEW_CACHE_NELEMS_2,
101) != NC_EINVAL) ERR;
/* Change chunk cache again. */
if (nc_set_chunk_cache_ints(NEW_CACHE_SIZE_2, NEW_CACHE_NELEMS_2,
(int)(NEW_CACHE_PREEMPTION_2 * 100))) ERR;
if (nc_get_chunk_cache_ints(&cache_size_int_in, &cache_nelems_int_in,
&cache_preemption_int_in)) ERR;
if (cache_size_int_in != NEW_CACHE_SIZE_2 || cache_nelems_int_in != NEW_CACHE_NELEMS_2 ||
cache_preemption_int_in != (int)(NEW_CACHE_PREEMPTION_2 * 100)) ERR;
/* This will fail, except for netcdf-4/hdf5, which permits any
* name. */

29
nc_test4/tst_parallel3.c
View File

@ -1,22 +1,9 @@
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by the Board of Trustees of the University of Illinois. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the files COPYING and Copyright.html. COPYING can be found at the root *
* of the source code distribution tree; Copyright.html can be found at the *
* root level of an installed copy of the electronic HDF5 document set and *
* is linked from the top-level documents page. It can also be found at *
* http://hdf.ncsa.uiuc.edu/HDF5/doc/Copyright.html. If you do not have *
* access to either file, you may request a copy from hdfhelp@ncsa.uiuc.edu. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Main driver of the Parallel NetCDF4 tests
*
*/
/* This test of netCDF-4 parallel I/O was contributed by the HDF5
* team. */
#include <nc_tests.h>
#include "err_macros.h"
@ -45,9 +32,9 @@
dimension to be divided evenly, the best set of number of processor
should be 2 power n. However, for NetCDF4 tests, the following numbers
are generally treated as good numbers:
1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256
1,2,3,4,6,8,12,16
The maximum number of processor is 256.*/
The maximum number of processor is 16.*/
int test_pio(int);
int test_pio_attr(int);
@ -64,7 +51,7 @@ int main(int argc, char **argv)
{
int mpi_size, mpi_rank; /* mpi variables */
int i;
int NUMP[16] ={1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256};
int NUMP[8] ={1,2,3,4,6,8,12,16};
int size_flag = 0;
/* Un-buffer the stdout and stderr */
@ -78,7 +65,7 @@ int main(int argc, char **argv)
if (mpi_rank == 0)
printf("\n*** Testing more advanced parallel access.\n");
for (i = 0; i < 16; i++){
for (i = 0; i < 8; i++){
if(mpi_size == NUMP[i])
{
size_flag = 1;
@ -88,7 +75,7 @@ int main(int argc, char **argv)
if(!size_flag){
printf("mpi_size is wrong\n");
printf(" The number of processor must be chosen from\n");
printf(" 1,2,3,4,6,8,12,16,24,32,48,64,96,128,192,256 \n");
printf(" 1,2,3,4,6,8,12,16 \n");
return -1;
}
@ -711,7 +698,7 @@ int test_pio_extend(int flag){
int dimsVrtx[2];
size_t start[2];
size_t count[2];
int vertices[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
int vertices[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &procs);

297
nc_test4/tst_parallel5.c Normal file
View File

@ -0,0 +1,297 @@
/* Copyright 2018, UCAR/Unidata See COPYRIGHT file for copying and
* redistribution conditions.
*
* This program tests netcdf-4 parallel I/O. In this test I write data
* on one task, while writing 0 items on others.
*
* Ed Hartnett
*/
#include <nc_tests.h>
#include "err_macros.h"
#include <mpi.h>
#define FILE "tst_parallel5.nc"
#define VAR_NAME "TheIrishRover"
#define DIM_NAME "number_of_masts"
#define MASTS 27
#define NDIMS1 1
#define DIMSIZE 4
#define NUM_PROC 4
#define NUM_SLABS 10
#define NUM_ACCESS_TESTS 2
int
main(int argc, char **argv)
{
int mpi_size, mpi_rank;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
int ncid, v1id, dimid;
size_t start[NDIMS1] = {0}, count[NDIMS1] = {0};
int data = MASTS;
int data_in = TEST_VAL_42;
int acc;
/* Initialize MPI. */
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_size);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
/* Require exactly 4 tasks. */
if (mpi_size != NUM_PROC) ERR;
if (!mpi_rank)
printf("\n*** Testing parallel I/O.\n");
if (!mpi_rank)
printf("*** testing whether we can write 0 elements from some tasks...");
{
for (acc = 0; acc < NUM_ACCESS_TESTS; acc++)
{
/* Create a parallel netcdf-4 file. */
/*nc_set_log_level(3);*/
if (nc_create_par(FILE, NC_NETCDF4|NC_MPIIO, comm, info, &ncid)) ERR;
/* Create a dimension. */
if (nc_def_dim(ncid, DIM_NAME, DIMSIZE, &dimid)) ERR;
/* Create one var. */
if (nc_def_var(ncid, VAR_NAME, NC_INT, NDIMS1, &dimid, &v1id)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
if (!mpi_rank)
count[0] = 1;
if (nc_var_par_access(ncid, v1id, acc ? NC_COLLECTIVE : NC_INDEPENDENT)) ERR;
/* Write phoney data. */
if (nc_put_vara_int(ncid, v1id, start, count, &data)) ERR;
if (nc_sync(ncid)) ERR;
/* Read phoney data. */
if (nc_get_vara_int(ncid, v1id, start, count, &data_in)) ERR;
/* Task 0 has MASTS, the others have data_in remaining, as
* initialized, at TEST_VAL_42. */
if (data_in != (mpi_rank ? TEST_VAL_42 : MASTS)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
}
if (!mpi_rank)
SUMMARIZE_ERR;
if (!mpi_rank)
printf("*** testing enum type and parallel I/O...");
{
for (acc = 0; acc < NUM_ACCESS_TESTS; acc++)
{
#define ENUM_NAME "cargo"
#define ENUM_VAR_NAME "in_the_hold_of_the_Irish_Rover"
#define NUM_ENUM_FIELDS 8
int typeid;
int f;
char field_name[NUM_ENUM_FIELDS][NC_MAX_NAME + 1] = {"bags of the best Sligo rags", "barrels of bones",
"bails of old nanny goats' tails", "barrels of stones",
"dogs", "hogs", "barrels of porter",
"sides of old blind horses hides"};
unsigned long long field_value[NUM_ENUM_FIELDS] = {1000000, 2000000, 3000000, 4000000,
5000000, 6000000, 7000000, 8000000};
unsigned long long data = 1000000, data_in = TEST_VAL_42;
/* Create a parallel netcdf-4 file. */
/*nc_set_log_level(3);*/
if (nc_create_par(FILE, NC_NETCDF4|NC_MPIIO, comm, info, &ncid)) ERR;
/* Create a dimension. */
if (nc_def_dim(ncid, DIM_NAME, DIMSIZE, &dimid)) ERR;
/* Create an enum type. */
if (nc_def_enum(ncid, NC_UINT64, ENUM_NAME, &typeid)) ERR;
for (f = 0; f < NUM_ENUM_FIELDS; f++)
if (nc_insert_enum(ncid, typeid, field_name[f], &field_value[f])) ERR;
/* Create one var. */
if (nc_def_var(ncid, ENUM_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
if (!mpi_rank)
count[0] = 1;
if (nc_var_par_access(ncid, v1id, acc ? NC_COLLECTIVE : NC_INDEPENDENT)) ERR;
/* Write phoney data. */
if (nc_put_vara(ncid, v1id, start, count, &data)) ERR;
if (nc_sync(ncid)) ERR;
/* Read phoney data. */
if (nc_get_vara(ncid, v1id, start, count, &data_in)) ERR;
/* Task 0 has 1000000, the others have data_in remaining, as
* initialized, at TEST_VAL_42. */
if (data_in != (mpi_rank ? TEST_VAL_42 : 1000000)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
}
if (!mpi_rank)
SUMMARIZE_ERR;
if (!mpi_rank)
printf("*** testing compound type and parallel I/O...");
{
for (acc = 0; acc < NUM_ACCESS_TESTS; acc++)
{
#define COMPOUND_NAME "crew_info"
#define COMPOUND_VAR_NAME "whale_of_a_crew"
#define NUM_CREW 5
#define CREW_DIM_NAME "number_of_crew"
int typeid;
struct crew
{
char name[NC_MAX_NAME + 1];
char description[NC_MAX_NAME + 1];
char origin[NC_MAX_NAME + 1];
int age;
};
struct crew data = {"Mick McCann", "the skipper of the Irish Rover",
"from the banks of the Bann", 42};
struct crew data_in = {"", "", "", -42};
int dim_size = NC_MAX_NAME + 1;
/* Create a parallel netcdf-4 file. */
/*nc_set_log_level(3);*/
if (nc_create_par(FILE, NC_NETCDF4|NC_MPIIO, comm, info, &ncid)) ERR;
/* Create a dimension. */
if (nc_def_dim(ncid, CREW_DIM_NAME, NUM_CREW, &dimid)) ERR;
/* Create a compound type. */
if (nc_def_compound(ncid, sizeof(struct crew), COMPOUND_NAME, &typeid)) ERR;
if (nc_insert_array_compound(ncid, typeid, "name", NC_COMPOUND_OFFSET(struct crew, name), NC_CHAR, 1, &dim_size));
if (nc_insert_array_compound(ncid, typeid, "description", NC_COMPOUND_OFFSET(struct crew, description), NC_CHAR, 1, &dim_size));
if (nc_insert_array_compound(ncid, typeid, "origin", NC_COMPOUND_OFFSET(struct crew, origin), NC_CHAR, 1, &dim_size));
if (nc_insert_compound(ncid, typeid, "age", NC_COMPOUND_OFFSET(struct crew, age), NC_INT));
/* Create one var. */
if (nc_def_var(ncid, COMPOUND_VAR_NAME, typeid, NDIMS1, &dimid, &v1id)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
if (!mpi_rank)
count[0] = 1;
if (nc_var_par_access(ncid, v1id, acc ? NC_COLLECTIVE : NC_INDEPENDENT)) ERR;
/* Write phoney data. */
if (nc_put_vara(ncid, v1id, start, count, &data)) ERR;
if (nc_sync(ncid)) ERR;
/* Read phoney data. */
if (nc_get_vara(ncid, v1id, start, count, &data_in)) ERR;
/* Task 0 has data, the others have nothing. */
if (!mpi_rank)
{
if (strcmp(data_in.name, data.name) || strcmp(data_in.description, data.description) ||
strcmp(data_in.origin, data.origin) || data_in.age != data.age) ERR;
}
else
{
if (strcmp(data_in.name, "") || strcmp(data_in.description, "") ||
strcmp(data_in.origin, "") || data_in.age != -42) ERR;
}
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
}
if (!mpi_rank)
SUMMARIZE_ERR;
if (!mpi_rank)
printf("*** testing string type and parallel I/O...");
{
for (acc = 0; acc < NUM_ACCESS_TESTS; acc++)
{
#define STORY_VAR_NAME "fate_of_the_Irish_Rover"
#define STORY_DIM_NAME "number_of_lines"
#define STORY_LEN 8
char *story[STORY_LEN] = {"We had sailed seven years when the measles broke out",
"And the ship lost it's way in the fog",
"And that whale of the crew was reduced down to two",
"Just myself and the captain's old dog",
"Then the ship struck a rock, oh Lord what a shock",
"The bulkhead was turned right over",
"Turned nine times around, and the poor dog was drowned",
"I'm the last of the Irish Rover"};
char *story_in[STORY_LEN];
int s;
/* Create a netcdf-4 file. Turns out that HDF5 does not
* support VLEN writes with parallel I/O. Strings are
* VLENS. So here I write a file with task 0 and then read it
* with all tasks. */
if (!mpi_rank)
{
if (nc_create(FILE, NC_NETCDF4, &ncid)) ERR;
/* Create a dimension. */
if (nc_def_dim(ncid, STORY_DIM_NAME, STORY_LEN, &dimid)) ERR;
/* Create one var. */
if (nc_def_var(ncid, STORY_VAR_NAME, NC_STRING, NDIMS1, &dimid, &v1id)) ERR;
/* Write metadata to file. */
if (nc_enddef(ncid)) ERR;
/* Set up slab for this process. */
count[0] = STORY_LEN;
/* Write phoney data. */
if (nc_put_vara(ncid, v1id, start, count, story)) ERR;
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
/* Now try parallel read. */
if (nc_open_par(FILE, NC_MPIIO, comm, info, &ncid)) ERR;
/* Task 0 reads all 8 lines, other tasks read 0. */
if (nc_get_vara(ncid, v1id, start, count, story_in)) ERR;
if (!mpi_rank)
{
for (s = 0; s < STORY_LEN; s++)
if (strcmp(story_in[s], story[s])) ERR;
if (nc_free_string(STORY_LEN, (char **)story_in)) ERR;
}
/* Close the netcdf file. */
if (nc_close(ncid)) ERR;
}
}
if (!mpi_rank)
SUMMARIZE_ERR;
/* Shut down MPI. */
MPI_Finalize();
if (!mpi_rank)
FINAL_RESULTS;
return 0;
}

54
nc_test4/tst_sync.c
View File

@ -86,5 +86,59 @@ main(int argc, char **argv)
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing sync without enddef...");
{
int ncid, var1_id, var2_id, var3_id, dimid;
int ndims, nvars, natts, unlimdimid, dimid_in;
nc_type xtype_in;
char name_in[NC_MAX_NAME + 1];
/* Create a file. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME, 1, &dimid)) ERR;
if (nc_def_var(ncid, VAR1_NAME, NC_FLOAT, 1, &dimid, &var1_id)) ERR;
if (nc_def_var(ncid, VAR2_NAME, NC_FLOAT, 1, &dimid, &var2_id)) ERR;
if (nc_def_var(ncid, DIM_NAME, NC_FLOAT, 1, &dimid, &var3_id)) ERR;
if (nc_sync(ncid)) ERR;
if (nc_close(ncid)) ERR;
/* Reopen the file and check it. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
if (ndims != 1 || nvars != 3 || natts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(ncid, 2, name_in, &xtype_in, &ndims, &dimid_in, &natts)) ERR;
if (strcmp(name_in, DIM_NAME) || xtype_in != NC_FLOAT || ndims != 1 ||
dimid_in != 0 || natts != 0) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing that sync failes without enddef for classic model...");
{
int ncid, var1_id, var2_id, var3_id, dimid;
int ndims, nvars, natts, unlimdimid, dimid_in;
nc_type xtype_in;
char name_in[NC_MAX_NAME + 1];
/* Create a file. */
if (nc_create(FILE_NAME, NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR;
if (nc_def_dim(ncid, DIM_NAME, 1, &dimid)) ERR;
if (nc_def_var(ncid, VAR1_NAME, NC_FLOAT, 1, &dimid, &var1_id)) ERR;
if (nc_def_var(ncid, VAR2_NAME, NC_FLOAT, 1, &dimid, &var2_id)) ERR;
if (nc_def_var(ncid, DIM_NAME, NC_FLOAT, 1, &dimid, &var3_id)) ERR;
if (nc_sync(ncid) != NC_EINDEFINE) ERR;
if (nc_enddef(ncid)) ERR;
if (nc_sync(ncid)) ERR;
if (nc_close(ncid)) ERR;
/* Reopen the file and check it. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq(ncid, &ndims, &nvars, &natts, &unlimdimid)) ERR;
if (ndims != 1 || nvars != 3 || natts != 0 || unlimdimid != -1) ERR;
if (nc_inq_var(ncid, 2, name_in, &xtype_in, &ndims, &dimid_in, &natts)) ERR;
if (strcmp(name_in, DIM_NAME) || xtype_in != NC_FLOAT || ndims != 1 ||
dimid_in != 0 || natts != 0) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}

59
nc_test4/tst_vars4.c
View File

@ -3,20 +3,13 @@
See COPYRIGHT file for conditions of use.
Test netcdf-4 variables.
$Id$
Ed Hartnett
*/
#include <nc_tests.h>
#include "err_macros.h"
#define FILE_NAME "tst_vars4.nc"
int
main(int argc, char **argv)
{
printf("\n*** Testing netcdf-4 variable functions, even more.\n");
printf("**** testing Jeff's dimension problem...");
{
#define NDIMS2 2
#define NUM_VARS 1
#define Y_NAME "y"
@ -25,6 +18,12 @@ main(int argc, char **argv)
#define XDIM_LEN 2
#define YDIM_LEN 5
int
main(int argc, char **argv)
{
printf("\n*** Testing netcdf-4 variable functions, even more.\n");
printf("**** testing Jeff's dimension problem...");
{
int varid, ncid, dims[NDIMS2], dims_in[NDIMS2];
int ndims, nvars, ngatts, unlimdimid, natts;
char name_in[NC_MAX_NAME + 1];
@ -62,5 +61,49 @@ main(int argc, char **argv)
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing chunking turned on by fletcher...");
{
int varid, ncid, dims[NDIMS2];
int storage_in;
size_t chunksizes_in[NDIMS2];
if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
if (nc_def_var_fletcher32(ncid, varid, NC_FLETCHER32)) ERR;
if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
printf("**** testing chunking turned on by shuffle...");
{
int varid, ncid, dims[NDIMS2];
int storage_in;
size_t chunksizes_in[NDIMS2];
if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR;
if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR;
if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR;
if (nc_def_var(ncid, VAR_NAME, NC_FLOAT, 2, dims, &varid)) ERR;
if (nc_def_var_deflate(ncid, varid, NC_SHUFFLE, 0, 0)) ERR;
if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
if (nc_close(ncid)) ERR;
/* Open the file and check. */
if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR;
if (nc_inq_var_chunking(ncid, varid, &storage_in, chunksizes_in)) ERR;
if (chunksizes_in[0] != XDIM_LEN || chunksizes_in[1] != YDIM_LEN) ERR;
if (nc_close(ncid)) ERR;
}
SUMMARIZE_ERR;
FINAL_RESULTS;
}

2
ncdump/Makefile.am
View File

@ -130,7 +130,7 @@ tst_hdf5_offset.sh run_ncgen_nc4_tests.sh tst_nccopy3_subset.sh \
ref_nccopy3_subset.nc ref_test_corrupt_magic.nc tst_ncgen_shared.sh \
tst_ncgen4.sh tst_ncgen4_classic.sh tst_ncgen4_diff.sh \
tst_ncgen4_cycle.sh tst_null_byte_padding.sh \
ref_null_byte_padding_test.nc
ref_null_byte_padding_test.nc ref_tst_irish_rover.nc
# The L512.bin file is file containing exactly 512 bytes each of value 0.
# It is used for creating hdf5 files with varying offsets for testing.

BIN
ncdump/ref_tst_irish_rover.nc Normal file
View File

Binary file not shown.

4
ncdump/tst_netcdf4.sh
View File

@ -103,5 +103,9 @@ diff -b tst_special_atts.cdl $srcdir/ref_tst_special_atts.cdl ; ERR
#fi
#rm -fr ./ignore_tst_netcdf4
# This should work, but does not. See github issue 982.
#echo "*** creating tst_output_irish_rover.cdl from ref_tst_irish_rover.nc..."
#${NCDUMP} ref_tst_irish_rover.nc > tst_output_irish_rover.cdl
echo "*** All ncgen and ncdump test output for netCDF-4 format passed!"
exit 0