mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2024-12-09 07:32:32 +08:00
Code Issues Packages Projects Releases Wiki Activity
a87ca572fe
hdf5/examples/ph5_subfiling.c
Dana Robinson 93748d4ab5
Convert hbool_t --> bool in examples (#3492)
2023-09-04 22:59:41 -05:00

552 lines
17 KiB
C
Raw Blame History

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Copyright by The HDF Group. *
* All rights reserved. *
* *
* This file is part of HDF5. The full HDF5 copyright notice, including *
* terms governing use, modification, and redistribution, is contained in *
* the COPYING file, which can be found at the root of the source code *
* distribution tree, or in https://www.hdfgroup.org/licenses. *
* If you do not have access to either file, you may request a copy from *
* help@hdfgroup.org. *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
* Example of using HDF5's Subfiling VFD to write to an
* HDF5 file that is striped across multiple subfiles
*
* If the HDF5_NOCLEANUP environment variable is set, the
* files that this example creates will not be removed as
* the example finishes.
*
* In general, the current working directory in which compiling
* is done, is not suitable for parallel I/O and there is no
* standard pathname for parallel file systems. In some cases,
* the parallel file name may even need some parallel file type
* prefix such as: "pfs:/GF/...". Therefore, this example parses
* the HDF5_PARAPREFIX environment variable for a prefix, if one
* is needed.
*/
#include <stdlib.h>
#include "hdf5.h"
#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_SUBFILING_VFD)
#define EXAMPLE_FILE "h5_subfiling_default_example.h5"
#define EXAMPLE_FILE2 "h5_subfiling_custom_example.h5"
#define EXAMPLE_FILE3 "h5_subfiling_precreate_example.h5"
#define EXAMPLE_DSET_NAME "DSET"
#define EXAMPLE_DSET_DIMS 2
/* Have each MPI rank write 16MiB of data */
#define EXAMPLE_DSET_NY 4194304
/* Dataset datatype */
#define EXAMPLE_DSET_DATATYPE H5T_NATIVE_INT
typedef int EXAMPLE_DSET_C_DATATYPE;
/* Cleanup created files */
static void
cleanup(char *filename, hid_t fapl_id)
{
bool do_cleanup = getenv(HDF5_NOCLEANUP) ? false : true;
if (do_cleanup)
H5Fdelete(filename, fapl_id);
}
/*
* An example of using the HDF5 Subfiling VFD with
* its default settings of 1 subfile per node, with
* a stripe size of 32MiB
*/
static void
subfiling_write_default(hid_t fapl_id, int mpi_size, int mpi_rank)
{
EXAMPLE_DSET_C_DATATYPE *data;
hsize_t dset_dims[EXAMPLE_DSET_DIMS];
hsize_t start[EXAMPLE_DSET_DIMS];
hsize_t count[EXAMPLE_DSET_DIMS];
hid_t file_id;
hid_t subfiling_fapl;
hid_t dset_id;
hid_t filespace;
char filename[512];
char *par_prefix;
/*
* Make a copy of the FAPL so we don't disturb
* it for the other examples
*/
subfiling_fapl = H5Pcopy(fapl_id);
/*
* Set Subfiling VFD on FAPL using default settings
* (use IOC VFD, 1 IOC per node, 32MiB stripe size)
*
* Note that all of Subfiling's configuration settings
* can be adjusted with environment variables as well
* in this case.
*/
H5Pset_fapl_subfiling(subfiling_fapl, NULL);
/*
* OPTIONAL: Set alignment of objects in HDF5 file to
* be equal to the Subfiling stripe size.
* Choosing a Subfiling stripe size and HDF5
* object alignment value that are some
* multiple of the disk block size can
* generally help performance by ensuring
* that I/O is well-aligned and doesn't
* excessively cross stripe boundaries.
*
* Note that this option can substantially
* increase the size of the resulting HDF5
* files, so it is a good idea to keep an eye
* on this.
*/
H5Pset_alignment(subfiling_fapl, 0, 33554432); /* Align to default 32MiB stripe size */
/* Parse any parallel prefix and create filename */
par_prefix = getenv("HDF5_PARAPREFIX");
snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "",
EXAMPLE_FILE);
/*
* Create a new file collectively
*/
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
/*
* Create the dataspace for the dataset. The first
* dimension varies with the number of MPI ranks
* while the second dimension is fixed.
*/
dset_dims[0] = mpi_size;
dset_dims[1] = EXAMPLE_DSET_NY;
filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL);
/*
* Create the dataset with default properties
*/
dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
/*
* Each MPI rank writes from a contiguous memory
* region to the hyperslab in the file
*/
start[0] = mpi_rank;
start[1] = 0;
count[0] = 1;
count[1] = dset_dims[1];
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL);
/*
* Initialize data buffer
*/
data = malloc(count[0] * count[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
for (size_t i = 0; i < count[0] * count[1]; i++) {
data[i] = mpi_rank + i;
}
/*
* Write to dataset
*/
H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data);
/*
* Close/release resources.
*/
free(data);
H5Dclose(dset_id);
H5Sclose(filespace);
H5Fclose(file_id);
cleanup(EXAMPLE_FILE, subfiling_fapl);
H5Pclose(subfiling_fapl);
}
/*
* An example of using the HDF5 Subfiling VFD with
* custom settings
*/
static void
subfiling_write_custom(hid_t fapl_id, int mpi_size, int mpi_rank)
{
EXAMPLE_DSET_C_DATATYPE *data;
H5FD_subfiling_config_t subf_config;
H5FD_ioc_config_t ioc_config;
hsize_t dset_dims[EXAMPLE_DSET_DIMS];
hsize_t start[EXAMPLE_DSET_DIMS];
hsize_t count[EXAMPLE_DSET_DIMS];
hid_t file_id;
hid_t subfiling_fapl;
hid_t dset_id;
hid_t filespace;
char filename[512];
char *par_prefix;
/*
* Make a copy of the FAPL so we don't disturb
* it for the other examples
*/
subfiling_fapl = H5Pcopy(fapl_id);
/*
* Get a default Subfiling and IOC configuration
*/
H5Pget_fapl_subfiling(subfiling_fapl, &subf_config);
H5Pget_fapl_ioc(subfiling_fapl, &ioc_config);
/*
* Set Subfiling configuration to use a 1MiB
* stripe size and the SELECT_IOC_EVERY_NTH_RANK
* selection method. By default, without a setting
* in the H5FD_SUBFILING_IOC_SELECTION_CRITERIA
* environment variable, this will use every MPI
* rank as an I/O concentrator.
*/
subf_config.shared_cfg.stripe_size = 1048576;
subf_config.shared_cfg.ioc_selection = SELECT_IOC_EVERY_NTH_RANK;
/*
* Set IOC configuration to use 2 worker threads
* per IOC instead of the default setting and
* update IOC configuration with new subfiling
* configuration.
*/
ioc_config.thread_pool_size = 2;
/*
* Set our new configuration on the IOC
* FAPL used for Subfiling
*/
H5Pset_fapl_ioc(subf_config.ioc_fapl_id, &ioc_config);
/*
* Finally, set our new Subfiling configuration
* on the original FAPL
*/
H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
/*
* OPTIONAL: Set alignment of objects in HDF5 file to
* be equal to the Subfiling stripe size.
* Choosing a Subfiling stripe size and HDF5
* object alignment value that are some
* multiple of the disk block size can
* generally help performance by ensuring
* that I/O is well-aligned and doesn't
* excessively cross stripe boundaries.
*
* Note that this option can substantially
* increase the size of the resulting HDF5
* files, so it is a good idea to keep an eye
* on this.
*/
H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */
/* Parse any parallel prefix and create filename */
par_prefix = getenv("HDF5_PARAPREFIX");
snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "",
EXAMPLE_FILE2);
/*
* Create a new file collectively
*/
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
/*
* Create the dataspace for the dataset. The first
* dimension varies with the number of MPI ranks
* while the second dimension is fixed.
*/
dset_dims[0] = mpi_size;
dset_dims[1] = EXAMPLE_DSET_NY;
filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL);
/*
* Create the dataset with default properties
*/
dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
/*
* Each MPI rank writes from a contiguous memory
* region to the hyperslab in the file
*/
start[0] = mpi_rank;
start[1] = 0;
count[0] = 1;
count[1] = dset_dims[1];
H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, NULL, count, NULL);
/*
* Initialize data buffer
*/
data = malloc(count[0] * count[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
for (size_t i = 0; i < count[0] * count[1]; i++) {
data[i] = mpi_rank + i;
}
/*
* Write to dataset
*/
H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data);
/*
* Close/release resources.
*/
free(data);
H5Dclose(dset_id);
H5Sclose(filespace);
H5Fclose(file_id);
cleanup(EXAMPLE_FILE2, subfiling_fapl);
H5Pclose(subfiling_fapl);
}
/*
* An example of pre-creating an HDF5 file on MPI rank
* 0 when using the HDF5 Subfiling VFD. In this case,
* the subfiling stripe count must be set so that rank
* 0 knows how many subfiles to pre-create.
*/
static void
subfiling_write_precreate(hid_t fapl_id, int mpi_size, int mpi_rank)
{
EXAMPLE_DSET_C_DATATYPE *data;
H5FD_subfiling_config_t subf_config;
hsize_t dset_dims[EXAMPLE_DSET_DIMS];
hsize_t start[EXAMPLE_DSET_DIMS];
hsize_t count[EXAMPLE_DSET_DIMS];
hid_t file_id;
hid_t subfiling_fapl;
hid_t dset_id;
hid_t filespace;
char filename[512];
char *par_prefix;
/*
* Make a copy of the FAPL so we don't disturb
* it for the other examples
*/
subfiling_fapl = H5Pcopy(fapl_id);
/*
* Get a default Subfiling and IOC configuration
*/
H5Pget_fapl_subfiling(subfiling_fapl, &subf_config);
/*
* Set the Subfiling stripe count so that rank
* 0 knows how many subfiles the logical HDF5
* file should consist of. In this case, use
* 5 subfiles with a default stripe size of
* 32MiB.
*/
subf_config.shared_cfg.stripe_count = 5;
/*
* OPTIONAL: Set alignment of objects in HDF5 file to
* be equal to the Subfiling stripe size.
* Choosing a Subfiling stripe size and HDF5
* object alignment value that are some
* multiple of the disk block size can
* generally help performance by ensuring
* that I/O is well-aligned and doesn't
* excessively cross stripe boundaries.
*
* Note that this option can substantially
* increase the size of the resulting HDF5
* files, so it is a good idea to keep an eye
* on this.
*/
H5Pset_alignment(subfiling_fapl, 0, 1048576); /* Align to custom 1MiB stripe size */
/* Parse any parallel prefix and create filename */
par_prefix = getenv("HDF5_PARAPREFIX");
snprintf(filename, sizeof(filename), "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "",
EXAMPLE_FILE3);
/* Set dataset dimensionality */
dset_dims[0] = mpi_size;
dset_dims[1] = EXAMPLE_DSET_NY;
if (mpi_rank == 0) {
/*
* Make sure only this rank opens the file
*/
H5Pset_mpi_params(subfiling_fapl, MPI_COMM_SELF, MPI_INFO_NULL);
/*
* Set the Subfiling VFD on our FAPL using
* our custom configuration
*/
H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
/*
* Create a new file on rank 0
*/
file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, subfiling_fapl);
/*
* Create the dataspace for the dataset. The first
* dimension varies with the number of MPI ranks
* while the second dimension is fixed.
*/
filespace = H5Screate_simple(EXAMPLE_DSET_DIMS, dset_dims, NULL);
/*
* Create the dataset with default properties
*/
dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, EXAMPLE_DSET_DATATYPE, filespace, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
/*
* Initialize data buffer
*/
data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
for (size_t i = 0; i < dset_dims[0] * dset_dims[1]; i++) {
data[i] = i;
}
/*
* Rank 0 writes to the whole dataset
*/
H5Dwrite(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, filespace, H5P_DEFAULT, data);
/*
* Close/release resources.
*/
free(data);
H5Dclose(dset_id);
H5Sclose(filespace);
H5Fclose(file_id);
}
MPI_Barrier(MPI_COMM_WORLD);
/*
* Use all MPI ranks to re-open the file and
* read back the dataset that was created
*/
H5Pset_mpi_params(subfiling_fapl, MPI_COMM_WORLD, MPI_INFO_NULL);
/*
* Use the same subfiling configuration as rank 0
* used to create the file
*/
H5Pset_fapl_subfiling(subfiling_fapl, &subf_config);
/*
* Re-open the file on all ranks
*/
file_id = H5Fopen(filename, H5F_ACC_RDONLY, subfiling_fapl);
/*
* Open the dataset that was created
*/
dset_id = H5Dopen2(file_id, EXAMPLE_DSET_NAME, H5P_DEFAULT);
/*
* Initialize data buffer
*/
data = malloc(dset_dims[0] * dset_dims[1] * sizeof(EXAMPLE_DSET_C_DATATYPE));
/*
* Read the dataset on all ranks
*/
H5Dread(dset_id, EXAMPLE_DSET_DATATYPE, H5S_BLOCK, H5S_ALL, H5P_DEFAULT, data);
/*
* Close/release resources.
*/
free(data);
H5Dclose(dset_id);
H5Fclose(file_id);
cleanup(EXAMPLE_FILE3, subfiling_fapl);
H5Pclose(subfiling_fapl);
}
int
main(int argc, char **argv)
{
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Info info = MPI_INFO_NULL;
hid_t fapl_id;
int mpi_size;
int mpi_rank;
int mpi_thread_required = MPI_THREAD_MULTIPLE;
int mpi_thread_provided = 0;
/* HDF5 Subfiling VFD requires MPI_Init_thread with MPI_THREAD_MULTIPLE */
MPI_Init_thread(&argc, &argv, mpi_thread_required, &mpi_thread_provided);
if (mpi_thread_provided < mpi_thread_required) {
printf("MPI_THREAD_MULTIPLE not supported\n");
MPI_Abort(comm, -1);
}
MPI_Comm_size(comm, &mpi_size);
MPI_Comm_rank(comm, &mpi_rank);
/*
* Set up File Access Property List with MPI
* parameters for the Subfiling VFD to use
*/
fapl_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_mpi_params(fapl_id, comm, info);
/* Use Subfiling VFD with default settings */
subfiling_write_default(fapl_id, mpi_size, mpi_rank);
/* Use Subfiling VFD with custom settings */
subfiling_write_custom(fapl_id, mpi_size, mpi_rank);
/*
* Use Subfiling VFD to precreate the HDF5
* file on MPI rank 0
*/
subfiling_write_precreate(fapl_id, mpi_size, mpi_rank);
H5Pclose(fapl_id);
if (mpi_rank == 0)
printf("PHDF5 example finished with no errors\n");
MPI_Finalize();
return 0;
}
#else
/* dummy program since HDF5 is not parallel-enabled */
int
main(void)
{
printf(
"Example program cannot run - HDF5 must be built with parallel support and Subfiling VFD support\n");
return 0;
}
#endif /* H5_HAVE_PARALLEL && H5_HAVE_SUBFILING_VFD */
Reference in New Issue View Git Blame Copy Permalink