mirror of
https://github.com/HDFGroup/hdf5.git
synced 2025-02-17 16:10:24 +08:00
H5fuse.sh optimization updates (#2806)
Changed to processing subfiles at the subfile level. Simplified parameter arguments. Enabled running it in parallel. Added option to specify subfiling configuration location.
This commit is contained in:
Scot Breitenfeld
GitHub
parent
81bb90e77b
commit
7973707c86
@ -14,6 +14,7 @@ BLD='\033[1m'
|
||||
GRN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
PUR='\033[0;35m'
|
||||
CYN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
############################################################
|
||||
@ -23,15 +24,40 @@ function usage() {
|
||||
echo ""
|
||||
# Display usage
|
||||
echo "Purpose: Combine subfiles into a single HDF5 file. Requires the subfiling
|
||||
configuration file either as a command-line argument, or the script will
|
||||
configuration file either as a command-line argument or the script will
|
||||
search for the *.config file in the current directory."
|
||||
echo ""
|
||||
echo "usage: h5fuse.sh [-h] [-f filename]"
|
||||
echo "-h Print this help."
|
||||
echo "-f filename Subfile configuration file."
|
||||
echo "usage: h5fuse.sh [-f filename] [-h] [-p] [-q] [-r] [-v] "
|
||||
echo "-f filename Subfile configuration file."
|
||||
echo "-h Print this help."
|
||||
echo "-q Quiet all output. [no]"
|
||||
echo "-p h5fuse.sh is being run in parallel, with more than one rank. [no]"
|
||||
echo "-r Remove subfiles after being processed. [no]"
|
||||
echo "-v Verbose output. [no]"
|
||||
echo ""
|
||||
}
|
||||
|
||||
function gen_mpi() {
|
||||
|
||||
# Program to determine MPI rank and size if being run in parallel (-p).
|
||||
|
||||
cat > "${c_src}" << EOL
|
||||
#include <mpi.h>
|
||||
#include <stdio.h>
|
||||
int main() {
|
||||
MPI_Init(NULL, NULL);
|
||||
int world_size;
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
|
||||
int world_rank;
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
|
||||
printf("%d %d", world_rank, world_size);
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Finalize();
|
||||
}
|
||||
EOL
|
||||
|
||||
}
|
||||
|
||||
############################################################
|
||||
############################################################
|
||||
# Main program #
|
||||
@ -43,14 +69,25 @@ function usage() {
|
||||
############################################################
|
||||
# Get the options
|
||||
file_config=""
|
||||
|
||||
while getopts ":h:f:" option; do
|
||||
verbose="false"
|
||||
quiet="false"
|
||||
rm_subf="false"
|
||||
parallel="false"
|
||||
while getopts "hpqrvf:" option; do
|
||||
case $option in
|
||||
f) # subfiling configuration file
|
||||
file_config=$OPTARG;;
|
||||
h) # display Help
|
||||
usage
|
||||
exit;;
|
||||
f) # subfiling configuration file
|
||||
file_config=$OPTARG;;
|
||||
p) # HDF5 fused file
|
||||
parallel="true";;
|
||||
q) # quiet all output
|
||||
quiet="true";;
|
||||
r) # remove completed subfiles
|
||||
rm_subf="true";;
|
||||
v) # verbose output
|
||||
verbose="true";;
|
||||
\?) # Invalid option
|
||||
echo -e "$RED ERROR: Invalid option ${BLD}-${OPTARG}${RED} $NC"
|
||||
usage
|
||||
@ -61,12 +98,20 @@ while getopts ":h:f:" option; do
|
||||
done
|
||||
|
||||
FAILED=1
|
||||
nfiles=1
|
||||
############################################################
|
||||
# Configure file checks #
|
||||
############################################################
|
||||
#
|
||||
SUBF_CONFDIR="${H5FD_SUBFILING_CONFIG_FILE_PREFIX:-.}"
|
||||
|
||||
# cd to the subfile configuration location
|
||||
if [ "$SUBF_CONFDIR" != "." ] || [ "$SUBF_CONFDIR" != "$PWD" ]; then
|
||||
cd "$SUBF_CONFDIR" || exit
|
||||
fi
|
||||
|
||||
# Try to find the config file
|
||||
if [ -z "$file_config" ]; then
|
||||
nfiles=$(find . -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m)
|
||||
nfiles=$(find "$SUBF_CONFDIR" -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m)
|
||||
if [[ "$nfiles" != "1" ]]; then
|
||||
if [[ "$nfiles" == "0" ]]; then
|
||||
echo -e "$RED Failed to find .config file in current directory. $NC"
|
||||
@ -78,7 +123,7 @@ if [ -z "$file_config" ]; then
|
||||
exit $FAILED
|
||||
fi
|
||||
fi
|
||||
file_config=$(find . -maxdepth 1 -type f -iname "*.config")
|
||||
file_config=$(find "$SUBF_CONFDIR" -maxdepth 1 -type f -iname '*.config')
|
||||
fi
|
||||
|
||||
if [ ! -f "$file_config" ]; then
|
||||
@ -104,62 +149,125 @@ if test -z "$subfile_dir"; then
|
||||
exit $FAILED
|
||||
fi
|
||||
|
||||
subfiles=( $( sed -e '1,/subfile_dir=/d' "$file_config" ) )
|
||||
#for i in "${subfiles[@]}"; do
|
||||
# echo "$i"
|
||||
#done
|
||||
# For bash 4.4+
|
||||
mapfile -t subfiles < <( sed -e '1,/subfile_dir=/d' "$file_config" )
|
||||
if [ ${#subfiles[@]} -eq 0 ]; then
|
||||
echo -e "$RED failed to find subfiles list in $file_config $NC"
|
||||
exit $FAILED
|
||||
fi
|
||||
nsubfiles=${#subfiles[@]}
|
||||
|
||||
rm -f "$hdf5_file"
|
||||
|
||||
## COMBINE SUBFILES INTO AN HDF5 FILE ##
|
||||
|
||||
skip=0
|
||||
status=$nfiles
|
||||
START="$(date +%s%N)"
|
||||
while [ "$status" -gt 0 ]; do
|
||||
icnt=0
|
||||
for i in "${subfiles[@]}"; do
|
||||
subfile="${subfile_dir}/${i}"
|
||||
# Verify the file exists
|
||||
if [ ! -f "${subfile}" ]; then
|
||||
echo -e "$RED ERROR: file \"${subfile}\" does not exist. $NC"
|
||||
exit $FAILED
|
||||
fi
|
||||
|
||||
# Verify the file is not being accessed by a process
|
||||
t_max=60
|
||||
t_sleep=1
|
||||
t_elapsed=0
|
||||
|
||||
while fuser -s "${subfile}"; do
|
||||
if [[ $((t_elapsed % 5)) -eq 0 ]]; then
|
||||
echo -e "$GRN waiting for process to finish accessing file \"${subfile}\" ... [${t_elapsed}s/${t_max}s] $NC"
|
||||
fi
|
||||
sleep $t_sleep
|
||||
t_elapsed=$((t_elapsed+t_sleep))
|
||||
if [[ $t_elapsed -ge $t_max ]]; then
|
||||
echo -e "$RED ERROR: file \"${subfile}\" still has process accessing it after ${t_elapsed}s $NC"
|
||||
exit $FAILED
|
||||
fi
|
||||
done
|
||||
|
||||
fsize=$(wc -c "${subfile}" | awk '{print $1}')
|
||||
if [ $((skip*stripe_size)) -le "$fsize" ]; then
|
||||
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip oflag=append conv=notrunc"
|
||||
echo -e "$GRN $EXEC $NC"
|
||||
err="$( $EXEC 2>&1 > /dev/null &)"
|
||||
icnt=$((icnt+1))
|
||||
else
|
||||
subfiles=("${subfiles[@]:0:icnt}" "${subfiles[@]:$((icnt+1))}")
|
||||
status=${#subfiles[@]}
|
||||
fi
|
||||
done; wait
|
||||
skip=$((skip+1))
|
||||
# Get the number of local subfiles
|
||||
subfiles_loc=()
|
||||
subfiles_size=()
|
||||
for i in "${subfiles[@]}"; do
|
||||
subfile="${subfile_dir}/${i}"
|
||||
if [ -f "${subfile}" ]; then
|
||||
subfiles_loc+=("$subfile")
|
||||
subfiles_size+=($(wc -c "${subfile}" | awk '{print $1}'))
|
||||
else
|
||||
subfiles_size+=(0)
|
||||
fi
|
||||
done
|
||||
|
||||
START="$(date +%s%N)"
|
||||
|
||||
mpi_rank=0
|
||||
mpi_size=1
|
||||
nstart=1
|
||||
nend=$nsubfiles
|
||||
|
||||
if [ "$parallel" == "true" ]; then
|
||||
|
||||
hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
|
||||
c_exec="h5fuse_"${hex}
|
||||
c_src=${c_exec}.c
|
||||
|
||||
# Generate and compile an MPI program to get MPI rank and size
|
||||
if [ ! -f "${c_src}" ]; then
|
||||
gen_mpi
|
||||
CC=@CC@
|
||||
${CC} "${c_src}" -o "${c_exec}"
|
||||
fi
|
||||
wait
|
||||
rank_size=$(./"${c_exec}")
|
||||
read -r mpi_rank mpi_size <<<"$rank_size"
|
||||
|
||||
rm -f "${c_src}" "${c_exec}"
|
||||
|
||||
# Divide the subfiles among the ranks
|
||||
iwork1=$(( nsubfiles / mpi_size ))
|
||||
iwork2=$(( nsubfiles % mpi_size ))
|
||||
min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 ))
|
||||
nstart=$(( mpi_rank * iwork1 + 1 + min ))
|
||||
nend=$(( nstart + iwork1 - 1 ))
|
||||
if [ $iwork2 -gt "$mpi_rank" ]; then
|
||||
nend=$(( nend + 1 ))
|
||||
fi
|
||||
fi
|
||||
|
||||
############################################################
|
||||
# COMBINE SUBFILES INTO AN HDF5 FILE #
|
||||
############################################################
|
||||
icnt=1
|
||||
skip=0
|
||||
seek=0
|
||||
seek_cnt=0
|
||||
for i in "${subfiles[@]}"; do
|
||||
|
||||
subfile="${subfile_dir}/${i}"
|
||||
|
||||
# bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs
|
||||
# ibs=BYTES read up to BYTES bytes at a time
|
||||
# obs=BYTES write BYTES bytes at a time
|
||||
# seek=N skip N obs-sized blocks at start of output
|
||||
# skip=N skip N ibs-sized blocks at start of input
|
||||
|
||||
status=1
|
||||
fsize=${subfiles_size[icnt-1]}
|
||||
if [ "$fsize" -eq "0" ]; then
|
||||
seek_cnt=$((seek_cnt+1))
|
||||
seek=$seek_cnt
|
||||
if [ "$rm_subf" == "true" ]; then
|
||||
if [ -f "${subfile}" ]; then
|
||||
\rm -f "$subfile"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then
|
||||
records_left=$fsize
|
||||
while [ "$status" -gt 0 ]; do
|
||||
if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then
|
||||
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc"
|
||||
if [ "$verbose" == "true" ]; then
|
||||
echo -e "$GRN $EXEC $NC"
|
||||
fi
|
||||
err=$( $EXEC 2>&1 1>/dev/null )
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "$CYN ERR: dd Utility Failed $NC"
|
||||
echo -e "$CYN MSG: $err $NC"
|
||||
exit $FAILED
|
||||
fi
|
||||
records_left=$((records_left-stripe_size))
|
||||
skip=$((skip+1))
|
||||
seek=$((seek_cnt+skip*nsubfiles))
|
||||
else
|
||||
status=0
|
||||
skip=0
|
||||
fi
|
||||
done; wait
|
||||
if [ "$rm_subf" == "true" ]; then
|
||||
\rm -f "$subfile"
|
||||
fi
|
||||
fi
|
||||
seek_cnt=$((seek_cnt+1))
|
||||
seek=$seek_cnt
|
||||
fi
|
||||
icnt=$(( icnt +1 ))
|
||||
done; wait
|
||||
|
||||
END=$(( $(date +%s%N) - START ))
|
||||
DURATION_SEC=$(awk -vp="$END" -vq=0.000000001 'BEGIN{printf "%.4f" ,p * q}')
|
||||
echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC"
|
||||
if [ "$quiet" == "false" ]; then
|
||||
echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC"
|
||||
fi
|
||||
Loading…
Reference in New Issue
Block a user