mirror/hdf5
2
0
Fork 0
mirror of https://github.com/HDFGroup/hdf5.git synced 2025-01-18 15:15:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

switch to using time function instead of date function (#3690)

Browse Source
This commit is contained in:
Scot Breitenfeld 2023-10-17 11:12:16 -05:00 committed by GitHub
parent 1b62827204
commit 74fabd144e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 105 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
utils/subfiling_vfd/h5fuse.sh.in
View File

@ -13,7 +13,6 @@
BLD='\033[1m'
GRN='\033[0;32m'
RED='\033[0;31m'
PUR='\033[0;35m'
CYN='\033[0;36m'
NC='\033[0m' # No Color
@ -58,6 +57,106 @@ EOL
}
function fuse {
# function for fusing the files
mpi_rank=0
mpi_size=1
nstart=1
nend=$nsubfiles
if [ "$parallel" == "true" ]; then
hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
c_exec="h5fuse_"${hex}
c_src=${c_exec}.c
# Generate and compile an MPI program to get MPI rank and size
if [ ! -f "${c_src}" ]; then
gen_mpi
CC=@CC@
${CC} "${c_src}" -o "${c_exec}"
fi
wait
rank_size=$(./"${c_exec}")
read -r mpi_rank mpi_size <<<"$rank_size"
rm -f "${c_src}" "${c_exec}"
# Divide the subfiles among the ranks
iwork1=$(( nsubfiles / mpi_size ))
iwork2=$(( nsubfiles % mpi_size ))
min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 ))
nstart=$(( mpi_rank * iwork1 + 1 + min ))
nend=$(( nstart + iwork1 - 1 ))
if [ $iwork2 -gt "$mpi_rank" ]; then
nend=$(( nend + 1 ))
fi
fi
############################################################
# COMBINE SUBFILES INTO AN HDF5 FILE #
############################################################
icnt=1
skip=0
seek=0
seek_cnt=0
for i in "${subfiles[@]}"; do
subfile="${subfile_dir}/${i}"
# bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs
# ibs=BYTES read up to BYTES bytes at a time
# obs=BYTES write BYTES bytes at a time
# seek=N skip N obs-sized blocks at start of output
# skip=N skip N ibs-sized blocks at start of input
status=1
fsize=${subfiles_size[icnt-1]}
if [ "$fsize" -eq "0" ]; then
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
if [ "$rm_subf" == "true" ]; then
if [ -f "${subfile}" ]; then
\rm -f "$subfile"
fi
fi
else
if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then
records_left=$fsize
while [ "$status" -gt 0 ]; do
if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc"
if [ "$verbose" == "true" ]; then
echo -e "$GRN $EXEC $NC"
fi
err=$( $EXEC 2>&1 1>/dev/null )
if [ $? -ne 0 ]; then
echo -e "$CYN ERR: dd Utility Failed $NC"
echo -e "$CYN MSG: $err $NC"
exit $FAILED
fi
records_left=$((records_left-stripe_size))
skip=$((skip+1))
seek=$((seek_cnt+skip*nsubfiles))
else
status=0
skip=0
fi
done; wait
if [ "$rm_subf" == "true" ]; then
\rm -f "$subfile"
fi
fi
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
fi
icnt=$(( icnt +1 ))
done; wait
}
############################################################
############################################################
# Main program #
@ -166,104 +265,9 @@ for i in "${subfiles[@]}"; do
fi
done
START="$(date +%s%N)"
mpi_rank=0
mpi_size=1
nstart=1
nend=$nsubfiles
if [ "$parallel" == "true" ]; then
hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
c_exec="h5fuse_"${hex}
c_src=${c_exec}.c
# Generate and compile an MPI program to get MPI rank and size
if [ ! -f "${c_src}" ]; then
gen_mpi
CC=@CC@
${CC} "${c_src}" -o "${c_exec}"
fi
wait
rank_size=$(./"${c_exec}")
read -r mpi_rank mpi_size <<<"$rank_size"
rm -f "${c_src}" "${c_exec}"
# Divide the subfiles among the ranks
iwork1=$(( nsubfiles / mpi_size ))
iwork2=$(( nsubfiles % mpi_size ))
min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 ))
nstart=$(( mpi_rank * iwork1 + 1 + min ))
nend=$(( nstart + iwork1 - 1 ))
if [ $iwork2 -gt "$mpi_rank" ]; then
nend=$(( nend + 1 ))
fi
fi
############################################################
# COMBINE SUBFILES INTO AN HDF5 FILE #
############################################################
icnt=1
skip=0
seek=0
seek_cnt=0
for i in "${subfiles[@]}"; do
subfile="${subfile_dir}/${i}"
# bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs
# ibs=BYTES read up to BYTES bytes at a time
# obs=BYTES write BYTES bytes at a time
# seek=N skip N obs-sized blocks at start of output
# skip=N skip N ibs-sized blocks at start of input
status=1
fsize=${subfiles_size[icnt-1]}
if [ "$fsize" -eq "0" ]; then
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
if [ "$rm_subf" == "true" ]; then
if [ -f "${subfile}" ]; then
\rm -f "$subfile"
fi
fi
else
if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then
records_left=$fsize
while [ "$status" -gt 0 ]; do
if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc"
if [ "$verbose" == "true" ]; then
echo -e "$GRN $EXEC $NC"
fi
err=$( $EXEC 2>&1 1>/dev/null )
if [ $? -ne 0 ]; then
echo -e "$CYN ERR: dd Utility Failed $NC"
echo -e "$CYN MSG: $err $NC"
exit $FAILED
fi
records_left=$((records_left-stripe_size))
skip=$((skip+1))
seek=$((seek_cnt+skip*nsubfiles))
else
status=0
skip=0
fi
done; wait
if [ "$rm_subf" == "true" ]; then
\rm -f "$subfile"
fi
fi
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
fi
icnt=$(( icnt +1 ))
done; wait
END=$(( $(date +%s%N) - START ))
DURATION_SEC=$(awk -vp="$END" -vq=0.000000001 'BEGIN{printf "%.4f" ,p * q}')
if [ "$quiet" == "false" ]; then
echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC"
fi
TIMEFORMAT="COMPLETION TIME = %R s"
time fuse
else
fuse
fi