Skip to content

Commit

Permalink
switch to using time function instead of date function (HDFGroup#3690)
Browse files Browse the repository at this point in the history
  • Loading branch information
@brtnfld @jhendersonHDF
brtnfld authored and jhendersonHDF committed Oct 18, 2023
1 parent c3b8078 commit 72a5f2b
Showing 1 changed file with 105 additions and 101 deletions.
206 changes: 105 additions & 101 deletions utils/subfiling_vfd/h5fuse.sh.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
BLD='\033[1m'
GRN='\033[0;32m'
RED='\033[0;31m'
PUR='\033[0;35m'
CYN='\033[0;36m'
NC='\033[0m' # No Color

Expand Down Expand Up @@ -58,6 +57,106 @@ EOL

}

function fuse {

# function for fusing the files

mpi_rank=0
mpi_size=1
nstart=1
nend=$nsubfiles

if [ "$parallel" == "true" ]; then

hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
c_exec="h5fuse_"${hex}
c_src=${c_exec}.c

# Generate and compile an MPI program to get MPI rank and size
if [ ! -f "${c_src}" ]; then
gen_mpi
CC=@CC@
${CC} "${c_src}" -o "${c_exec}"
fi
wait
rank_size=$(./"${c_exec}")
read -r mpi_rank mpi_size <<<"$rank_size"

rm -f "${c_src}" "${c_exec}"

# Divide the subfiles among the ranks
iwork1=$(( nsubfiles / mpi_size ))
iwork2=$(( nsubfiles % mpi_size ))
min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 ))
nstart=$(( mpi_rank * iwork1 + 1 + min ))
nend=$(( nstart + iwork1 - 1 ))
if [ $iwork2 -gt "$mpi_rank" ]; then
nend=$(( nend + 1 ))
fi
fi

############################################################
# COMBINE SUBFILES INTO AN HDF5 FILE #
############################################################
icnt=1
skip=0
seek=0
seek_cnt=0
for i in "${subfiles[@]}"; do

subfile="${subfile_dir}/${i}"

# bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs
# ibs=BYTES read up to BYTES bytes at a time
# obs=BYTES write BYTES bytes at a time
# seek=N skip N obs-sized blocks at start of output
# skip=N skip N ibs-sized blocks at start of input

status=1
fsize=${subfiles_size[icnt-1]}
if [ "$fsize" -eq "0" ]; then
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
if [ "$rm_subf" == "true" ]; then
if [ -f "${subfile}" ]; then
\rm -f "$subfile"
fi
fi
else
if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then
records_left=$fsize
while [ "$status" -gt 0 ]; do
if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc"
if [ "$verbose" == "true" ]; then
echo -e "$GRN $EXEC $NC"
fi
err=$( $EXEC 2>&1 1>/dev/null )
if [ $? -ne 0 ]; then
echo -e "$CYN ERR: dd Utility Failed $NC"
echo -e "$CYN MSG: $err $NC"
exit $FAILED
fi
records_left=$((records_left-stripe_size))
skip=$((skip+1))
seek=$((seek_cnt+skip*nsubfiles))
else
status=0
skip=0
fi
done; wait
if [ "$rm_subf" == "true" ]; then
\rm -f "$subfile"
fi
fi
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
fi
icnt=$(( icnt +1 ))
done; wait

}

############################################################
############################################################
# Main program #
Expand Down Expand Up @@ -166,104 +265,9 @@ for i in "${subfiles[@]}"; do
fi
done

START="$(date +%s%N)"

mpi_rank=0
mpi_size=1
nstart=1
nend=$nsubfiles

if [ "$parallel" == "true" ]; then

hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom)
c_exec="h5fuse_"${hex}
c_src=${c_exec}.c

# Generate and compile an MPI program to get MPI rank and size
if [ ! -f "${c_src}" ]; then
gen_mpi
CC=@CC@
${CC} "${c_src}" -o "${c_exec}"
fi
wait
rank_size=$(./"${c_exec}")
read -r mpi_rank mpi_size <<<"$rank_size"

rm -f "${c_src}" "${c_exec}"

# Divide the subfiles among the ranks
iwork1=$(( nsubfiles / mpi_size ))
iwork2=$(( nsubfiles % mpi_size ))
min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 ))
nstart=$(( mpi_rank * iwork1 + 1 + min ))
nend=$(( nstart + iwork1 - 1 ))
if [ $iwork2 -gt "$mpi_rank" ]; then
nend=$(( nend + 1 ))
fi
fi

############################################################
# COMBINE SUBFILES INTO AN HDF5 FILE #
############################################################
icnt=1
skip=0
seek=0
seek_cnt=0
for i in "${subfiles[@]}"; do

subfile="${subfile_dir}/${i}"

# bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs
# ibs=BYTES read up to BYTES bytes at a time
# obs=BYTES write BYTES bytes at a time
# seek=N skip N obs-sized blocks at start of output
# skip=N skip N ibs-sized blocks at start of input

status=1
fsize=${subfiles_size[icnt-1]}
if [ "$fsize" -eq "0" ]; then
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
if [ "$rm_subf" == "true" ]; then
if [ -f "${subfile}" ]; then
\rm -f "$subfile"
fi
fi
else
if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then
records_left=$fsize
while [ "$status" -gt 0 ]; do
if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then
EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc"
if [ "$verbose" == "true" ]; then
echo -e "$GRN $EXEC $NC"
fi
err=$( $EXEC 2>&1 1>/dev/null )
if [ $? -ne 0 ]; then
echo -e "$CYN ERR: dd Utility Failed $NC"
echo -e "$CYN MSG: $err $NC"
exit $FAILED
fi
records_left=$((records_left-stripe_size))
skip=$((skip+1))
seek=$((seek_cnt+skip*nsubfiles))
else
status=0
skip=0
fi
done; wait
if [ "$rm_subf" == "true" ]; then
\rm -f "$subfile"
fi
fi
seek_cnt=$((seek_cnt+1))
seek=$seek_cnt
fi
icnt=$(( icnt +1 ))
done; wait

END=$(( $(date +%s%N) - START ))
DURATION_SEC=$(awk -vp="$END" -vq=0.000000001 'BEGIN{printf "%.4f" ,p * q}')
if [ "$quiet" == "false" ]; then
echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC"
fi
TIMEFORMAT="COMPLETION TIME = %R s"
time fuse
else
fuse
fi

0 comments on commit 72a5f2b

Please sign in to comment.