Skip to content

Commit

Permalink
Merge remote-tracking branch 'canonical/feature/vfd_swmr' into vfd_sw…
Browse files Browse the repository at this point in the history 
…mr/thg_standards
  • Loading branch information
@derobins
derobins committed May 3, 2021
2 parents a2f3915 + 4442691 commit c83ca64
Show file tree
Hide file tree
Showing 9 changed files with 5,539 additions and 517 deletions.
4 changes: 4 additions & 0 deletions test/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \
vfd_swmr_group_reader vfd_swmr_group_writer \
vfd_swmr_vlstr_reader vfd_swmr_vlstr_writer \
vfd_swmr_zoo_reader vfd_swmr_zoo_writer \
vfd_swmr_attrdset_reader vfd_swmr_attrdset_writer \
swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \
mirror_vfd
if HAVE_SHARED_CONDITIONAL
Expand Down Expand Up @@ -173,6 +174,9 @@ vfd_swmr_zoo_reader_SOURCES=vfd_swmr_zoo_writer.c genall5.c
vfd_swmr_bigset_reader_SOURCES=vfd_swmr_bigset_writer.c
vfd_swmr_group_reader_SOURCES=vfd_swmr_group_writer.c

vfd_swmr_attrdset_writer_SOURCES=vfd_swmr_attrdset_writer.c
vfd_swmr_attrdset_reader_SOURCES=vfd_swmr_attrdset_writer.c

# Additional target for running timing test
timings _timings: testmeta
@for timing in $(TIMINGS) dummy; do \
Expand Down
17 changes: 10 additions & 7 deletions test/genall5.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
* of the same name.
*/

#include <err.h>
#include "cache_common.h"
#include "vfd_swmr_common.h" /* for below_speed_limit() */
#include "genall5.h"
Expand Down Expand Up @@ -412,7 +413,7 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks)
gid = H5Gopen2(fid, group_name, H5P_DEFAULT);

if (gid <= 0) {
failure_mssg = "vrfy_ns_grp_c: H5Gopen2() failed";
failure_mssg = "vrfy_ns_grp_c: H5Gopen2 failed";
return false;
}

Expand Down Expand Up @@ -2753,12 +2754,14 @@ tend_zoo(hid_t fid, const char *base_path, struct timespec *lastmsgtime, zoo_con
}
out:
if (!ok) {
if (HDstrcmp(failure_mssg, last_failure_mssg) != 0)
*lastmsgtime = (struct timespec){.tv_sec = 0, .tv_nsec = 0};

if (below_speed_limit(lastmsgtime, &config.msgival)) {
last_failure_mssg = failure_mssg;
HDfprintf(stderr, "%s: %s", __func__, failure_mssg);
/* Currently not used: this step makes sure the operation doesn't take too long.
* Any test that sets config.msgival or lastmsgtime to 0 will skip this step */
if (strcmp(failure_mssg, last_failure_mssg) != 0 && ((config.msgival.tv_sec || config.msgival.tv_nsec))
&& (lastmsgtime->tv_sec || lastmsgtime->tv_nsec)) {
if (below_speed_limit(lastmsgtime, &config.msgival)) {
last_failure_mssg = failure_mssg;
warnx("%s: %s", __func__, failure_mssg);
}
}
}
return ok;
Expand Down
230 changes: 216 additions & 14 deletions test/testvfdswmr.sh.in
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /bin/bash
#!/usr/bin/env bash
#
# Copyright by The HDF Group.
# Copyright by the Board of Trustees of the University of Illinois.
Expand Down Expand Up @@ -46,25 +46,28 @@ nsofterrors=0 # soft errors are expected to occur some of the time
## 1: Default run.
## 2+: Quick run
###############################################################################
if [ -z $HDF5TestExpress ]; then # Set to default when not set
if [[ -z $HDF5TestExpress ]]; then # Set to default when not set
HDF5TestExpress=1
fi
##
##Default setting
BIGSET_n=25 # -n option: # of iterations
BIGSET_few_s=20 # -s option: # of datasets (for few_big test)
BIGSET_many_s=500 # -s option: # of datasets (for many_small test)
GROUP_n=100 # -n option: # of groups (for vfd_swmr_group_writer.c)
GROUP_n=40 # -n option: # of groups (for group test)
GROUP_attr_n=1 # -n option: # of groups (for group attribute test)

if [[ "$HDF5TestExpress" -eq 0 ]] ; then # Setting for exhaustive run
BIGSET_n=50
BIGSET_few_s=40
BIGSET_many_s=1000
GROUP_n=400
GROUP_attr_n=4
elif [[ "$HDF5TestExpress" -gt 1 ]]; then # Setting for quick run
BIGSET_n=10
BIGSET_few_s=10
BIGSET_many_s=100
GROUP_n=40
GROUP_n=20
fi

###############################################################################
Expand All @@ -74,6 +77,12 @@ WRITER_MESSAGE=VFD_SWMR_WRITER_MESSAGE # The message file created by writer tha
# This should be the same as the define in "./swmr_common.h"
MESSAGE_TIMEOUT=300 # Message timeout length in secs
# This should be the same as the define in "./h5test.h"

###############################################################################
## For attrdset test: definitions for fifo files to coordinate test runs
###############################################################################
FIFO_WRITER_TO_READER=fifo_attrdset_writer_to_reader
FIFO_READER_TO_WRITER=fifo_attrdset_reader_to_writer

###############################################################################
## short hands and function definitions
Expand Down Expand Up @@ -146,8 +155,9 @@ if [ $rc -ne 0 ] ; then
exit 0
fi

#all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups attrdset"
all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups"
all_tests="${all_tests} few_big many_small"
all_tests="${all_tests} groups_attrs os_groups_attrs few_big many_small"
tests=${all_tests}

if [ $# -gt 0 ]; then
Expand Down Expand Up @@ -191,6 +201,7 @@ mkdir vfd_swmr_test

cd vfd_swmr_test


# Loop over index types
for index_type in "-i ea" "-i b2"
do
Expand Down Expand Up @@ -575,24 +586,23 @@ done
# read and written by VFD SWMR.
#
if [ ${do_zoo:-no} = yes ]; then
[ -e ./fifo ] && rm -f ./fifo
mkfifo -m 0600 ./fifo
rm -f ./shared_tick_num
echo launch vfd_swmr_zoo_writer
STDIN_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_writer \
../vfd_swmr_zoo_writer -m 1000 -q &
catch_out_err_and_rc vfd_swmr_zoo_writer \
../vfd_swmr_zoo_writer -q &
pid_writer=$!

STDOUT_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_reader \
../vfd_swmr_zoo_reader -q -W &
# -l is the expected maximal number of ticks from the writer's finishing zoo creation or deletion
# to the reader's finishing validation of zoo creation or deletion
catch_out_err_and_rc vfd_swmr_zoo_reader \
../vfd_swmr_zoo_reader -l 4 -q &
pid_reader=$!

# Wait for the reader to finish before signalling the
# writer to quit: the writer holds the file open so that the
# reader will find the shadow file when it opens
# the .h5 file.
wait $pid_reader
kill -USR1 $(cat vfd_swmr_zoo_writer.pid)
wait $pid_writer

# Collect exit code of the reader
Expand All @@ -608,13 +618,61 @@ if [ ${do_zoo:-no} = yes ]; then
fi

# Clean up output files
rm -f ./fifo
rm -f vfd_swmr_zoo_writer.{out,rc}
rm -f vfd_swmr_zoo_reader.*.{out,rc}
fi

# attrdset test
#for options in "-p -g -a 10 -v -m -d 10 -c 3 -u 5" "-k -a 20 -v -m -d 5"; do
# #
# # Test a few big datasets of one and two dimensions.
# #
# if [ ${do_attrdset:-no} = no ]; then
# continue
# fi
# # Clean up any existing fifo files from previous runs
# if [ -e ./$FIFO_WRITER_TO_READER ]; then # If writer fifo file is found
# rm -f ./$FIFO_WRITER_TO_READER
# fi
# if [ -e ./$FIFO_READER_TO_WRITER ]; then # If reader fifo file is found
# rm -f ./$FIFO_READER_TO_WRITER
# fi
# #
# echo launch vfd_swmr_attrdset_writer attrdset, options $options
# catch_out_err_and_rc vfd_swmr_attrdset_writer \
# ../vfd_swmr_attrdset_writer $options &
# pid_writer=$!
#
# catch_out_err_and_rc vfd_swmr_attrdset_reader \
# ../vfd_swmr_attrdset_reader $options &
# pid_reader=$!
#
# # Wait for the reader to finish before signaling the
# # writer to quit: the writer holds the file open so that the
# # reader will find the shadow file when it opens
# # the .h5 file.
# wait $pid_reader
# wait $pid_writer
#
# Make sure that we can create GROUP_n groups (40, 100, or 400 depending on the HDF5TestExpress level)
# # Collect exit code of the reader
# if [ $(cat vfd_swmr_attrdset_reader.rc) -ne 0 ]; then
# echo reader had error
# nerrors=$((nerrors + 1))
# fi
#
# # Collect exit code of the writer
# if [ $(cat vfd_swmr_attrdset_writer.rc) -ne 0 ]; then
# echo writer had error
# nerrors=$((nerrors + 1))
# fi
#
# # Clean up output files
# rm -f vfd_swmr_attrdset_writer.{out,rc}
# rm -f vfd_swmr_attrdset_reader.*.{out,rc}
#done

#
# Make sure that we can create GROUP_n groups (20, 40, or 400 depending on the HDF5TestExpress level)
# while a reader waits for each to appear.
#
if [ ${do_groups:-no} = yes ]; then
Expand Down Expand Up @@ -651,6 +709,150 @@ if [ ${do_groups:-no} = yes ]; then
rm -f vfd_swmr_group_reader.*.{out,rc}
fi

# The group attribute test takes longer.
# So for standard run and quick run, we
# shorten the number of tests. The standard
# run covers all the features we need to
# test. The quick run doesn't cover the
# attribute storage change between dense and
# compact.
# The exhaustive run tries to test a feature
# per test from scratch.
#
grp_attr_list=(
"compact"
"dense"
"compact-del"
"dense-del"
"compact-add-to-dense"
"dense-del-to-compact"
"modify"
"add-vstr"
"remove-vstr"
"modify-vstr"
)
grp_sub_attr_list=(
"dense-del-to-compact"
"modify"
"remove-vstr"
"modify-vstr"
)

grp_short_sub_attr_list=(
"dense"
"modify"
"remove-vstr"
"modify-vstr"
)

if [[ "$HDF5TestExpress" -eq 1 ]] ; then #Setting for standard run
grp_attr_list=("${grp_sub_attr_list[@]}")
elif [[ "$HDF5TestExpress" -gt 1 ]] ; then #Setting for quick run
grp_attr_list=("${grp_short_sub_attr_list[@]}")
fi

for options in ${grp_attr_list[*]}; do
if [ ${do_groups_attrs:-no} = no ]; then
continue
fi
echo launch vfd_swmr_group attribute: $options
catch_out_err_and_rc vfd_swmr_group_writer \
../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options &
pid_writer=$!

catch_out_err_and_rc vfd_swmr_group_reader \
../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options &
pid_reader=$!

# Wait for the reader to finish before signalling the
# writer to quit: the writer holds the file open so that the
# reader will find the shadow file when it opens
# the .h5 file.
wait $pid_reader
wait $pid_writer

# Collect exit code of the reader
if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
echo reader had error
nerrors=$((nerrors + 1))
fi

# Collect exit code of the writer
if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
echo writer had error
nerrors=$((nerrors + 1))
fi

# Clean up output files
rm -f vfd_swmr_group_writer.{out,rc}
rm -f vfd_swmr_group_reader.*.{out,rc}
done

# The following tests are for add/del/modify attributes for
# groups created with the old-style.
# Check https://portal.hdfgroup.org/display/HDF5/Groups for
# the detailed group implementation note.
# The 'compact' and 'compact-del' are the attribute addition
# and deletion tests. Other test names have the same meaning
# as those of the new-style group tests.
#
os_grp_attr_list=(
"compact"
"compact-del"
"modify"
"add-vstr"
"remove-vstr"
"modify-vstr"
)
os_grp_sub_attr_list=(
"modify"
"remove-vstr"
"modify-vstr"
)
if [[ "$HDF5TestExpress" -gt 0 ]] ; then #Setting for standard run
os_grp_attr_list=("${os_grp_sub_attr_list[@]}")
fi

for options in ${os_grp_attr_list[*]}; do
if [ ${do_os_groups_attrs:-no} = no ]; then
continue
fi
echo launch vfd_swmr_group attribute with old-style group: $options
catch_out_err_and_rc vfd_swmr_group_writer \
../vfd_swmr_group_writer -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options &
pid_writer=$!

catch_out_err_and_rc vfd_swmr_group_reader \
../vfd_swmr_group_reader -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options &
pid_reader=$!

# Wait for the reader to finish before signalling the
# writer to quit: the writer holds the file open so that the
# reader will find the shadow file when it opens
# the .h5 file.
wait $pid_reader
wait $pid_writer

# Collect exit code of the reader
if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then
echo reader had error
nerrors=$((nerrors + 1))
fi

# Collect exit code of the writer
if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then
echo writer had error
nerrors=$((nerrors + 1))
fi

# Clean up output files
rm -f vfd_swmr_group_writer.{out,rc}
rm -f vfd_swmr_group_reader.*.{out,rc}
done




for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do
if [ ${do_many_small:-no} = no ]; then
continue
Expand Down
9 changes: 0 additions & 9 deletions test/vfd_swmr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -848,15 +848,6 @@ test_writer_create_open_flush(void)
return 1;
} /* test_writer_create_open_flush() */

/* Sleep for `tenths` tenths of a second */
static void
decisleep(uint32_t tenths)
{
uint64_t nsec = tenths * 100 * 1000 * 1000;

H5_nanosleep(nsec);
}

/*-------------------------------------------------------------------------
* Function: test_writer_md()
*
Expand Down
Loading

0 comments on commit c83ca64

Please sign in to comment.