From b63e2d881f859f0c7d8596be2759dd096e324f48 Mon Sep 17 00:00:00 2001 From: John Wren Kennedy Date: Mon, 9 Sep 2019 17:11:07 -0600 Subject: [PATCH] ZTS: Introduce targeted corruption in file blocks filetest_001_pos verifies that various checksum algorithms detect corruption by overwriting the underlying vdev on which a file resides. It is possible for the overwrite to miss the blocks of a file, causing a spurious failure. This change introduces a function to corrupt the individual blocks of a file as determined by zdb. Reviewed-by: Brian Behlendorf Reviewed-by: Ryan Moeller Signed-off-by: John Kennedy Closes #9288 --- tests/zfs-tests/include/blkdev.shlib | 88 ++++++++++++++++++- .../functional/checksum/filetest_001_pos.ksh | 27 ++---- 2 files changed, 96 insertions(+), 19 deletions(-) diff --git a/tests/zfs-tests/include/blkdev.shlib b/tests/zfs-tests/include/blkdev.shlib index 87500e92a398..af3324683b0d 100644 --- a/tests/zfs-tests/include/blkdev.shlib +++ b/tests/zfs-tests/include/blkdev.shlib @@ -12,7 +12,7 @@ # # Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. -# Copyright (c) 2012, 2016 by Delphix. All rights reserved. +# Copyright (c) 2012, 2019 by Delphix. All rights reserved. # Copyright 2016 Nexenta Systems, Inc. # Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved. # Copyright (c) 2017 Lawrence Livermore National Security, LLC. @@ -465,3 +465,89 @@ function get_pool_devices #testpool #devdir fi echo $out } + +# +# Write to standard out giving the level, device name, offset and length +# of all blocks in an input file. The offset and length are in units of +# 512 byte blocks. In the case of mirrored vdevs, only the first +# device is listed, as the levels, blocks and offsets will be the same +# on other devices. Note that this function only works with mirrored +# or non-redundant pools, not raidz. +# +# The output of this function can be used to introduce corruption at +# varying levels of indirection. +# +function list_file_blocks # input_file +{ + typeset input_file=$1 + + [[ -f $input_file ]] || log_fail "Couldn't find $input_file" + + typeset ds="$(zfs list -H -o name $input_file)" + typeset pool="${ds%%/*}" + typeset inum="$(stat -c '%i' $input_file)" + + # + # Establish a mapping between vdev ids as shown in a DVA and the + # pathnames they correspond to in ${VDEV_MAP[]}. + # + eval $(zdb -C $pool | awk ' + BEGIN { + printf("typeset VDEV_MAP\n"); + looking = 0; + } + /^ children/ { + id = $1; + looking = 1; + } + /path: / && looking == 1 { + print id" "$2; + looking = 0; + } + ' | sed -n 's/^children\[\([0-9]\)\]: \(.*\)$/VDEV_MAP[\1]=\2/p') + + # + # The awk below parses the output of zdb, printing out the level + # of each block along with vdev id, offset and length. The last + # two are converted to decimal in the while loop. 4M is added to + # the offset to compensate for the first two labels and boot + # block. Lastly, the offset and length are printed in units of + # 512b blocks for ease of use with dd. + # + log_must zpool sync -f + typeset level path offset length + zdb -ddddd $ds $inum | awk -F: ' + BEGIN { looking = 0 } + /^Indirect blocks:/ { looking = 1} + /^\t\tsegment / { looking = 0} + /L[0-8]/ && looking == 1 { print $0} + ' | sed -n 's/^.*\(L[0-9]\) \([0-9]*\):\([0-9a-f]*\):\([0-9a-f]*\) .*$/\1 \2 \3 \4/p' | \ + while read level path offset length; do + offset=$((16#$offset)) # Conversion from hex + length=$((16#$length)) + offset="$(((offset + 4 * 1024 * 1024) / 512))" + length="$((length / 512))" + echo "$level ${VDEV_MAP[$path]} $offset $length" + done 2>/dev/null +} + +function corrupt_blocks_at_level # input_file corrupt_level +{ + typeset input_file=$1 + typeset corrupt_level="L${2:-0}" + typeset level path offset length + + [[ -f $input_file ]] || log_fail "Couldn't find $input_file" + + + log_must list_file_blocks $input_file | \ + while read level path offset length; do + if [[ $level = $corrupt_level ]]; then + log_must dd if=/dev/urandom of=$path bs=512 \ + count=$length seek=$offset conv=notrunc + fi + done + + # This is necessary for pools made of loop devices. + sync +} diff --git a/tests/zfs-tests/tests/functional/checksum/filetest_001_pos.ksh b/tests/zfs-tests/tests/functional/checksum/filetest_001_pos.ksh index ccc60a661d0e..27dad072631d 100755 --- a/tests/zfs-tests/tests/functional/checksum/filetest_001_pos.ksh +++ b/tests/zfs-tests/tests/functional/checksum/filetest_001_pos.ksh @@ -21,7 +21,7 @@ # # -# Copyright (c) 2018 by Delphix. All rights reserved. +# Copyright (c) 2018, 2019 by Delphix. All rights reserved. # . $STF_SUITE/include/libtest.shlib @@ -32,8 +32,8 @@ # Sanity test to make sure checksum algorithms work. # For each checksum, create a file in the pool using that checksum. Verify # that there are no checksum errors. Next, for each checksum, create a single -# file in the pool using that checksum, scramble the underlying vdev, and -# verify that we correctly catch the checksum errors. +# file in the pool using that checksum, corrupt the file, and verify that we +# correctly catch the checksum errors. # # STRATEGY: # Test 1 @@ -46,11 +46,9 @@ # Test 2 # 6. For each checksum: # 7. Create a file using the checksum -# 8. Export the pool -# 9. Scramble the data on one of the underlying VDEVs -# 10. Import the pool -# 11. Scrub the pool -# 12. Verify that there are checksum errors +# 8. Corrupt all level 0 blocks in the file +# 9. Scrub the pool +# 10. Verify that there are checksum errors verify_runnable "both" @@ -66,8 +64,6 @@ log_assert "Create and read back files with using different checksum algorithms" log_onexit cleanup WRITESZ=1048576 -SKIPCNT=$(((4194304 / $WRITESZ) * 2)) -WRITECNT=$((($MINVDEVSIZE / $WRITESZ) - $SKIPCNT)) # Get a list of vdevs in our pool set -A array $(get_disklist_fullpath) @@ -96,7 +92,7 @@ log_must [ $cksum -eq 0 ] rm -fr $TESTDIR/* -log_assert "Test scrambling the disk and seeing checksum errors" +log_assert "Test corrupting the files and seeing checksum errors" typeset -i j=1 while [[ $j -lt ${#CHECKSUM_TYPES[*]} ]]; do type=${CHECKSUM_TYPES[$j]} @@ -104,14 +100,9 @@ while [[ $j -lt ${#CHECKSUM_TYPES[*]} ]]; do log_must file_write -o overwrite -f $TESTDIR/test_$type \ -b $WRITESZ -c 5 -d R - log_must zpool export $TESTPOOL + # Corrupt the level 0 blocks of this file + corrupt_blocks_at_level $TESTDIR/test_$type - # Scramble the data on the first vdev in our pool. Skip the first - # and last 16MB of data, then scramble the rest after that. - log_must dd if=/dev/zero of=$firstvdev bs=$WRITESZ skip=$SKIPCNT \ - count=$WRITECNT - - log_must zpool import $TESTPOOL log_must zpool scrub $TESTPOOL log_must wait_scrubbed $TESTPOOL