ZTS: Introduce targeted corruption in file blocks

filetest_001_pos verifies that various checksum algorithms detect
corruption by overwriting the underlying vdev on which a file resides.
It is possible for the overwrite to miss the blocks of a file, causing a
spurious failure. This change introduces a function to corrupt the
individual blocks of a file as determined by zdb.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Ryan Moeller <ryan@ixsystems.com>
Signed-off-by: John Kennedy <john.kennedy@delphix.com>
Closes #9288

tests/zfs-tests/include/blkdev.shlib

@@ -12,7 +12,7 @@
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
-# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
+# Copyright (c) 2012, 2019 by Delphix. All rights reserved.
 # Copyright 2016 Nexenta Systems, Inc.
 # Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 # Copyright (c) 2017 Lawrence Livermore National Security, LLC.
@@ -465,3 +465,89 @@ function get_pool_devices #testpool #devdir
 	fi
 	echo $out
 }
+
+#
+# Write to standard out giving the level, device name, offset and length
+# of all blocks in an input file. The offset and length are in units of
+# 512 byte blocks. In the case of mirrored vdevs, only the first
+# device is listed, as the levels, blocks and offsets will be the same
+# on other devices. Note that this function only works with mirrored
+# or non-redundant pools, not raidz.
+#
+# The output of this function can be used to introduce corruption at
+# varying levels of indirection.
+#
+function list_file_blocks # input_file
+{
+	typeset input_file=$1
+
+	[[ -f $input_file ]] || log_fail "Couldn't find $input_file"
+
+	typeset ds="$(zfs list -H -o name $input_file)"
+	typeset pool="${ds%%/*}"
+	typeset inum="$(stat -c '%i' $input_file)"
+
+	#
+	# Establish a mapping between vdev ids as shown in a DVA and the
+	# pathnames they correspond to in ${VDEV_MAP[]}.
+	#
+	eval $(zdb -C $pool | awk '
+		BEGIN {
+			printf("typeset VDEV_MAP\n");
+			looking = 0;
+		}
+		/^            children/ {
+			id = $1;
+			looking = 1;
+		}
+		/path: / && looking == 1 {
+			print id" "$2;
+			looking = 0;
+		}
+	' | sed -n 's/^children\[\([0-9]\)\]: \(.*\)$/VDEV_MAP[\1]=\2/p')
+
+	#
+	# The awk below parses the output of zdb, printing out the level
+	# of each block along with vdev id, offset and length. The last
+	# two are converted to decimal in the while loop. 4M is added to
+	# the offset to compensate for the first two labels and boot
+	# block. Lastly, the offset and length are printed in units of
+	# 512b blocks for ease of use with dd.
+	#
+	log_must zpool sync -f
+	typeset level path offset length
+	zdb -ddddd $ds $inum | awk -F: '
+		BEGIN { looking = 0 }
+		/^Indirect blocks:/ { looking = 1}
+		/^\t\tsegment / { looking = 0}
+		/L[0-8]/ && looking == 1 { print $0}
+	' | sed -n 's/^.*\(L[0-9]\) \([0-9]*\):\([0-9a-f]*\):\([0-9a-f]*\) .*$/\1 \2 \3 \4/p' | \
+	while read level path offset length; do
+		offset=$((16#$offset))  # Conversion from hex
+		length=$((16#$length))
+		offset="$(((offset + 4 * 1024 * 1024) / 512))"
+		length="$((length / 512))"
+		echo "$level ${VDEV_MAP[$path]} $offset $length"
+	done 2>/dev/null
+}
+
+function corrupt_blocks_at_level # input_file corrupt_level
+{
+	typeset input_file=$1
+	typeset corrupt_level="L${2:-0}"
+	typeset level path offset length
+
+	[[ -f $input_file ]] || log_fail "Couldn't find $input_file"
+
+
+	log_must list_file_blocks $input_file | \
+	    while read level path offset length; do
+		if [[ $level = $corrupt_level ]]; then
+			log_must dd if=/dev/urandom of=$path bs=512 \
+			    count=$length seek=$offset conv=notrunc
+		fi
+	    done
+
+	# This is necessary for pools made of loop devices.
+	sync
+}

tests/zfs-tests/tests/functional/checksum/filetest_001_pos.ksh

@@ -21,7 +21,7 @@
 #
 
 #
-# Copyright (c) 2018 by Delphix. All rights reserved.
+# Copyright (c) 2018, 2019 by Delphix. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
@@ -32,8 +32,8 @@
 # Sanity test to make sure checksum algorithms work.
 # For each checksum, create a file in the pool using that checksum.  Verify
 # that there are no checksum errors.  Next, for each checksum, create a single
-# file in the pool using that checksum, scramble the underlying vdev, and
-# verify that we correctly catch the checksum errors.
+# file in the pool using that checksum, corrupt the file, and verify that we
+# correctly catch the checksum errors.
 #
 # STRATEGY:
 # Test 1
@@ -46,11 +46,9 @@
 # Test 2
 # 6. For each checksum:
 # 7.	Create a file using the checksum
-# 8.	Export the pool
-# 9.	Scramble the data on one of the underlying VDEVs
-# 10.	Import the pool
-# 11.	Scrub the pool
-# 12.	Verify that there are checksum errors
+# 8.	Corrupt all level 0 blocks in the file
+# 9.	Scrub the pool
+# 10.	Verify that there are checksum errors
 
 verify_runnable "both"
 
@@ -66,8 +64,6 @@ log_assert "Create and read back files with using different checksum algorithms"
 log_onexit cleanup
 
 WRITESZ=1048576
-SKIPCNT=$(((4194304 / $WRITESZ) * 2))
-WRITECNT=$((($MINVDEVSIZE / $WRITESZ) - $SKIPCNT))
 
 # Get a list of vdevs in our pool
 set -A array $(get_disklist_fullpath)
@@ -96,22 +92,17 @@ log_must [ $cksum -eq 0 ]
 
 rm -fr $TESTDIR/*
 
-log_assert "Test scrambling the disk and seeing checksum errors"
+log_assert "Test corrupting the files and seeing checksum errors"
 typeset -i j=1
 while [[ $j -lt ${#CHECKSUM_TYPES[*]} ]]; do
 	type=${CHECKSUM_TYPES[$j]}
 	log_must zfs set checksum=$type $TESTPOOL
 	log_must file_write -o overwrite -f $TESTDIR/test_$type \
 	    -b $WRITESZ -c 5 -d R
 
-	log_must zpool export $TESTPOOL
+	# Corrupt the level 0 blocks of this file
+	corrupt_blocks_at_level $TESTDIR/test_$type
 
-	# Scramble the data on the first vdev in our pool.  Skip the first
-	# and last 16MB of data, then scramble the rest after that.
-	log_must dd if=/dev/zero of=$firstvdev bs=$WRITESZ skip=$SKIPCNT \
-	    count=$WRITECNT
-
-	log_must zpool import $TESTPOOL
 	log_must zpool scrub $TESTPOOL
 	log_must wait_scrubbed $TESTPOOL