SNP Direct DM-Verity Boot (#1952)

* Working DM-Verity boot using 5..15 kernel

Signed-off-by: Ken Gordon <Ken.Gordon@microsoft.com>
Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Working to boot 6.1 or 5.15 kernels with vhd supplied userland and merkle tree.

Signed-off-by: Ken Gordon <Ken.Gordon@microsoft.com>
Signed-off-by: Joe Powell <joepowell@microsoft.com>

* PR #1886 changes which are required or gcs cannot start on 6.1

Signed-off-by: Ken Gordon <Ken.Gordon@microsoft.com>
Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Use "modern" igvm tooling from github repo.

Signed-off-by: Ken Gordon <Ken.Gordon@microsoft.com>
Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Clean up Makefile

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Add boot doc

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Remove startup_2 as it is now redundant

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Tidying

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* print opts

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* debug

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* debug

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Remove extra err

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Rm fmt

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Clean up startups

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Kick CI

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Add HvSock port annotation

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Clean up merge

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Mark ups pre-rebasing

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* gofmt

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* More concise Makefile snp target

Signed-off-by: Joe Powell <joepowell@microsoft.com>

* Apply nits

Signed-off-by: Joe Powell <joepowell@microsoft.com>

---------

Signed-off-by: Ken Gordon <Ken.Gordon@microsoft.com>
Signed-off-by: Joe Powell <joepowell@microsoft.com>
Co-authored-by: Ken Gordon <Ken.Gordon@microsoft.com>

Makefile

@@ -29,12 +29,23 @@ ifeq "$(DEV_BUILD)" "1"
 DELTA_TARGET=out/delta-dev.tar.gz
 endif
 
+ifeq "$(SNP_BUILD)" "1"
+DELTA_TARGET=out/delta-snp.tar.gz
+endif
+
 # The link aliases for gcstools
 GCS_TOOLS=\
 	generichook \
 	install-drivers
 
-.PHONY: all always rootfs test
+# Common path prefix.
+PATH_PREFIX:=
+# These have PATH_PREFIX prepended to obtain the full path in recipies e.g. $(PATH_PREFIX)/$(VMGS_TOOL)
+VMGS_TOOL:=
+IGVM_TOOL:=
+KERNEL_PATH:=
+
+.PHONY: all always rootfs test snp simple
 
 .DEFAULT_GOAL := all
 
@@ -49,9 +60,58 @@ test:
 
 rootfs: out/rootfs.vhd
 
-out/rootfs.vhd: out/rootfs.tar.gz bin/cmd/tar2ext4
+snp: out/kernelinitrd.vmgs out/rootfs.hash.vhd out/rootfs.vhd out/v2056.vmgs
+
+simple: out/simple.vmgs snp
+
+%.vmgs: %.bin
+	rm -f $@
+	# du -BM returns the size of the bin file in M, eg 7M. The sed command replaces the M with *1024*1024 and then bc does the math to convert to bytes
+	$(PATH_PREFIX)/$(VMGS_TOOL) create --filepath $@ --filesize `du -BM $< | sed  "s/M.*/*1024*1024/" | bc`
+	$(PATH_PREFIX)/$(VMGS_TOOL) write --filepath $@ --datapath $< -i=8
+
+# Simplest debug UVM used to test changes to the linux kernel. No dmverity protection. Boots an initramdisk rather than directly booting a vhd disk.
+out/simple.bin: out/initrd.img $(PATH_PREFIX)/$(KERNEL_PATH) boot/startup_simple.sh
+	rm -f $@
+	python3 $(PATH_PREFIX)/$(IGVM_TOOL) -o $@ -kernel $(PATH_PREFIX)/$(KERNEL_PATH) -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 rdinit=/startup_simple.sh" -rdinit out/initrd.img -vtl 0
+
+ROOTFS_DEVICE:=/dev/sda
+VERITY_DEVICE:=/dev/sdb
+# Debug build for use with uvmtester. UVM with dm-verity protected vhd disk mounted directly via the kernel command line. Ignores corruption in dm-verity protected disk. (Use dmesg to see if dm-verity is ignoring data corruption.)
+out/v2056.bin: out/rootfs.vhd out/rootfs.hash.vhd $(PATH_PREFIX)/$(KERNEL_PATH) out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.salt boot/startup_v2056.sh
+	rm -f $@
+	python3 $(PATH_PREFIX)/$(IGVM_TOOL) -o $@ -kernel $(PATH_PREFIX)/$(KERNEL_PATH) -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) 0 sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt) 1 ignore_corruption\" init=/startup_v2056.sh"  -vtl 0
+
+# Full UVM with dm-verity protected vhd disk mounted directly via the kernel command line.
+out/kernelinitrd.bin: out/rootfs.vhd out/rootfs.hash.vhd out/rootfs.hash.datasectors out/rootfs.hash.datablocksize out/rootfs.hash.hashblocksize out/rootfs.hash.datablocks out/rootfs.hash.rootdigest out/rootfs.hash.salt $(PATH_PREFIX)/$(KERNEL_PATH) boot/startup.sh
+	rm -f $@
+	python3 $(PATH_PREFIX)/$(IGVM_TOOL) -o $@ -kernel $(PATH_PREFIX)/$(KERNEL_PATH) -append "8250_core.nr_uarts=0 panic=-1 debug loglevel=7 root=/dev/dm-0 dm-mod.create=\"dmverity,,,ro,0 $(shell cat out/rootfs.hash.datasectors) verity 1 $(ROOTFS_DEVICE) $(VERITY_DEVICE) $(shell cat out/rootfs.hash.datablocksize) $(shell cat out/rootfs.hash.hashblocksize) $(shell cat out/rootfs.hash.datablocks) 0 sha256 $(shell cat out/rootfs.hash.rootdigest) $(shell cat out/rootfs.hash.salt)\" init=/startup.sh"  -vtl 0
+
+# Rule to make a vhd from a file. This is used to create the rootfs.hash.vhd from rootfs.hash.
+%.vhd: % bin/cmd/tar2ext4
+	./bin/cmd/tar2ext4 -only-vhd -i $< -o $@
+
+# Rule to make a vhd from an ext4 file. This is used to create the rootfs.vhd from rootfs.ext4.
+%.vhd: %.ext4 bin/cmd/tar2ext4
+	./bin/cmd/tar2ext4 -only-vhd -i $< -o $@
+
+%.hash %.hash.info %.hash.datablocks %.hash.rootdigest %hash.datablocksize %.hash.datasectors %.hash.hashblocksize: %.ext4 %.hash.salt
+	veritysetup format --no-superblock --salt $(shell cat out/rootfs.hash.salt) $< $*.hash > $*.hash.info
+    # Retrieve info required by dm-verity at boot time
+    # Get the blocksize of rootfs
+	cat $*.hash.info | awk '/^Root hash:/{ print $$3 }' > $*.hash.rootdigest
+	cat $*.hash.info | awk '/^Salt:/{ print $$2 }' > $*.hash.salt
+	cat $*.hash.info | awk '/^Data block size:/{ print $$4 }' > $*.hash.datablocksize
+	cat $*.hash.info | awk '/^Hash block size:/{ print $$4 }' > $*.hash.hashblocksize
+	cat $*.hash.info | awk '/^Data blocks:/{ print $$3 }' > $*.hash.datablocks
+	echo $$(( $$(cat $*.hash.datablocks) * $$(cat $*.hash.datablocksize) / 512 )) > $*.hash.datasectors
+
+out/rootfs.hash.salt:
+	hexdump -vn32 -e'8/4 "%08X" 1 "\n"' /dev/random > $@
+
+out/rootfs.ext4: out/rootfs.tar.gz bin/cmd/tar2ext4
 	gzip -f -d ./out/rootfs.tar.gz
-	bin/cmd/tar2ext4 -vhd -i ./out/rootfs.tar -o $@
+	./bin/cmd/tar2ext4 -i ./out/rootfs.tar -o $@
 
 out/rootfs.tar.gz: out/initrd.img
 	rm -rf rootfs-conv
@@ -74,6 +134,20 @@ out/delta-dev.tar.gz: out/delta.tar.gz bin/internal/tools/snp-report
 	tar -zcf $@ -C rootfs-dev .
 	rm -rf rootfs-dev
 
+out/delta-snp.tar.gz: out/delta.tar.gz bin/internal/tools/snp-report boot/startup_v2056.sh boot/startup_simple.sh boot/startup.sh
+	rm -rf rootfs-snp
+	mkdir rootfs-snp
+	tar -xzf out/delta.tar.gz -C rootfs-snp
+	cp boot/startup_v2056.sh rootfs-snp/startup_v2056.sh
+	cp boot/startup_simple.sh rootfs-snp/startup_simple.sh
+	cp boot/startup.sh rootfs-snp/startup.sh
+	cp bin/internal/tools/snp-report rootfs-snp/bin/
+	chmod a+x rootfs-snp/startup_v2056.sh
+	chmod a+x rootfs-snp/startup_simple.sh
+	chmod a+x rootfs-snp/startup.sh
+	tar -zcf $@ -C rootfs-snp .
+	rm -rf rootfs-snp
+
 out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths Makefile
 	@mkdir -p out
 	rm -rf rootfs
@@ -94,7 +168,10 @@ out/delta.tar.gz: bin/init bin/vsockexec bin/cmd/gcs bin/cmd/gcstools bin/cmd/ho
 	tar -zcf $@ -C rootfs .
 	rm -rf rootfs
 
-bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths bin/cmd/tar2ext4 bin/internal/tools/snp-report:
+out/containerd-shim-runhcs-v1.exe:
+	GOOS=windows $(GO_BUILD) -o $@ $(SRCROOT)/cmd/containerd-shim-runhcs-v1
+
+bin/cmd/gcs bin/cmd/gcstools bin/cmd/hooks/wait-paths bin/cmd/tar2ext4 bin/internal/tools/snp-report bin/cmd/dmverity-vhd:
 	@mkdir -p $(dir $@)
 	GOOS=linux $(GO_BUILD) -o $@ $(SRCROOT)/$(@:bin/%=%)
 
@@ -108,4 +185,4 @@ bin/init: init/init.o vsockexec/vsock.o
 
 %.o: %.c
 	@mkdir -p $(dir $@)
-	$(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<

boot/bootstrap.md

@@ -0,0 +1,24 @@
+# UVM Boot Info
+
+For understanding the UVM's boot sequence it's useful to think of the UVM as consisting of:
+- Linux kernel
+- Kernel command line
+  - The command line is a set of parameters the kernel understands which correspond to actions it will perform during boot.
+- Root filesystem (rootfs) disk
+  - This contains all the files that exist when first starting the VM.
+- Startup script
+  - Stored in the rootfs disk. This scripts does the last bits of setup required to get the VM ready for use.
+- Hash disk (SNP Mode only)
+  - Containing DM-Verity hash data (read more below about DM-Verity and SNP mode below).
+
+
+## The SNP Mode UVM boot sequence.
+- The vmgs (kernel + commandline) file is loaded into memory.
+- The instructions from the kernel command line are performed, the kernel:
+    - Checks the hash disk's hash data (a merkle tree) is consistent.
+    - Checks the hash disk's root hash matches the root hash in the kernel command line. The boot fails if not because the integrity of the UVM cannot be confirmed.
+    - Makes the rootfs disk available as a dm-verity device.
+    - Mounts the dm-verity rootfs device.
+    - Sets the newly mounted disk as the root filesystem
+    - Finds and runs the startup script (which is specified in the kernel command line) from the rootfs to initialise the system.
+    - Anytime that data is read from the dm-verity rootfs, that data's integrity is checked on the fly by comparing the data's hash with the hash data on the hash disk.

boot/startup.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+export PATH="/usr/bin:/usr/local/bin:/bin:/root/bin:/sbin:/usr/sbin:/usr/local/sbin"
+export HOME="/root"
+
+/init -e 1 /bin/vsockexec -o 109 -e 109 /bin/gcs -v4 -log-format json -loglevel debug

boot/startup_simple.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+export PATH="/usr/bin:/usr/local/bin:/bin:/root/bin:/sbin:/usr/sbin:/usr/local/sbin"
+export HOME="/root"
+
+/bin/vsockexec -o 2056 -e 2056 echo Running startup_simple.sh
+/bin/vsockexec -o 2056 -e 2056 date
+
+/bin/vsockexec -o 2056 -e 2056 echo /init -e 1 /bin/vsockexec -o 2056 -e 109 /bin/gcs -v4 -log-format text -loglevel debug -logfile /tmp/gcs.log
+/init -e 1 /bin/vsockexec -o 2056 -e 109 /bin/gcs -v4 -log-format text -loglevel debug -logfile /tmp/gcs.log
+
+/bin/vsockexec -o 2056 -e 2056 echo dmesg
+/bin/vsockexec -o 2056 -e 2056 dmesg
+
+/bin/vsockexec -o 2056 -e 2056 echo sleeping 2
+/bin/vsockexec -o 2056 -e 2056 sleep 2
+
+/bin/vsockexec -o 2056 -e 2056 ls -Rl /dev/se*
+

boot/startup_v2056.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+
+export PATH="/usr/bin:/usr/local/bin:/bin:/root/bin:/sbin:/usr/sbin:/usr/local/sbin"
+export HOME="/root"
+
+/bin/vsockexec -o 2056 -e 2056 echo Running startup_v2056.sh
+/bin/vsockexec -o 2056 -e 2056 date
+
+/bin/vsockexec -o 2056 -e 2056 echo /init -e 1 /bin/vsockexec -o 2056 -e 109 /bin/gcs -v4 -log-format text -loglevel debug -logfile /tmp/gcs.log
+/init -e 1 /bin/vsockexec -o 2056 -e 109 /bin/gcs -v4 -log-format text -loglevel debug -logfile /tmp/gcs.log
+
+/bin/vsockexec -o 2056 -e 2056 echo ls -l /dev/dm*
+/bin/vsockexec -o 2056 -e 2056 ls -l /dev/dm*
+/bin/vsockexec -o 2056 -e 2056 echo ls -l /dev/mapper
+/bin/vsockexec -o 2056 -e 2056 ls -l /dev/mapper
+/bin/vsockexec -o 2056 -e 2056 echo ls -l /dev/mapper
+/bin/vsockexec -o 2056 -e 2056 ls -l /dev/mapper
+
+#/bin/vsockexec -o 2056 -e 2056 /bin/snp-report
+
+# need init to have run before top shows much
+/bin/vsockexec -o 2056 -e 2056 top -n 1
+
+/bin/vsockexec -o 2056 -e 2056 echo tmp
+/bin/vsockexec -o 2056 -e 2056 ls -la /tmp
+
+/bin/vsockexec -o 2056 -e 2056 /bin/dmesg
+
+sleep 1
+/bin/vsockexec -o 2056 -e 2056 echo Thats all folks...
+
+
+
+

cmd/tar2ext4/tar2ext4.go

@@ -15,6 +15,7 @@ var (
 	overlay      = flag.Bool("overlay", false, "produce overlayfs-compatible layer image")
 	convertSlash = flag.Bool("convert-slash", false, "convert backslashes ('\\') in path names to slashes ('/')")
 	vhd          = flag.Bool("vhd", false, "add a VHD footer to the end of the image")
+	onlyVhd      = flag.Bool("only-vhd", false, "adds a VHD footer to the end of the file but does not convert to ext4; this implies '-vhd' and ignores all other options")
 	inlineData   = flag.Bool("inline", false, "write small file data into the inode; not compatible with DAX")
 )
 
@@ -48,6 +49,9 @@ func main() {
 		if *vhd {
 			opts = append(opts, tar2ext4.AppendVhdFooter)
 		}
+		if *onlyVhd {
+			opts = append(opts, tar2ext4.OnlyAppendVhdFooter)
+		}
 		if *inlineData {
 			opts = append(opts, tar2ext4.InlineData)
 		}

ext4/tar2ext4/tar2ext4.go

@@ -18,11 +18,12 @@ import (
 )
 
 type params struct {
-	convertWhiteout  bool
-	convertBackslash bool
-	appendVhdFooter  bool
-	appendDMVerity   bool
-	ext4opts         []compactext4.Option
+	convertWhiteout     bool
+	convertBackslash    bool
+	appendVhdFooter     bool
+	onlyAppendVhdFooter bool
+	appendDMVerity      bool
+	ext4opts            []compactext4.Option
 }
 
 // Option is the type for optional parameters to Convert.
@@ -46,6 +47,12 @@ func AppendVhdFooter(p *params) {
 	p.appendVhdFooter = true
 }
 
+// OnlyAppendVhdFooter instructs the converter not to convert but still to add a fixed VHD footer to the
+// file.
+func OnlyAppendVhdFooter(p *params) {
+	p.onlyAppendVhdFooter = true
+}
+
 // AppendDMVerity instructs the converter to add a dmverity Merkle tree for
 // the ext4 filesystem after the filesystem and before the optional VHD footer
 func AppendDMVerity(p *params) {
@@ -201,6 +208,14 @@ func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
 		opt(&p)
 	}
 
+	if p.onlyAppendVhdFooter {
+		_, err := io.Copy(w, r)
+		if err != nil {
+			return err
+		}
+		return ConvertToVhd(w)
+	}
+
 	if err := ConvertTarToExt4(r, w, options...); err != nil {
 		return err
 	}