From 3ae6bf9f72beb3c28aa4dd73e95e3bae15f194e0 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Wed, 11 Dec 2024 22:57:35 -0500 Subject: [PATCH 1/5] amd-sev: Create KVM_X86_SNP_VM type KVM requires a VM intended to be run with SEV-SNP encryption to be created with the KVM_X86_SNP_VM type. Signed-off-by: Tyler Fanelli --- src/vmm/src/linux/vstate.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/linux/vstate.rs b/src/vmm/src/linux/vstate.rs index 77726880..e755fc56 100644 --- a/src/vmm/src/linux/vstate.rs +++ b/src/vmm/src/linux/vstate.rs @@ -507,8 +507,12 @@ impl Vm { #[cfg(feature = "amd-sev")] pub fn new(kvm: &Kvm, tee_config: &TeeConfig) -> Result { + const KVM_X86_SNP_VM: u64 = 4; + //create fd for interacting with kvm-vm specific functions - let vm_fd = kvm.create_vm().map_err(Error::VmFd)?; + let vm_fd = kvm + .create_vm_with_type(KVM_X86_SNP_VM) + .map_err(Error::VmFd)?; let supported_cpuid = kvm .get_supported_cpuid(KVM_MAX_CPUID_ENTRIES) From 6c1f4bbb5cf5c650bb6f05577af30d6d40dc28aa Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Wed, 11 Dec 2024 23:32:45 -0500 Subject: [PATCH 2/5] amd-sev: Use kvm_guest_memfd for memory registering Signed-off-by: Tyler Fanelli --- src/vmm/src/linux/tee/amdsnp.rs | 45 +++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/vmm/src/linux/tee/amdsnp.rs b/src/vmm/src/linux/tee/amdsnp.rs index c338e22f..91a255aa 100644 --- a/src/vmm/src/linux/tee/amdsnp.rs +++ b/src/vmm/src/linux/tee/amdsnp.rs @@ -9,7 +9,10 @@ use arch::x86_64::layout::*; use sev::firmware::{guest::GuestPolicy, host::Firmware}; use sev::launch::snp::*; -use kvm_bindings::{kvm_enc_region, CpuId, KVM_CPUID_FLAG_SIGNIFCANT_INDEX}; +use kvm_bindings::{ + kvm_create_guest_memfd, kvm_userspace_memory_region2, CpuId, KVM_CPUID_FLAG_SIGNIFCANT_INDEX, + KVM_MEM_GUEST_MEMFD, +}; use kvm_ioctls::VmFd; use vm_memory::{ Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap, @@ -86,24 +89,44 @@ impl AmdSnp { vm_fd: &VmFd, guest_mem: &GuestMemoryMmap, ) -> Result, Error> { - let vm_rfd = vm_fd.as_raw_fd(); - let fw_rfd = self.fw.as_raw_fd(); - - let launcher = Launcher::new(vm_rfd, fw_rfd).map_err(Error::CreateLauncher)?; - for region in guest_mem.iter() { // It's safe to unwrap because the guest address is valid. let host_addr = guest_mem.get_host_address(region.start_addr()).unwrap(); - let enc_region = kvm_enc_region { - addr: host_addr as u64, + + // Create guest_memfd struct. + let gmem = kvm_create_guest_memfd { size: region.len(), + flags: 0, //Unused. + reserved: [0; 6], }; - vm_fd - .register_enc_memory_region(&enc_region) - .map_err(|_| Error::MemoryEncryptRegion)?; + // Create KVM guest_memfd. + let fd = vm_fd.create_guest_memfd(gmem).unwrap(); + + // Create memory region. + let mem_region = kvm_userspace_memory_region2 { + slot: 0, + flags: KVM_MEM_GUEST_MEMFD, + guest_phys_addr: region.start_addr().0, + memory_size: region.len(), + userspace_addr: host_addr as u64, + guest_memfd_offset: 0, + guest_memfd: fd as u32, + pad1: 0, + pad2: [0; 14], + }; + + // Set the memory region. + unsafe { + vm_fd.set_user_memory_region2(mem_region).unwrap(); + } } + let vm_rfd = vm_fd.as_raw_fd(); + let fw_rfd = self.fw.as_raw_fd(); + + let launcher = Launcher::new(vm_rfd, fw_rfd).map_err(Error::CreateLauncher)?; + let mut policy = GuestPolicy(0); policy.set_smt_allowed(1); From d2670e7b1909e9e525daafd6ffd3526538e9fcad Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Thu, 12 Dec 2024 00:14:43 -0500 Subject: [PATCH 3/5] amd-sev: Update sev dependency to 5.0.0 Signed-off-by: Tyler Fanelli --- Cargo.lock | 42 +++++++++++++++++++++++++-------- src/vmm/Cargo.toml | 4 ++-- src/vmm/src/linux/tee/amdsnp.rs | 28 ++++++++++------------ 3 files changed, 47 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d1d5e22..3c65b424 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -360,7 +360,7 @@ dependencies = [ "polly", "rand", "rutabaga_gfx", - "thiserror", + "thiserror 1.0.69", "utils", "virtio-bindings", "vm-memory", @@ -689,13 +689,15 @@ dependencies = [ [[package]] name = "kbs-types" -version = "0.8.0" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21350cefefc9715198c3c5319a5eb23ce4cc89b4b567599fb88f0d4a011c1d2d" +checksum = "f7b7dad1b68c1fee4a7749f9163f4ac7c7f9cd21df3f6dcc5790f8978e12ccf1" dependencies = [ + "base64", "serde", "serde_json", "sev", + "thiserror 2.0.6", ] [[package]] @@ -1021,7 +1023,7 @@ dependencies = [ "nix 0.27.1", "once_cell", "pipewire-sys", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1138,7 +1140,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1213,7 +1215,7 @@ dependencies = [ "nix 0.26.4", "pkg-config", "remain", - "thiserror", + "thiserror 1.0.69", "winapi", "zerocopy 0.6.6", ] @@ -1300,9 +1302,9 @@ dependencies = [ [[package]] name = "sev" -version = "4.0.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97bd0b2e2d937951add10c8512a2dacc6ad29b39e5c5f26565a3e443329857d" +checksum = "b06afe5192a43814047ea0072f4935f830a1de3c8cb43b56c90ae6918468b94d" dependencies = [ "base64", "bincode", @@ -1413,7 +1415,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +dependencies = [ + "thiserror-impl 2.0.6", ] [[package]] @@ -1427,6 +1438,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio" version = "1.41.1" @@ -1571,7 +1593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1720e7240cdc739f935456eb77f370d7e9b2a3909204da1e2b47bef1137a013" dependencies = [ "libc", - "thiserror", + "thiserror 1.0.69", "winapi", ] diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 25ed38d7..ab7e4be0 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -28,12 +28,12 @@ polly = { path = "../polly" } # Dependencies for amd-sev codicon = { version = "3.0.0", optional = true } -kbs-types = { version = "0.8.0", features = ["tee-sev", "tee-snp"], optional = true } +kbs-types = { version = "0.9.2", features = ["tee-sev", "tee-snp"], optional = true } procfs = { version = "0.12", optional = true } rdrand = { version = "^0.8", optional = true } serde = { version = "1.0.125", optional = true } serde_json = { version = "1.0.64", optional = true } -sev = { version = "4.0.0", features = ["openssl"], optional = true } +sev = { version = "5.0.0", features = ["openssl"], optional = true } curl = { version = "0.4", optional = true } nix = "0.24.1" diff --git a/src/vmm/src/linux/tee/amdsnp.rs b/src/vmm/src/linux/tee/amdsnp.rs index 91a255aa..e0d99e92 100644 --- a/src/vmm/src/linux/tee/amdsnp.rs +++ b/src/vmm/src/linux/tee/amdsnp.rs @@ -6,8 +6,11 @@ use std::{ use crate::vstate::MeasuredRegion; use arch::x86_64::layout::*; -use sev::firmware::{guest::GuestPolicy, host::Firmware}; use sev::launch::snp::*; +use sev::{ + error::FirmwareError as SevFirmwareError, + firmware::{guest::GuestPolicy, host::Firmware}, +}; use kvm_bindings::{ kvm_create_guest_memfd, kvm_userspace_memory_region2, CpuId, KVM_CPUID_FLAG_SIGNIFCANT_INDEX, @@ -22,12 +25,12 @@ use vm_memory::{ pub enum Error { CpuIdWrite, CpuIdFull, - CreateLauncher(std::io::Error), + CreateLauncher(SevFirmwareError), GuestMemoryWrite(vm_memory::GuestMemoryError), GuestMemoryRead(vm_memory::GuestMemoryError), - LaunchStart(std::io::Error), - LaunchUpdate(std::io::Error), - LaunchFinish(std::io::Error), + LaunchStart(SevFirmwareError), + LaunchUpdate(SevFirmwareError), + LaunchFinish(SevFirmwareError), MemoryEncryptRegion, OpenFirmware(std::io::Error), } @@ -130,7 +133,7 @@ impl AmdSnp { let mut policy = GuestPolicy(0); policy.set_smt_allowed(1); - let start = Start::new(None, policy, false, [0; 16]); + let start = Start::new(policy, [0; 16]); let launcher = launcher.start(start).map_err(Error::LaunchStart)?; @@ -304,7 +307,6 @@ impl AmdSnp { launcher: &mut Launcher, page_type: PageType, ) -> Result<(), Error> { - let dp = VmplPerms::empty(); let ga = GuestAddress(region.guest_addr); /* @@ -319,15 +321,11 @@ impl AmdSnp { let ptr = bytes.ptr_guard().as_ptr(); let slice: &[u8] = unsafe { slice::from_raw_parts(ptr, region.size) }; - let update = Update::new( - region.guest_addr >> 12, - slice, - false, - page_type, - (dp, dp, dp), - ); + let update = Update::new(region.guest_addr >> 12, slice, page_type); - launcher.update_data(update).map_err(Error::LaunchUpdate) + launcher + .update_data(update, ga.0, region.size.try_into().unwrap()) + .map_err(Error::LaunchUpdate) } pub fn vm_measure( From 535914fd50656cc0dc3d5c1559a887acf92495d5 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Thu, 12 Dec 2024 01:23:17 -0500 Subject: [PATCH 4/5] amd-sev: Set guest_memfd memory regions in memory_init Signed-off-by: Tyler Fanelli --- src/vmm/src/linux/tee/amdsnp.rs | 40 ++-------------------------- src/vmm/src/linux/vstate.rs | 46 ++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/vmm/src/linux/tee/amdsnp.rs b/src/vmm/src/linux/tee/amdsnp.rs index e0d99e92..05e0633b 100644 --- a/src/vmm/src/linux/tee/amdsnp.rs +++ b/src/vmm/src/linux/tee/amdsnp.rs @@ -12,10 +12,7 @@ use sev::{ firmware::{guest::GuestPolicy, host::Firmware}, }; -use kvm_bindings::{ - kvm_create_guest_memfd, kvm_userspace_memory_region2, CpuId, KVM_CPUID_FLAG_SIGNIFCANT_INDEX, - KVM_MEM_GUEST_MEMFD, -}; +use kvm_bindings::{CpuId, KVM_CPUID_FLAG_SIGNIFCANT_INDEX}; use kvm_ioctls::VmFd; use vm_memory::{ Bytes, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion, GuestRegionMmap, @@ -90,41 +87,8 @@ impl AmdSnp { pub fn vm_prepare( &self, vm_fd: &VmFd, - guest_mem: &GuestMemoryMmap, + _guest_mem: &GuestMemoryMmap, ) -> Result, Error> { - for region in guest_mem.iter() { - // It's safe to unwrap because the guest address is valid. - let host_addr = guest_mem.get_host_address(region.start_addr()).unwrap(); - - // Create guest_memfd struct. - let gmem = kvm_create_guest_memfd { - size: region.len(), - flags: 0, //Unused. - reserved: [0; 6], - }; - - // Create KVM guest_memfd. - let fd = vm_fd.create_guest_memfd(gmem).unwrap(); - - // Create memory region. - let mem_region = kvm_userspace_memory_region2 { - slot: 0, - flags: KVM_MEM_GUEST_MEMFD, - guest_phys_addr: region.start_addr().0, - memory_size: region.len(), - userspace_addr: host_addr as u64, - guest_memfd_offset: 0, - guest_memfd: fd as u32, - pad1: 0, - pad2: [0; 14], - }; - - // Set the memory region. - unsafe { - vm_fd.set_user_memory_region2(mem_region).unwrap(); - } - } - let vm_rfd = vm_fd.as_raw_fd(); let fw_rfd = self.fw.as_raw_fd(); diff --git a/src/vmm/src/linux/vstate.rs b/src/vmm/src/linux/vstate.rs index e755fc56..2d026c02 100644 --- a/src/vmm/src/linux/vstate.rs +++ b/src/vmm/src/linux/vstate.rs @@ -38,6 +38,8 @@ use crate::vmm_config::machine_config::CpuFeaturesTemplate; use arch::aarch64::gic::GICDevice; #[cfg(target_arch = "x86_64")] use cpuid::{c3, filter_cpuid, t2, VmSpec}; +#[cfg(not(feature = "amd-sev"))] +use kvm_bindings::kvm_userspace_memory_region; #[cfg(target_arch = "x86_64")] use kvm_bindings::{ kvm_clock_data, kvm_debugregs, kvm_irqchip, kvm_lapic_state, kvm_mp_state, kvm_pit_config, @@ -45,7 +47,11 @@ use kvm_bindings::{ Msrs, KVM_CLOCK_TSC_STABLE, KVM_IRQCHIP_IOAPIC, KVM_IRQCHIP_PIC_MASTER, KVM_IRQCHIP_PIC_SLAVE, KVM_MAX_CPUID_ENTRIES, KVM_PIT_SPEAKER_DUMMY, }; -use kvm_bindings::{kvm_userspace_memory_region, KVM_API_VERSION}; + +#[cfg(feature = "amd-sev")] +use kvm_bindings::{kvm_create_guest_memfd, kvm_userspace_memory_region2, KVM_MEM_GUEST_MEMFD}; + +use kvm_bindings::KVM_API_VERSION; use kvm_ioctls::*; use utils::eventfd::EventFd; use utils::signal::{register_signal_handler, sigrtmin, Killable}; @@ -562,6 +568,8 @@ impl Vm { if guest_mem.num_regions() > kvm_max_memslots { return Err(Error::NotEnoughMemorySlots); } + + #[cfg(not(feature = "amd-sev"))] for region in guest_mem.iter() { // It's safe to unwrap because the guest address is valid. let host_addr = guest_mem.get_host_address(region.start_addr()).unwrap(); @@ -583,6 +591,42 @@ impl Vm { self.next_mem_slot += 1; } + #[cfg(feature = "amd-sev")] + for region in guest_mem.iter() { + // It's safe to unwrap because the guest address is valid. + let host_addr = guest_mem.get_host_address(region.start_addr()).unwrap(); + + // Create guest_memfd struct. + let gmem = kvm_create_guest_memfd { + size: region.len(), + flags: 0, // Unused. + reserved: [0; 6], + }; + + // Create KVM guest_memfd. + let guest_memfd = self.fd.create_guest_memfd(gmem).unwrap(); + + // Create memory region. + let memory_region = kvm_userspace_memory_region2 { + slot: self.next_mem_slot, + flags: KVM_MEM_GUEST_MEMFD, + guest_phys_addr: region.start_addr().raw_value(), + memory_size: region.len(), + userspace_addr: host_addr as u64, + guest_memfd_offset: 0, + guest_memfd: guest_memfd as u32, + pad1: 0, + pad2: [0; 14], + }; + + // Set the memory region. + unsafe { + self.fd.set_user_memory_region2(memory_region).unwrap(); + } + + self.next_mem_slot += 1; + } + #[cfg(target_arch = "x86_64")] self.fd .set_tss_address(arch::x86_64::layout::KVM_TSS_ADDRESS as usize) From 3e916673b4e99c73c64ec967ec3d9c03f59f9441 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Tue, 17 Dec 2024 01:47:51 -0500 Subject: [PATCH 5/5] vmm/vstate: Set memory attributes of regions Co-developed-by: Jake Correnti Signed-off-by: Tyler Fanelli --- src/vmm/src/linux/vstate.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/vmm/src/linux/vstate.rs b/src/vmm/src/linux/vstate.rs index 2d026c02..28b0066c 100644 --- a/src/vmm/src/linux/vstate.rs +++ b/src/vmm/src/linux/vstate.rs @@ -49,7 +49,10 @@ use kvm_bindings::{ }; #[cfg(feature = "amd-sev")] -use kvm_bindings::{kvm_create_guest_memfd, kvm_userspace_memory_region2, KVM_MEM_GUEST_MEMFD}; +use kvm_bindings::{ + kvm_create_guest_memfd, kvm_memory_attributes, kvm_userspace_memory_region2, + KVM_MEM_GUEST_MEMFD, +}; use kvm_bindings::KVM_API_VERSION; use kvm_ioctls::*; @@ -119,6 +122,9 @@ pub enum Error { /// Error creating memory map for SHM region. ShmMmap(io::Error), #[cfg(feature = "amd-sev")] + /// Error setting KVM memory attributes of a region. + SetMemoryAttributes(kvm_ioctls::Error), + #[cfg(feature = "amd-sev")] /// Error initializing the Secure Virtualization Backend (SEV). SevSecVirtInit(SevError), #[cfg(feature = "amd-sev")] @@ -278,6 +284,10 @@ impl Display for Error { ), SetUserMemoryRegion(e) => write!(f, "Cannot set the memory regions: {e}"), ShmMmap(e) => write!(f, "Error creating memory map for SHM region: {e}"), + #[cfg(feature = "amd-sev")] + SetMemoryAttributes(e) => { + write!(f, "Error setting KVM memory attributes of a region: {e}") + } #[cfg(feature = "tee")] SevSecVirtInit(e) => { write!( @@ -624,6 +634,18 @@ impl Vm { self.fd.set_user_memory_region2(memory_region).unwrap(); } + let attr = kvm_memory_attributes { + address: region.start_addr().raw_value(), + size: region.len(), + // KVM_MEMORY_ATTRIBUTE_PRIVATE, + attributes: 1 << 3, + flags: 0, + }; + + self.fd + .set_memory_attributes(attr) + .map_err(Error::SetMemoryAttributes)?; + self.next_mem_slot += 1; }