From 1d62cb044fc973b3b8006466c31ef3f35ecbca63 Mon Sep 17 00:00:00 2001 From: Tommy McCormick <16668493+mccormickt@users.noreply.github.com> Date: Sun, 10 Nov 2024 14:33:45 -0500 Subject: [PATCH] Fix: Setup cgroupfs to run cells when running as PID1 (#533) * fix: setup cgroupfs for running cells as pid1 * chore: remove unneeded deps * fix: trait function name collisions * fix(tests): set default executable to never-ending command --- auraed/src/init/fs.rs | 33 ++++++++- auraed/src/init/system_runtimes/mod.rs | 2 +- .../system_runtimes/pid1_system_runtime.rs | 65 ++++++++++++++--- auraed/tests/common/cells.rs | 2 +- .../vms_start_must_start_vm_with_auraed.rs | 73 +++++++++++++++---- client/macros/src/service.rs | 2 +- 6 files changed, 144 insertions(+), 33 deletions(-) diff --git a/auraed/src/init/fs.rs b/auraed/src/init/fs.rs index b4c35191c..404d10aee 100644 --- a/auraed/src/init/fs.rs +++ b/auraed/src/init/fs.rs @@ -13,7 +13,8 @@ * SPDX-License-Identifier: Apache-2.0 * \* -------------------------------------------------------------------------- */ -use nix::mount::MsFlags; +use lazy_static::lazy_static; +use nix::{mount::MsFlags, sys::stat::Mode}; use std::io; use tracing::{error, info}; @@ -21,6 +22,30 @@ use tracing::{error, info}; pub(crate) enum FsError { #[error("Failed to mount {spec:?} due to error: {source}")] MountFailure { spec: MountSpec, source: io::Error }, + #[error(transparent)] + FileCreationFailure(#[from] nix::errno::Errno), +} + +lazy_static! { + pub static ref CHMOD_0755: Mode = Mode::S_IRWXU + | Mode::S_IRGRP + | Mode::S_IXGRP + | Mode::S_IROTH + | Mode::S_IXOTH; + pub static ref CHMOD_0555: Mode = Mode::S_IRUSR + | Mode::S_IXUSR + | Mode::S_IRGRP + | Mode::S_IXGRP + | Mode::S_IROTH + | Mode::S_IXOTH; + pub static ref CHMOD_1777: Mode = + Mode::S_IRWXU | Mode::S_IRWXG | Mode::S_IRWXO | Mode::S_ISVTX; + pub static ref COMMON_MNT_FLAGS: MsFlags = + MsFlags::MS_NODEV | MsFlags::MS_NOEXEC | MsFlags::MS_NOSUID; + pub static ref CGROUP_MNT_FLAGS: MsFlags = MsFlags::MS_NODEV + | MsFlags::MS_NOEXEC + | MsFlags::MS_NOSUID + | MsFlags::MS_RELATIME; } #[derive(Debug)] @@ -28,6 +53,8 @@ pub(crate) struct MountSpec { pub source: Option<&'static str>, pub target: &'static str, pub fstype: Option<&'static str>, + pub flags: MsFlags, + pub data: Option<&'static str>, } impl MountSpec { @@ -38,8 +65,8 @@ impl MountSpec { self.source, self.target, self.fstype, - MsFlags::empty(), - None::<&str>, + self.flags, + self.data, ) { error!("Failed to mount {:?}", self); return Err(FsError::MountFailure { diff --git a/auraed/src/init/system_runtimes/mod.rs b/auraed/src/init/system_runtimes/mod.rs index d36fe9d73..facf27898 100644 --- a/auraed/src/init/system_runtimes/mod.rs +++ b/auraed/src/init/system_runtimes/mod.rs @@ -107,4 +107,4 @@ async fn create_tcp_socket_stream( let sock = TcpListener::bind(&socket_addr).await?; info!("TCP Access Socket created: {:?}", socket_addr); Ok(SocketStream::Tcp(TcpListenerStream::new(sock))) -} \ No newline at end of file +} diff --git a/auraed/src/init/system_runtimes/pid1_system_runtime.rs b/auraed/src/init/system_runtimes/pid1_system_runtime.rs index 5221b055c..f3f4d388d 100644 --- a/auraed/src/init/system_runtimes/pid1_system_runtime.rs +++ b/auraed/src/init/system_runtimes/pid1_system_runtime.rs @@ -15,8 +15,15 @@ use super::{SocketStream, SystemRuntime, SystemRuntimeError}; use crate::init::{ - fs::MountSpec, logging, network, power::spawn_thread_power_button_listener, - system_runtimes::create_tcp_socket_stream, BANNER, + fs::{FsError, MountSpec, CGROUP_MNT_FLAGS, CHMOD_0755, COMMON_MNT_FLAGS}, + logging, network, + power::spawn_thread_power_button_listener, + system_runtimes::create_tcp_socket_stream, + BANNER, +}; +use nix::{ + mount::MsFlags, + unistd::{mkdir, symlinkat}, }; use std::{net::SocketAddr, path::Path}; use tonic::async_trait; @@ -62,20 +69,24 @@ impl SystemRuntime for Pid1SystemRuntime { info!("Running as pid 1"); trace!("Configure filesystem"); - // NOTE: THESE TODOS WERE ALL HERE, BUT... - // if you are here, you are auraed is true pid 1 - // Container -> use container_system_runtime.rs - // Cell -> use cell_system_runtime.rs - - // TODO We need to determine how we want to handle mountings these filesystems. - // TODO From within the context of a container (cgroup trailing / in cgroup namespace) - // TODO We likely to do not need to mount these filesystems. - // TODO Do we want to have a way to "try" these mounts and continue without erroring? + mkdir("/dev/pts", *CHMOD_0755).map_err(FsError::FileCreationFailure)?; + MountSpec { + source: Some("devpts"), + target: "/dev/pts", + fstype: Some("devpts"), + flags: MsFlags::MS_NOEXEC + | MsFlags::MS_NOSUID + | MsFlags::MS_NOATIME, + data: Some("mode=0620,gid=5,ptmxmode=666"), + } + .mount()?; MountSpec { source: Some("sysfs"), target: "/sys", fstype: Some("sysfs"), + flags: *COMMON_MNT_FLAGS, + data: None, } .mount()?; @@ -83,6 +94,35 @@ impl SystemRuntime for Pid1SystemRuntime { source: Some("proc"), target: "/proc", fstype: Some("proc"), + flags: *COMMON_MNT_FLAGS, + data: None, + } + .mount()?; + + MountSpec { + source: Some("run"), + target: "/run", + fstype: Some("tmpfs"), + flags: MsFlags::MS_NOSUID | MsFlags::MS_NODEV, + data: Some("mode=0755"), + } + .mount()?; + + symlinkat("/proc/self/fd", None, "/dev/fd") + .map_err(FsError::FileCreationFailure)?; + symlinkat("/proc/self/fd/0", None, "/dev/stdin") + .map_err(FsError::FileCreationFailure)?; + symlinkat("/proc/self/fd/1", None, "/dev/stdout") + .map_err(FsError::FileCreationFailure)?; + symlinkat("/proc/self/fd/2", None, "/dev/stderr") + .map_err(FsError::FileCreationFailure)?; + + MountSpec { + source: Some("cgroup2"), + target: "/sys/fs/cgroup", + fstype: Some("cgroup2"), + flags: *CGROUP_MNT_FLAGS, + data: None, } .mount()?; @@ -90,6 +130,8 @@ impl SystemRuntime for Pid1SystemRuntime { source: Some("debugfs"), target: "/sys/kernel/debug", fstype: Some("debugfs"), + flags: *COMMON_MNT_FLAGS, + data: None, } .mount()?; @@ -100,7 +142,6 @@ impl SystemRuntime for Pid1SystemRuntime { const DEFAULT_NET_DEV_IPV6_GATEWAY: &str = "fe80::1"; const DEFAULT_NET_DEV_IPV6_SUBNET: &str = "/64"; - // show_dir("/sys/class/net/", false); // Show available network interfaces let network = network::Network::connect()?; network .init(&network::Config { diff --git a/auraed/tests/common/cells.rs b/auraed/tests/common/cells.rs index 2c14a82e7..9c4c68448 100644 --- a/auraed/tests/common/cells.rs +++ b/auraed/tests/common/cells.rs @@ -93,7 +93,7 @@ impl ExecutableBuilder { pub fn new() -> Self { Self { name: format!("ae-sleeper-{}", uuid::Uuid::new_v4()), - command: "sleep 400".to_string(), + command: "tail -f /dev/null".to_string(), description: String::from("description"), } } diff --git a/auraed/tests/vms_start_must_start_vm_with_auraed.rs b/auraed/tests/vms_start_must_start_vm_with_auraed.rs index 3c7dabecf..37ccae157 100644 --- a/auraed/tests/vms_start_must_start_vm_with_auraed.rs +++ b/auraed/tests/vms_start_must_start_vm_with_auraed.rs @@ -13,10 +13,16 @@ * SPDX-License-Identifier: Apache-2.0 * \* -------------------------------------------------------------------------- */ +use client::cells::cell_service::CellServiceClient; use client::discovery::discovery_service::DiscoveryServiceClient; use client::vms::vm_service::VmServiceClient; use client::{Client, ClientError}; +use common::cells::{ + CellServiceAllocateRequestBuilder, CellServiceStartRequestBuilder, +}; use common::remote_auraed_client; +use log::info; +use proto::cells::{CellServiceFreeRequest, CellServiceStopRequest}; use proto::discovery::DiscoverRequest; use proto::vms::{ RootDrive, VirtualMachine, VmServiceAllocateRequest, VmServiceListRequest, @@ -31,8 +37,9 @@ async fn vms_with_auraed() { let vm_id = format!("ae-test-vm-{}", uuid::Uuid::new_v4()); let client = common::auraed_client().await; let res = retry!( - client - .allocate(VmServiceAllocateRequest { + VmServiceClient::allocate( + &client, + VmServiceAllocateRequest { machine: Some(VirtualMachine { id: vm_id.clone(), mem_size_mb: 1024, @@ -51,17 +58,20 @@ async fn vms_with_auraed() { drive_mounts: vec![], auraed_address: String::new() }), - }) - .await + } + ) + .await ); assert!(res.is_ok(), "{:?}", res); - let vm = client - .start(VmServiceStartRequest { vm_id: vm_id.clone() }) - .await - .expect("failed to start vm") - .into_inner(); + let vm = VmServiceClient::start( + &client, + VmServiceStartRequest { vm_id: vm_id.clone() }, + ) + .await + .expect("failed to start vm") + .into_inner(); // Try for 5 seconds to get a client to the VM let mut remote_client: Result = @@ -70,8 +80,7 @@ async fn vms_with_auraed() { if remote_client.is_ok() { break; } - let vm = client - .list(VmServiceListRequest {}) + let vm = VmServiceClient::list(&client, VmServiceListRequest {}) .await .expect("failed to list vms") .into_inner() @@ -89,13 +98,47 @@ async fn vms_with_auraed() { // NOTE: for now this passes when cloud-hypervisor is running a VM with auraed // as PID 1 with a tuntap device at the provided scope_id - let res = remote_client - .expect("could not build remote client") - .discover(DiscoverRequest {}) - .await; + let remote_client = remote_client.expect("could not build remote client"); + let res = remote_client.discover(DiscoverRequest {}).await; assert!(res.is_ok(), "{:?}", res); let res = res.expect("this shouldn't happen").into_inner(); assert!(res.healthy); + + let res = CellServiceClient::allocate( + &remote_client, + CellServiceAllocateRequestBuilder::new().build(), + ) + .await; + assert!(res.is_ok(), "{:?}", res); + let cell_name = res.unwrap().into_inner().cell_name; + info!("Allocated cell: {}", cell_name); + + let res = CellServiceClient::start( + &remote_client, + CellServiceStartRequestBuilder::new() + .cell_name(cell_name.clone()) + .executable_name("sleeper".into()) + .build(), + ) + .await; + assert!(res.is_ok(), "{:?}", res); + + CellServiceClient::stop( + &remote_client, + CellServiceStopRequest { + cell_name: Some(cell_name.clone()), + executable_name: "sleeper".into(), + }, + ) + .await + .expect("failed to stop cell"); + + CellServiceClient::free( + &remote_client, + CellServiceFreeRequest { cell_name }, + ) + .await + .expect("failed to free cell"); } diff --git a/client/macros/src/service.rs b/client/macros/src/service.rs index 1f9fbc930..8d929a9c6 100644 --- a/client/macros/src/service.rs +++ b/client/macros/src/service.rs @@ -134,4 +134,4 @@ pub(crate) fn service(input: TokenStream) -> TokenStream { }; expanded.into() -} \ No newline at end of file +}