Fix: Setup cgroupfs to run cells when running as PID1 (#533)

* fix: setup cgroupfs for running cells as pid1

* chore: remove unneeded deps

* fix: trait function name collisions

* fix(tests): set default executable to never-ending command

auraed/src/init/fs.rs

@@ -13,21 +13,48 @@
  * SPDX-License-Identifier: Apache-2.0                                        *
 \* -------------------------------------------------------------------------- */
 
-use nix::mount::MsFlags;
+use lazy_static::lazy_static;
+use nix::{mount::MsFlags, sys::stat::Mode};
 use std::io;
 use tracing::{error, info};
 
 #[derive(thiserror::Error, Debug)]
 pub(crate) enum FsError {
     #[error("Failed to mount {spec:?} due to error: {source}")]
     MountFailure { spec: MountSpec, source: io::Error },
+    #[error(transparent)]
+    FileCreationFailure(#[from] nix::errno::Errno),
+}
+
+lazy_static! {
+    pub static ref CHMOD_0755: Mode = Mode::S_IRWXU
+        | Mode::S_IRGRP
+        | Mode::S_IXGRP
+        | Mode::S_IROTH
+        | Mode::S_IXOTH;
+    pub static ref CHMOD_0555: Mode = Mode::S_IRUSR
+        | Mode::S_IXUSR
+        | Mode::S_IRGRP
+        | Mode::S_IXGRP
+        | Mode::S_IROTH
+        | Mode::S_IXOTH;
+    pub static ref CHMOD_1777: Mode =
+        Mode::S_IRWXU | Mode::S_IRWXG | Mode::S_IRWXO | Mode::S_ISVTX;
+    pub static ref COMMON_MNT_FLAGS: MsFlags =
+        MsFlags::MS_NODEV | MsFlags::MS_NOEXEC | MsFlags::MS_NOSUID;
+    pub static ref CGROUP_MNT_FLAGS: MsFlags = MsFlags::MS_NODEV
+        | MsFlags::MS_NOEXEC
+        | MsFlags::MS_NOSUID
+        | MsFlags::MS_RELATIME;
 }
 
 #[derive(Debug)]
 pub(crate) struct MountSpec {
     pub source: Option<&'static str>,
     pub target: &'static str,
     pub fstype: Option<&'static str>,
+    pub flags: MsFlags,
+    pub data: Option<&'static str>,
 }
 
 impl MountSpec {
@@ -38,8 +65,8 @@ impl MountSpec {
             self.source,
             self.target,
             self.fstype,
-            MsFlags::empty(),
-            None::<&str>,
+            self.flags,
+            self.data,
         ) {
             error!("Failed to mount {:?}", self);
             return Err(FsError::MountFailure {

auraed/src/init/system_runtimes/mod.rs

@@ -107,4 +107,4 @@ async fn create_tcp_socket_stream(
     let sock = TcpListener::bind(&socket_addr).await?;
     info!("TCP Access Socket created: {:?}", socket_addr);
     Ok(SocketStream::Tcp(TcpListenerStream::new(sock)))
-}
+}

auraed/src/init/system_runtimes/pid1_system_runtime.rs

@@ -15,8 +15,15 @@
 
 use super::{SocketStream, SystemRuntime, SystemRuntimeError};
 use crate::init::{
-    fs::MountSpec, logging, network, power::spawn_thread_power_button_listener,
-    system_runtimes::create_tcp_socket_stream, BANNER,
+    fs::{FsError, MountSpec, CGROUP_MNT_FLAGS, CHMOD_0755, COMMON_MNT_FLAGS},
+    logging, network,
+    power::spawn_thread_power_button_listener,
+    system_runtimes::create_tcp_socket_stream,
+    BANNER,
+};
+use nix::{
+    mount::MsFlags,
+    unistd::{mkdir, symlinkat},
 };
 use std::{net::SocketAddr, path::Path};
 use tonic::async_trait;
@@ -62,34 +69,69 @@ impl SystemRuntime for Pid1SystemRuntime {
         info!("Running as pid 1");
         trace!("Configure filesystem");
 
-        // NOTE: THESE TODOS WERE ALL HERE, BUT...
-        //       if you are here, you are auraed is true pid 1
-        //       Container -> use container_system_runtime.rs
-        //       Cell -> use cell_system_runtime.rs
-
-        // TODO We need to determine how we want to handle mountings these filesystems.
-        // TODO From within the context of a container (cgroup trailing / in cgroup namespace)
-        // TODO We likely to do not need to mount these filesystems.
-        // TODO Do we want to have a way to "try" these mounts and continue without erroring?
+        mkdir("/dev/pts", *CHMOD_0755).map_err(FsError::FileCreationFailure)?;
+        MountSpec {
+            source: Some("devpts"),
+            target: "/dev/pts",
+            fstype: Some("devpts"),
+            flags: MsFlags::MS_NOEXEC
+                | MsFlags::MS_NOSUID
+                | MsFlags::MS_NOATIME,
+            data: Some("mode=0620,gid=5,ptmxmode=666"),
+        }
+        .mount()?;
 
         MountSpec {
             source: Some("sysfs"),
             target: "/sys",
             fstype: Some("sysfs"),
+            flags: *COMMON_MNT_FLAGS,
+            data: None,
         }
         .mount()?;
 
         MountSpec {
             source: Some("proc"),
             target: "/proc",
             fstype: Some("proc"),
+            flags: *COMMON_MNT_FLAGS,
+            data: None,
+        }
+        .mount()?;
+
+        MountSpec {
+            source: Some("run"),
+            target: "/run",
+            fstype: Some("tmpfs"),
+            flags: MsFlags::MS_NOSUID | MsFlags::MS_NODEV,
+            data: Some("mode=0755"),
+        }
+        .mount()?;
+
+        symlinkat("/proc/self/fd", None, "/dev/fd")
+            .map_err(FsError::FileCreationFailure)?;
+        symlinkat("/proc/self/fd/0", None, "/dev/stdin")
+            .map_err(FsError::FileCreationFailure)?;
+        symlinkat("/proc/self/fd/1", None, "/dev/stdout")
+            .map_err(FsError::FileCreationFailure)?;
+        symlinkat("/proc/self/fd/2", None, "/dev/stderr")
+            .map_err(FsError::FileCreationFailure)?;
+
+        MountSpec {
+            source: Some("cgroup2"),
+            target: "/sys/fs/cgroup",
+            fstype: Some("cgroup2"),
+            flags: *CGROUP_MNT_FLAGS,
+            data: None,
         }
         .mount()?;
 
         MountSpec {
             source: Some("debugfs"),
             target: "/sys/kernel/debug",
             fstype: Some("debugfs"),
+            flags: *COMMON_MNT_FLAGS,
+            data: None,
         }
         .mount()?;
 
@@ -100,7 +142,6 @@ impl SystemRuntime for Pid1SystemRuntime {
         const DEFAULT_NET_DEV_IPV6_GATEWAY: &str = "fe80::1";
         const DEFAULT_NET_DEV_IPV6_SUBNET: &str = "/64";
 
-        // show_dir("/sys/class/net/", false); // Show available network interfaces
         let network = network::Network::connect()?;
         network
             .init(&network::Config {

auraed/tests/common/cells.rs

@@ -93,7 +93,7 @@ impl ExecutableBuilder {
     pub fn new() -> Self {
         Self {
             name: format!("ae-sleeper-{}", uuid::Uuid::new_v4()),
-            command: "sleep 400".to_string(),
+            command: "tail -f /dev/null".to_string(),
             description: String::from("description"),
         }
     }

auraed/tests/vms_start_must_start_vm_with_auraed.rs

@@ -13,10 +13,16 @@
  * SPDX-License-Identifier: Apache-2.0                                        *
 \* -------------------------------------------------------------------------- */
 
+use client::cells::cell_service::CellServiceClient;
 use client::discovery::discovery_service::DiscoveryServiceClient;
 use client::vms::vm_service::VmServiceClient;
 use client::{Client, ClientError};
+use common::cells::{
+    CellServiceAllocateRequestBuilder, CellServiceStartRequestBuilder,
+};
 use common::remote_auraed_client;
+use log::info;
+use proto::cells::{CellServiceFreeRequest, CellServiceStopRequest};
 use proto::discovery::DiscoverRequest;
 use proto::vms::{
     RootDrive, VirtualMachine, VmServiceAllocateRequest, VmServiceListRequest,
@@ -31,8 +37,9 @@ async fn vms_with_auraed() {
     let vm_id = format!("ae-test-vm-{}", uuid::Uuid::new_v4());
     let client = common::auraed_client().await;
     let res = retry!(
-        client
-            .allocate(VmServiceAllocateRequest {
+        VmServiceClient::allocate(
+            &client,
+            VmServiceAllocateRequest {
                 machine: Some(VirtualMachine {
                     id: vm_id.clone(),
                     mem_size_mb: 1024,
@@ -51,17 +58,20 @@ async fn vms_with_auraed() {
                     drive_mounts: vec![],
                     auraed_address: String::new()
                 }),
-            })
-            .await
+            }
+        )
+        .await
     );
 
     assert!(res.is_ok(), "{:?}", res);
 
-    let vm = client
-        .start(VmServiceStartRequest { vm_id: vm_id.clone() })
-        .await
-        .expect("failed to start vm")
-        .into_inner();
+    let vm = VmServiceClient::start(
+        &client,
+        VmServiceStartRequest { vm_id: vm_id.clone() },
+    )
+    .await
+    .expect("failed to start vm")
+    .into_inner();
 
     // Try for 5 seconds to get a client to the VM
     let mut remote_client: Result<Client, ClientError> =
@@ -70,8 +80,7 @@ async fn vms_with_auraed() {
         if remote_client.is_ok() {
             break;
         }
-        let vm = client
-            .list(VmServiceListRequest {})
+        let vm = VmServiceClient::list(&client, VmServiceListRequest {})
             .await
             .expect("failed to list vms")
             .into_inner()
@@ -89,13 +98,47 @@ async fn vms_with_auraed() {
 
     // NOTE: for now this passes when cloud-hypervisor is running a VM with auraed
     // as PID 1 with a tuntap device at the provided scope_id
-    let res = remote_client
-        .expect("could not build remote client")
-        .discover(DiscoverRequest {})
-        .await;
+    let remote_client = remote_client.expect("could not build remote client");
 
+    let res = remote_client.discover(DiscoverRequest {}).await;
     assert!(res.is_ok(), "{:?}", res);
 
     let res = res.expect("this shouldn't happen").into_inner();
     assert!(res.healthy);
+
+    let res = CellServiceClient::allocate(
+        &remote_client,
+        CellServiceAllocateRequestBuilder::new().build(),
+    )
+    .await;
+    assert!(res.is_ok(), "{:?}", res);
+    let cell_name = res.unwrap().into_inner().cell_name;
+    info!("Allocated cell: {}", cell_name);
+
+    let res = CellServiceClient::start(
+        &remote_client,
+        CellServiceStartRequestBuilder::new()
+            .cell_name(cell_name.clone())
+            .executable_name("sleeper".into())
+            .build(),
+    )
+    .await;
+    assert!(res.is_ok(), "{:?}", res);
+
+    CellServiceClient::stop(
+        &remote_client,
+        CellServiceStopRequest {
+            cell_name: Some(cell_name.clone()),
+            executable_name: "sleeper".into(),
+        },
+    )
+    .await
+    .expect("failed to stop cell");
+
+    CellServiceClient::free(
+        &remote_client,
+        CellServiceFreeRequest { cell_name },
+    )
+    .await
+    .expect("failed to free cell");
 }

client/macros/src/service.rs

@@ -134,4 +134,4 @@ pub(crate) fn service(input: TokenStream) -> TokenStream {
     };
 
     expanded.into()
-}
+}