From dc59761695a341dc9db3a037f9447e2d887aca86 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Thu, 7 Sep 2023 15:26:23 -0700 Subject: [PATCH 1/6] Clean up indentation in run-enclave performance test script This doesn't change the way it's interpreted, but it was bugging me. --- perf-tests/run_enclave.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perf-tests/run_enclave.sh b/perf-tests/run_enclave.sh index 77eb418d..2d556e56 100755 --- a/perf-tests/run_enclave.sh +++ b/perf-tests/run_enclave.sh @@ -25,9 +25,9 @@ cat > ${config} <<- EOF cpu_count: ${VCPUS} egress: allow: - - "**" + - "**" ingress: - - listen_port: 8082 + - listen_port: 8082 EOF enclaver build -f ${config} From 37de4dd253bb6de7c73d2a4c5ba07fe6c50ddf31 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Thu, 7 Sep 2023 15:28:28 -0700 Subject: [PATCH 2/6] Minor cleanup of translation from CLISuccess to ExitCode --- enclaver/src/bin/enclaver-run/main.rs | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/enclaver/src/bin/enclaver-run/main.rs b/enclaver/src/bin/enclaver-run/main.rs index f7859230..3cdf65ac 100644 --- a/enclaver/src/bin/enclaver-run/main.rs +++ b/enclaver/src/bin/enclaver-run/main.rs @@ -54,20 +54,15 @@ enum CLISuccess { impl Termination for CLISuccess { fn report(self) -> ExitCode { + use CLISuccess::*; + use EnclaveExitStatus::*; + match self { - CLISuccess::EnclaveStatus(EnclaveExitStatus::Exited(code)) => { - ExitCode::from(code as u8) - } - CLISuccess::EnclaveStatus(EnclaveExitStatus::Signaled(_signal)) => { - ExitCode::from(ENCLAVE_SIGNALED_EXIT_CODE) - } - CLISuccess::EnclaveStatus(EnclaveExitStatus::Fatal(_err)) => { - ExitCode::from(ENCLAVE_FATAL) - } - CLISuccess::EnclaveStatus(EnclaveExitStatus::Cancelled) => { - ExitCode::from(ENCLAVER_INTERRUPTED) - }, - CLISuccess::Ok => ExitCode::SUCCESS, + EnclaveStatus(Exited(code)) => ExitCode::from(code as u8), + EnclaveStatus(Signaled(_signal)) => ExitCode::from(ENCLAVE_SIGNALED_EXIT_CODE), + EnclaveStatus(Fatal(_err)) => ExitCode::from(ENCLAVE_FATAL), + EnclaveStatus(Cancelled) => ExitCode::from(ENCLAVER_INTERRUPTED), + Ok => ExitCode::SUCCESS, } } } From c5626fd71487e7b70bf6e868a1e9425cfa7810b5 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Fri, 8 Sep 2023 10:47:39 -0700 Subject: [PATCH 3/6] Don't build with --all-features This makes it tricky to add optional development features, since we might not want to release with those features enabled. --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/release.yaml | 4 ++-- build/local_image_deps.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e2d503aa..320a16e5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,10 +21,10 @@ jobs: - run: rustup toolchain install $RUSTUP_TOOLCHAIN - run: rustup component add clippy - # Check with --all-features (ie a Linux build) + # Check all binaries (ie a Linux build) - uses: actions-rs/clippy-check@v1.0.7 with: - args: --all-features --manifest-path enclaver/Cargo.toml + args: --features=run_enclave,odyn --manifest-path enclaver/Cargo.toml token: ${{ secrets.GITHUB_TOKEN }} # Check with only default features (ie a Mac build) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4757108f..455546d7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -15,11 +15,11 @@ jobs: - target: 'x86_64-unknown-linux-musl' host: 'ubuntu-latest' uses_musl: true - extra_cargo_args: '--all-features' + extra_cargo_args: '--features=run_enclave,odyn' - target: 'aarch64-unknown-linux-musl' host: 'ubuntu-latest' uses_musl: true - extra_cargo_args: '--all-features' + extra_cargo_args: '--features=run_enclave,odyn' - target: 'x86_64-apple-darwin' host: 'macos-latest' - target: 'aarch64-apple-darwin' diff --git a/build/local_image_deps.sh b/build/local_image_deps.sh index c68c028b..ddff2c05 100755 --- a/build/local_image_deps.sh +++ b/build/local_image_deps.sh @@ -27,7 +27,7 @@ cd $enclaver_dir docker_build_dir=$(mktemp -d) trap "rm --force --recursive ${docker_build_dir}" EXIT -cargo build --target $rust_target --all-features +cargo build --target $rust_target --features run_enclave,odyn cp $rust_target_dir/odyn $docker_build_dir/ cp $rust_target_dir/enclaver-run $docker_build_dir/ From ac602d63b85fcc7da137dfe101de9703649018e2 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Fri, 8 Sep 2023 11:11:11 -0700 Subject: [PATCH 4/6] Be explicit about use of buildx Docker plugin The build failures can be rather confusing for folks who aren't using Docker Desktop. At least with buildx in the invocation, there's a hint that an additional component is necessary. --- build/local_image_deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/local_image_deps.sh b/build/local_image_deps.sh index ddff2c05..3f93a44d 100755 --- a/build/local_image_deps.sh +++ b/build/local_image_deps.sh @@ -32,12 +32,12 @@ cargo build --target $rust_target --features run_enclave,odyn cp $rust_target_dir/odyn $docker_build_dir/ cp $rust_target_dir/enclaver-run $docker_build_dir/ -docker build \ +docker buildx build \ -f ../build/dockerfiles/odyn-dev.dockerfile \ -t ${odyn_tag} \ ${docker_build_dir} -docker build \ +docker buildx build \ -f ../build/dockerfiles/runtimebase-dev.dockerfile \ -t ${wrapper_base_tag} \ ${docker_build_dir} From 442686c57188a99849d7a657dcb93a63b59709be Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Mon, 11 Sep 2023 17:01:46 -0700 Subject: [PATCH 5/6] Run enclaver by default when unspecified This allows you to use `cargo run` without having to specify which binary. --- enclaver/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/enclaver/Cargo.toml b/enclaver/Cargo.toml index e0a24890..2bd9a08e 100644 --- a/enclaver/Cargo.toml +++ b/enclaver/Cargo.toml @@ -3,6 +3,7 @@ name = "enclaver" version = "0.2.0" edition = "2021" rust-version = "1.68" +default-run = "enclaver" [[bin]] name = "odyn" From ae8e6cccb2441ac038df096dd04d9d350a67ee7e Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Thu, 7 Sep 2023 17:16:45 -0700 Subject: [PATCH 6/6] Enable tracing behind feature flag This adds the "tracing" feature, which enables tokio's tracing functionality. This allows `tokio-console` to attach to a running instance and display interesting (and hopefully helpful) information about the async tasks. In addition to the feature, the "tokio_unstable" config option needs to be passed to rustc: RUSTFLAGS="--cfg tokio_unstable" cargo build ... In order to allow multiple layers of the enclave to be measured, the console subscriber in each binary does not run on the default port. You'll need to specify which one you want to connect to: - 51000 - odyn - 51001 - enclaver-run (supervisor) - 51002 - enclaver Note that both odyn and enclaver-run are configured by default to listen on all interfaces. This is to make it easier to connect tokio-console to these processes which run in a seperate network context, but be careful if running manually or with `--net=host`, as anyone on the network will be able to connect. Also note that when enclaver creates the container image tarball (via tokio_tar), many tasks are spawned; so many that it will likely exhaust the memory of your machine. Consider lowering the retention time substantially using `TOKIO_CONSOLE_RETENTION`. `1s` seemed to work okay for a machine with 4 GiB free. --- .github/workflows/ci.yaml | 8 + enclaver/Cargo.lock | 326 +++++++++++++++++++++++++- enclaver/Cargo.toml | 3 +- enclaver/src/bin/enclaver-run/main.rs | 11 +- enclaver/src/bin/enclaver/main.rs | 5 + enclaver/src/bin/odyn/main.rs | 6 + enclaver/src/proxy/egress_http.rs | 6 +- enclaver/src/proxy/ingress.rs | 12 +- enclaver/src/run.rs | 20 +- enclaver/src/utils.rs | 25 +- 10 files changed, 395 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 320a16e5..e67c613a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,14 @@ jobs: args: --features=run_enclave,odyn --manifest-path enclaver/Cargo.toml token: ${{ secrets.GITHUB_TOKEN }} + # Check with tracing enabled + - uses: actions-rs/clippy-check@v1.0.7 + env: + RUSTFLAGS: "-Dwarnings --cfg=tokio_unstable" + with: + args: --features=run_enclave,odyn,tracing --manifest-path enclaver/Cargo.toml + token: ${{ secrets.GITHUB_TOKEN }} + # Check with only default features (ie a Mac build) - uses: actions-rs/clippy-check@v1.0.7 with: diff --git a/enclaver/Cargo.lock b/enclaver/Cargo.lock index 91fcba0c..e9f05dde 100644 --- a/enclaver/Cargo.lock +++ b/enclaver/Cargo.lock @@ -406,6 +406,51 @@ dependencies = [ "zeroize", ] +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.2.1", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.68" @@ -614,6 +659,42 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "console-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2895653b4d9f1538a83970077cb01dfc77a4810524e51a110944688e916b18e" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures", + "hdrhistogram", + "humantime 2.1.0", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.5" @@ -645,6 +726,34 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -765,6 +874,7 @@ dependencies = [ "cbc", "circbuf", "clap", + "console-subscriber", "form_urlencoded", "futures", "futures-util", @@ -820,7 +930,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" dependencies = [ "atty", - "humantime", + "humantime 1.3.0", "log", "regex", "termcolor", @@ -880,6 +990,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1048,6 +1168,19 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +[[package]] +name = "hdrhistogram" +version = "7.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" +dependencies = [ + "base64 0.13.1", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "headers" version = "0.3.8" @@ -1143,6 +1276,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -1204,6 +1343,18 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyperlocal" version = "0.8.0" @@ -1304,6 +1455,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1368,6 +1528,21 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matchit" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" + [[package]] name = "memchr" version = "2.5.0" @@ -1748,6 +1923,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1810,8 +2017,17 @@ checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-automata", - "regex-syntax", + "regex-automata 0.3.6", + "regex-syntax 0.7.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", ] [[package]] @@ -1822,9 +2038,15 @@ checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.4", ] +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + [[package]] name = "regex-syntax" version = "0.7.4" @@ -2001,6 +2223,12 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.15" @@ -2190,6 +2418,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -2294,6 +2531,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "synstructure" version = "0.12.6" @@ -2354,6 +2597,16 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "thread_local" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +dependencies = [ + "cfg-if", + "once_cell", +] + [[package]] name = "time" version = "0.3.25" @@ -2426,9 +2679,20 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.3", "tokio-macros", + "tracing", "windows-sys", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -2525,6 +2789,34 @@ dependencies = [ "vsock", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64 0.21.2", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -2533,9 +2825,13 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -2584,6 +2880,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers", + "once_cell", + "regex", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", ] [[package]] @@ -2663,6 +2975,12 @@ dependencies = [ "getrandom", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.4" diff --git a/enclaver/Cargo.toml b/enclaver/Cargo.toml index 2bd9a08e..f773f848 100644 --- a/enclaver/Cargo.toml +++ b/enclaver/Cargo.toml @@ -70,7 +70,7 @@ cbc = { version = "0.1", features = [ "std", "block-padding" ] } aes = "0.8" sha2 = "0.10" ignore-result = "0.2.0" - +console-subscriber = { version = "0.1.10", optional = true } [dev-dependencies] assert2 = "0.3" @@ -83,3 +83,4 @@ run_enclave = ["proxy"] odyn = ["vsock", "proxy"] proxy = ["vsock"] vsock = ["dep:tokio-vsock", "dep:rtnetlink"] +tracing = ["dep:console-subscriber", "tokio/tracing"] diff --git a/enclaver/src/bin/enclaver-run/main.rs b/enclaver/src/bin/enclaver-run/main.rs index 3cdf65ac..3ebf6ff4 100644 --- a/enclaver/src/bin/enclaver-run/main.rs +++ b/enclaver/src/bin/enclaver-run/main.rs @@ -4,6 +4,7 @@ use enclaver::constants::{MANIFEST_FILE_NAME, RELEASE_BUNDLE_DIR, EIF_FILE_NAME} use enclaver::run::{Enclave, EnclaveExitStatus, EnclaveOpts}; use enclaver::manifest::load_manifest_raw; use enclaver::nitro_cli::NitroCLI; +use enclaver::utils; use log::info; use std::{ path::PathBuf, @@ -85,11 +86,11 @@ async fn run(args: Cli) -> Result { // enclave run. let cancel_task = { let cancellation = cancellation.clone(); - tokio::task::spawn(async move { + utils::spawn!("shutdown handler", async move { shutdown_signal.await; cancellation.cancel(); info!("shutdown signal received, terminating enclave"); - }) + })? }; let status = enclave.run(cancellation).await?; @@ -122,6 +123,12 @@ async fn describe_eif() -> Result { async fn main() -> Result { enclaver::utils::init_logging(); + #[cfg(feature = "tracing")] + console_subscriber::ConsoleLayer::builder() + .with_default_env() + .server_addr(([0, 0, 0, 0], 51001)) + .init(); + let args = Cli::parse(); match args.sub_command { diff --git a/enclaver/src/bin/enclaver/main.rs b/enclaver/src/bin/enclaver/main.rs index fc2e1c6b..d8dab160 100644 --- a/enclaver/src/bin/enclaver/main.rs +++ b/enclaver/src/bin/enclaver/main.rs @@ -160,5 +160,10 @@ async fn main() -> Result<()> { let args = Cli::parse(); + #[cfg(feature = "tracing")] + console_subscriber::ConsoleLayer::builder() + .with_default_env() + .server_addr(([127, 0, 0, 1], 51002)); + run(args).await } diff --git a/enclaver/src/bin/odyn/main.rs b/enclaver/src/bin/odyn/main.rs index 988184bd..75f82710 100644 --- a/enclaver/src/bin/odyn/main.rs +++ b/enclaver/src/bin/odyn/main.rs @@ -101,6 +101,12 @@ async fn main() { enclaver::utils::init_logging(); let args = CliArgs::parse(); + #[cfg(feature = "tracing")] + console_subscriber::ConsoleLayer::builder() + .with_default_env() + .server_addr(([0, 0, 0, 0], 51000)) + .init(); + if let Err(err) = run(&args).await { error!("Error: {err:#}"); std::process::exit(1); diff --git a/enclaver/src/proxy/egress_http.rs b/enclaver/src/proxy/egress_http.rs index 00f9aafd..fb5f7d68 100644 --- a/enclaver/src/proxy/egress_http.rs +++ b/enclaver/src/proxy/egress_http.rs @@ -1,6 +1,7 @@ use std::net::{Ipv4Addr, SocketAddrV4}; use std::sync::Arc; +use crate::utils; use anyhow::anyhow; use async_trait::async_trait; use futures::{Stream, StreamExt}; @@ -100,9 +101,10 @@ impl EnclaveHttpProxy { Ok((sock, _)) => { let egress_policy = egress_policy.clone(); - tokio::task::spawn(async move { + utils::spawn!("egress stream", async move { EnclaveHttpProxy::service_conn(sock, egress_port, egress_policy).await; - }); + }) + .expect("spawn egress stream"); } Err(err) => { error!("Accept failed: {err}"); diff --git a/enclaver/src/proxy/ingress.rs b/enclaver/src/proxy/ingress.rs index 5004329e..91bce458 100644 --- a/enclaver/src/proxy/ingress.rs +++ b/enclaver/src/proxy/ingress.rs @@ -1,7 +1,7 @@ use std::net::{Ipv4Addr, SocketAddrV4}; use std::sync::Arc; -use crate::vsock; +use crate::{utils, vsock}; use anyhow::Result; use futures::{Stream, StreamExt}; use log::{debug, error}; @@ -53,9 +53,10 @@ where let mut incoming = Box::into_pin(self.incoming); while let Some(stream) = incoming.next().await { - tokio::task::spawn(async move { + utils::spawn!("ingress stream", async move { EnclaveProxy::service_conn(stream, addr).await; - }); + }) + .expect("spawn ingress stream"); } } @@ -89,9 +90,10 @@ impl HostProxy { pub async fn serve(self, target_cid: u32, target_port: u32) { while let Ok((sock, _)) = self.listener.accept().await { // TODO: don't use detached tasks - tokio::task::spawn(async move { + utils::spawn!(&format!("host proxy ({target_port})"), async move { HostProxy::service_conn(sock, target_cid, target_port).await; - }); + }) + .expect("spawn host proxy"); } } diff --git a/enclaver/src/run.rs b/enclaver/src/run.rs index 7a73f363..12ad29b0 100644 --- a/enclaver/src/run.rs +++ b/enclaver/src/run.rs @@ -144,7 +144,7 @@ impl Enclave { self.attach_debug_console(&enclave_info.id).await?; } - self.start_odyn_log_stream(enclave_info.cid); + self.start_odyn_log_stream(enclave_info.cid)?; self.start_ingress_proxies(enclave_info.cid).await?; @@ -188,9 +188,9 @@ impl Enclave { let listen_port = item.listen_port; info!("starting ingress proxy on port {listen_port}"); let proxy = HostProxy::bind(listen_port).await?; - self.tasks.push(tokio::task::spawn(async move { + self.tasks.push(utils::spawn!("ingress proxy", async move { proxy.serve(cid, listen_port.into()).await; - })) + })?) } Ok(()) @@ -206,15 +206,15 @@ impl Enclave { info!("starting egress proxy on vsock port {HTTP_EGRESS_VSOCK_PORT}"); let proxy = HostHttpProxy::bind(HTTP_EGRESS_VSOCK_PORT)?; - self.tasks.push(tokio::task::spawn(async move { + self.tasks.push(utils::spawn!("egress proxy", async move { proxy.serve().await; - })); + })?); Ok(()) } - fn start_odyn_log_stream(&mut self, cid: u32) { - self.tasks.push(tokio::task::spawn(async move { + fn start_odyn_log_stream(&mut self, cid: u32) -> Result<()> { + self.tasks.push(utils::spawn!("odyn log stream", async move { info!("waiting for enclave to boot to stream logs"); let conn = loop { match VsockStream::connect(cid, APP_LOG_PORT).await { @@ -231,7 +231,9 @@ impl Enclave { if let Err(e) = utils::log_lines_from_stream("enclave", conn).await { error!("error reading log lines from enclave: {e}"); } - })); + })?); + + Ok(()) } async fn await_exit(cid: u32) -> Result { @@ -352,4 +354,4 @@ pub enum EnclaveExitStatus { Exited(i32), Signaled(i32), Fatal(String), -} \ No newline at end of file +} diff --git a/enclaver/src/utils.rs b/enclaver/src/utils.rs index 5c9e807e..93cdd310 100644 --- a/enclaver/src/utils.rs +++ b/enclaver/src/utils.rs @@ -9,6 +9,24 @@ use tokio_util::codec::{FramedRead, LinesCodec}; const LOG_LINE_MAX_LEN: usize = 4 * 1024; +#[cfg(feature = "tracing")] +#[macro_export] +macro_rules! spawn { + ($name:expr, $body:expr) => {{ + tokio::task::Builder::new().name($name).spawn($body) + }}; +} + +#[cfg(not(feature = "tracing"))] +#[macro_export] +macro_rules! spawn { + ($name:expr, $body:expr) => {{ + Result::<_, anyhow::Error>::Ok(tokio::task::spawn($body)) + }}; +} + +pub use spawn; + pub fn init_logging() { if std::env::var("RUST_LOG").is_err() { std::env::set_var("RUST_LOG", "info"); @@ -54,12 +72,11 @@ pub async fn register_shutdown_signal_handler() -> Result { let mut sigint = signal(SignalKind::interrupt())?; let mut sigterm = signal(SignalKind::terminate())?; - let f = tokio::task::spawn(async move { + spawn!("signal handler", async move { tokio::select! { _ = sigint.recv() => (), _ = sigterm.recv() => (), } - }); - - Ok(f) + }) + .map_err(Into::into) }