From 93bfd1aa5b3755087ee1a981adc7a6c873eb8d36 Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Wed, 9 Oct 2024 11:51:59 +0200 Subject: [PATCH 01/12] feat: intel-sgx attestation --- crates/notary/server/Cargo.toml | 8 + crates/notary/server/build.rs | 35 ++-- crates/notary/server/src/domain.rs | 6 +- crates/notary/server/src/lib.rs | 2 + crates/notary/server/src/server.rs | 13 ++ crates/notary/server/src/tee.rs | 174 ++++++++++++++++++ crates/notary/server/tee/Dockerfile | 25 +++ crates/notary/server/tee/Makefile | 63 +++++++ crates/notary/server/tee/README.md | 21 +++ crates/notary/server/tee/config/config.yaml | 45 +++++ .../tee/notary-server.manifest.template | 44 +++++ 11 files changed, 420 insertions(+), 16 deletions(-) create mode 100644 crates/notary/server/src/tee.rs create mode 100644 crates/notary/server/tee/Dockerfile create mode 100644 crates/notary/server/tee/Makefile create mode 100644 crates/notary/server/tee/README.md create mode 100644 crates/notary/server/tee/config/config.yaml create mode 100644 crates/notary/server/tee/notary-server.manifest.template diff --git a/crates/notary/server/Cargo.toml b/crates/notary/server/Cargo.toml index e2e1087e0e..e19f859091 100644 --- a/crates/notary/server/Cargo.toml +++ b/crates/notary/server/Cargo.toml @@ -3,6 +3,9 @@ name = "notary-server" version = "0.1.0-alpha.7" edition = "2021" +[features] +tee_quote = ["dep:mc-sgx-dcap-types", "dep:hex", "dep:rand_chacha", "dep:once_cell"] + [dependencies] tlsn-core = { workspace = true } tlsn-common = { workspace = true } @@ -49,3 +52,8 @@ tracing-subscriber = { workspace = true, features = ["env-filter"] } uuid = { workspace = true, features = ["v4", "fast-rng"] } ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] } zeroize = { workspace = true } + +mc-sgx-dcap-types = { version = "0.11.0", optional = true } +hex = { workspace = true, optional = true } +rand_chacha = { workspace = true, optional = true } +once_cell = { workspace = true, optional =true } \ No newline at end of file diff --git a/crates/notary/server/build.rs b/crates/notary/server/build.rs index 766117236d..ecc2873779 100644 --- a/crates/notary/server/build.rs +++ b/crates/notary/server/build.rs @@ -1,21 +1,26 @@ -use std::process::Command; +use std::{env, process::Command}; fn main() { - // Used to extract latest HEAD commit hash and timestamp for the /info endpoint - let output = Command::new("git") - .args(["show", "HEAD", "-s", "--format=%H,%cI"]) - .output() - .expect("Git command to get commit hash and timestamp should work during build process"); + if env::var("GIT_COMMIT_HASH").is_ok() && env::var("GIT_COMMIT_TIMESTAMP").is_ok() { + } else { + // Used to extract latest HEAD commit hash and timestamp for the /info endpoint + let output = Command::new("git") + .args(["show", "HEAD", "-s", "--format=%H,%cI"]) + .output() + .expect( + "Git command to get commit hash and timestamp should work during build process", + ); - let output_string = - String::from_utf8(output.stdout).expect("Git command should produce valid string output"); + let output_string = String::from_utf8(output.stdout) + .expect("Git command should produce valid string output"); - let (commit_hash, commit_timestamp) = output_string - .as_str() - .split_once(',') - .expect("Git commit hash and timestamp string output should be comma separated"); + let (commit_hash, commit_timestamp) = output_string + .as_str() + .split_once(',') + .expect("Git commit hash and timestamp string output should be comma separated"); - // Pass these 2 values as env var to the program - println!("cargo:rustc-env=GIT_COMMIT_HASH={}", commit_hash); - println!("cargo:rustc-env=GIT_COMMIT_TIMESTAMP={}", commit_timestamp); + // Pass these 2 values as env var to the program + println!("cargo:rustc-env=GIT_COMMIT_HASH={}", commit_hash); + println!("cargo:rustc-env=GIT_COMMIT_TIMESTAMP={}", commit_timestamp); + } } diff --git a/crates/notary/server/src/domain.rs b/crates/notary/server/src/domain.rs index 5485be620c..a269540507 100644 --- a/crates/notary/server/src/domain.rs +++ b/crates/notary/server/src/domain.rs @@ -1,7 +1,8 @@ pub mod auth; pub mod cli; pub mod notary; - +#[cfg(feature = "tee_quote")] +use crate::tee::Quote; use serde::{Deserialize, Serialize}; /// Response object of the /info API @@ -16,4 +17,7 @@ pub struct InfoResponse { pub git_commit_hash: String, /// Current git commit timestamp of notary-server pub git_commit_timestamp: String, + /// Hardware attestation + #[cfg(feature = "tee_quote")] + pub quote: Quote, } diff --git a/crates/notary/server/src/lib.rs b/crates/notary/server/src/lib.rs index 9353150dbd..15dd8a77b2 100644 --- a/crates/notary/server/src/lib.rs +++ b/crates/notary/server/src/lib.rs @@ -6,6 +6,8 @@ mod server; mod server_tracing; mod service; mod signing; +#[cfg(feature = "tee_quote")] +mod tee; mod util; pub use config::{ diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs index 7006073fc5..89646f432f 100644 --- a/crates/notary/server/src/server.rs +++ b/crates/notary/server/src/server.rs @@ -46,11 +46,19 @@ use crate::{ util::parse_csv_file, }; +#[cfg(feature = "tee_quote")] +use crate::tee::{ephemeral_keypair, quote}; + /// Start a TCP server (with or without TLS) to accept notarization request for /// both TCP and WebSocket clients #[tracing::instrument(skip(config))] pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> { + // tee uses ephemeral key + #[cfg(feature = "tee_quote")] + let (attestation_key, public_key) = ephemeral_keypair(); + // Load the private key for notarized transcript signing + #[cfg(not(feature = "tee_quote"))] let attestation_key = load_attestation_key(&config.notary_key).await?; let crypto_provider = build_crypto_provider(attestation_key); @@ -107,8 +115,10 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer ); // Parameters needed for the info endpoint + #[cfg(not(feature = "tee_quote"))] let public_key = std::fs::read_to_string(&config.notary_key.public_key_pem_path) .map_err(|err| eyre!("Failed to load notary public signing key for notarization: {err}"))?; + let version = env!("CARGO_PKG_VERSION").to_string(); let git_commit_hash = env!("GIT_COMMIT_HASH").to_string(); let git_commit_timestamp = env!("GIT_COMMIT_TIMESTAMP").to_string(); @@ -142,6 +152,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer public_key, git_commit_hash, git_commit_timestamp, + #[cfg(feature = "tee_quote")] + quote: quote().await, }), ) .into_response() @@ -231,6 +243,7 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider { } /// Load notary signing key for attestations from static file +#[allow(dead_code)] async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result { debug!("Loading notary server's signing key"); diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs new file mode 100644 index 0000000000..0a5895721c --- /dev/null +++ b/crates/notary/server/src/tee.rs @@ -0,0 +1,174 @@ +use base64::{engine::general_purpose::STANDARD, Engine}; +use mc_sgx_dcap_types::QlError; +use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; +use std::fs; + +use crate::signing::AttestationKey; +use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey}; +use pkcs8::{DecodePrivateKey, EncodePrivateKey, LineEnding}; +use rand_chacha::{ + rand_core::{OsRng, SeedableRng}, + ChaCha20Rng, +}; +use std::{ + fs::File, + io::{self, Read}, + path::Path, +}; +use tracing::{debug, error, instrument}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Quote { + raw_quote: Option, + mrsigner: Option, + mrenclave: Option, + error: Option, +} + +impl Default for Quote { + fn default() -> Quote { + Quote { + raw_quote: Some("".to_string()), + mrsigner: None, + mrenclave: None, + error: None, + } + } +} + +impl std::fmt::Debug for QuoteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QuoteError::IoError(err) => write!(f, "IoError: {:?}", err), + QuoteError::IntelQuoteLibrary(err) => { + write!(f, "IntelQuoteLibrary: {}", err) + } + } + } +} + +impl From for QuoteError { + fn from(err: io::Error) -> QuoteError { + QuoteError::IoError(err) + } +} + +enum QuoteError { + IoError(io::Error), + IntelQuoteLibrary(QlError), +} + +impl From for QuoteError { + fn from(src: QlError) -> Self { + Self::IntelQuoteLibrary(src) + } +} + +static PUBLIC_KEY: OnceCell = OnceCell::new(); + +#[instrument(level = "debug", skip_all)] +async fn gramine_quote() -> Result { + //// Check if the the gramine pseudo-hardware exists + if !Path::new("/dev/attestation/quote").exists() { + Quote::default(); + } + + // Reading attestation type + let mut attestation_file = File::open("/dev/attestation/attestation_type")?; + let mut attestation_type = String::new(); + attestation_file.read_to_string(&mut attestation_type)?; + debug!("Detected attestation type: {}", attestation_type); + + // Read `/dev/attestation/my_target_info` + let my_target_info = fs::read("/dev/attestation/my_target_info")?; + + // Write to `/dev/attestation/target_info` + fs::write("/dev/attestation/target_info", my_target_info)?; + + //// Writing the pubkey to bind the instance to the hw (note: this is not + //// mrsigner) + fs::write( + "/dev/attestation/user_report_data", + PUBLIC_KEY + .get() + .expect("pub_key_get") + .to_encoded_point(true) + .as_bytes(), + )?; + + //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote` + let mut quote_file = File::open("/dev/attestation/quote")?; + let mut quote = Vec::new(); + quote_file.read_to_end(&mut quote)?; + + if quote.len() < 432 { + error!("Quote data is too short, expected at least 432 bytes"); + return Err(QuoteError::IntelQuoteLibrary(QlError::InvalidReport)); + } + + //// Extract mrenclave: enclave image, and mrsigner: identity key bound to + //// enclave https://github.com/intel/linux-sgx/blob/main/common/inc/sgx_quote.h + let mrenclave = hex::encode("e[112..144]); + let mrsigner = hex::encode("e[176..208]); + + debug!("mrenclave: {}", mrenclave); + debug!("mrsigner: {}", mrsigner); + + //// Return the Quote struct with the extracted data + Ok(Quote { + raw_quote: Some(hex::encode(quote)), + mrsigner: Some(mrsigner), + mrenclave: Some(mrenclave), + error: None, + }) +} + +pub fn ephemeral_keypair() -> (AttestationKey, String) { + let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!"); + let signing_key = SigningKey::random(&mut rng); + let pem_string = signing_key + .clone() + .to_pkcs8_pem(LineEnding::LF) + .expect("to pem"); + let attkey = AttestationKey::from_pkcs8_pem(&pem_string).expect("from pem"); + let derk = signing_key + .verifying_key() + .to_encoded_point(true) + .to_bytes(); + let b64k = STANDARD.encode(derk.as_ref()); + let pem = format!( + "-----BEGIN PUBLIC KEY-----\n{}\n-----END PUBLIC KEY-----\n", + b64k + ); + + let _ = PUBLIC_KEY + .set(*signing_key.verifying_key()) + .map_err(|_| "Public key has already been set"); + (attkey, pem) +} + +pub async fn quote() -> Quote { + //// tee-detection logic will live here, for now its only gramine-sgx + match gramine_quote().await { + Ok(quote) => quote, + Err(err) => { + error!("Failed to retrieve quote: {:?}", err); + match err { + QuoteError::IoError(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("io".to_owned()), + }, + QuoteError::IntelQuoteLibrary(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("hw".to_owned()), + }, + } + } + } +} diff --git a/crates/notary/server/tee/Dockerfile b/crates/notary/server/tee/Dockerfile new file mode 100644 index 0000000000..8b64731647 --- /dev/null +++ b/crates/notary/server/tee/Dockerfile @@ -0,0 +1,25 @@ +#tlsnotary server for testing <> gramine sgx (gramine1.7, g++13, libiomp off :() +### notaryserverbuilds.azurecr.io/prod/notary-sgx + +FROM notaryserverbuilds.azurecr.io/prod/gramine AS teesdk + +ARG TOOLCHAIN=1.81.0 +ENV PATH=/root/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN set -eux \ + && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$TOOLCHAIN \ + && rustup target add \ + x86_64-unknown-linux-gnu + + +RUN apt update && apt install -y libssl-dev libclang-dev +ARG TLSN_TAG=quote-presentation +ARG TLSN_FT=tee_quote +RUN git clone --depth 1 -b $TLSN_TAG https://github.com/tlsnotary/tlsn /tlsn && \ + cargo build --release --bin notary-server --features $TLSN_FT --color always --manifest-path /tlsn/Cargo.toml +RUN cd tlsn/crates/notary/server/tee && gramine-sgx-gen-private-key && SGX=1 make + +FROM notaryserverbuilds.azurecr.io/prod/gramine AS teetime +WORKDIR /tee +COPY --from=teesdk tlsn/crates/notary/server/tee . +ENTRYPOINT ["gramine-sgx", "notary-server"] diff --git a/crates/notary/server/tee/Makefile b/crates/notary/server/tee/Makefile new file mode 100644 index 0000000000..b3a35341e9 --- /dev/null +++ b/crates/notary/server/tee/Makefile @@ -0,0 +1,63 @@ +# notary-server testing only +ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) + +SELF_EXE = ./notary-server + +.PHONY: all +all: $(SELF_EXE) notary-server.manifest +ifeq ($(SGX),1) +all: notary-server.manifest.sgx notary-server.sig +endif + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +# Note that we're compiling in release mode regardless of the DEBUG setting passed +# to Make, as compiling in debug mode results in an order of magnitude's difference in +# performance that makes testing by running a benchmark with ab painful. The primary goal +# of the DEBUG setting is to control Gramine's loglevel. +-include $(SELF_EXE).d # See also: .cargo/config.toml +$(SELF_EXE): $(ROOT_DIR)../Cargo.toml + cargo build --bin notary-server --release --features tee_quote + +notary-server.manifest: notary-server.manifest.template + cp ../../../../target/release/notary-server . && \ + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dself_exe=$(SELF_EXE) \ + $< $@ + +# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`), +# see the helloworld example for details on this workaround. +notary-server.manifest.sgx notary-server.sig: sgx_sign + @: + +.INTERMEDIATE: sgx_sign +sgx_sign: notary-server.manifest $(SELF_EXE) + gramine-sgx-sign \ + --manifest $< \ + --output $<.sgx + +ifeq ($(SGX),) +GRAMINE = gramine-direct +else +GRAMINE = gramine-sgx +endif + +.PHONY: start-gramine-server +start-gramine-server: all + $(GRAMINE) notary-server + +.PHONY: clean +clean: + $(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT + +.PHONY: distclean +distclean: clean + $(RM) -rf $(SELF_EXE) Cargo.lock + diff --git a/crates/notary/server/tee/README.md b/crates/notary/server/tee/README.md new file mode 100644 index 0000000000..d62b0f9dd7 --- /dev/null +++ b/crates/notary/server/tee/README.md @@ -0,0 +1,21 @@ +#### gramine with intel SGX +```bash +SGX=1 make +``` +```bash +SGX=1 make start-gramine-server +``` +#### gramine emulating SGX +``` +make +``` +``` +make start-gramine-server +``` +#### generate measurement without SGX hardware +``` +make +``` +``` +gramine-sgx-sigstruct-view --verbose --output-format=toml notary-server.sig +``` diff --git a/crates/notary/server/tee/config/config.yaml b/crates/notary/server/tee/config/config.yaml new file mode 100644 index 0000000000..49a505c470 --- /dev/null +++ b/crates/notary/server/tee/config/config.yaml @@ -0,0 +1,45 @@ +server: + name: "notary.codes" + host: "0.0.0.0" + port: 7047 + html-info: | + + + + + + + + + + + + + + +

notary server :: at your service

+

tlsnotary {version}

+ {public_key} +

remote attestation

+ available here + + +notarization: + max-sent-data: 4096 + max-recv-data: 16384 + +tls: + enabled: false + private-key-pem-path: "." + certificate-pem-path: "." + +notary-key: + private-key-pem-path: "." + public-key-pem-path: "." + +logging: + level: INFO + +authorization: + enabled: false + whitelist-csv-path: "." diff --git a/crates/notary/server/tee/notary-server.manifest.template b/crates/notary/server/tee/notary-server.manifest.template new file mode 100644 index 0000000000..db57ea47bf --- /dev/null +++ b/crates/notary/server/tee/notary-server.manifest.template @@ -0,0 +1,44 @@ +libos.entrypoint = "{{ self_exe }}" +loader.log_level = "{{ log_level }}" + +loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}" + +# See https://gramine.readthedocs.io/en/stable/performance.html#glibc-malloc-tuning +loader.env.MALLOC_ARENA_MAX = "1" + +# encrypted type not used +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" }, + +] + +# allowed enables rw -- will rm this once notary-server drops config file +# !!!! not hashed !!!! +# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#allowed-files-1 +sgx.allowed_files = [ + "file:./config/config.yaml", +] + +# hashed @ buildtime. at runtime => these files are +ro +# and can be accessed if hash matches manifest +# !!!! hashed !!!! +# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#trusted-files +sgx.trusted_files = [ + "file:{{ self_exe }}", + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", +] + +sgx.edmm_enable = false +sgx.remote_attestation = "dcap" +sgx.max_threads = 64 +sgx.enclave_size = "2G" +sys.disallow_subprocesses = true + + +#### tlsn rev +sgx.isvprodid = 7 +#### F +sgx.isvsvn = 46 From 387d52a7114f0ce1ba31e4241fe83da24ba32e42 Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Wed, 9 Oct 2024 14:57:27 +0200 Subject: [PATCH 02/12] fix return --- crates/notary/server/src/tee.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs index 0a5895721c..9cca82b44b 100644 --- a/crates/notary/server/src/tee.rs +++ b/crates/notary/server/src/tee.rs @@ -72,7 +72,7 @@ static PUBLIC_KEY: OnceCell = OnceCell::new(); async fn gramine_quote() -> Result { //// Check if the the gramine pseudo-hardware exists if !Path::new("/dev/attestation/quote").exists() { - Quote::default(); + return Ok(Quote::default()); } // Reading attestation type From 8b288ec04b17e6b62ec3ebf909c7f4e3f80847c7 Mon Sep 17 00:00:00 2001 From: mac Date: Wed, 9 Oct 2024 14:59:29 +0200 Subject: [PATCH 03/12] Update crates/notary/server/build.rs Co-authored-by: yuroitaki <25913766+yuroitaki@users.noreply.github.com> --- crates/notary/server/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/notary/server/build.rs b/crates/notary/server/build.rs index ecc2873779..f8181bff91 100644 --- a/crates/notary/server/build.rs +++ b/crates/notary/server/build.rs @@ -1,7 +1,7 @@ use std::{env, process::Command}; fn main() { - if env::var("GIT_COMMIT_HASH").is_ok() && env::var("GIT_COMMIT_TIMESTAMP").is_ok() { + if env::var("GIT_COMMIT_HASH").is_err() || env::var("GIT_COMMIT_TIMESTAMP").is_err() { } else { // Used to extract latest HEAD commit hash and timestamp for the /info endpoint let output = Command::new("git") From 5476770ed4c7cb8a247077123f9611beda4ec76d Mon Sep 17 00:00:00 2001 From: mac Date: Wed, 9 Oct 2024 14:59:39 +0200 Subject: [PATCH 04/12] Update crates/notary/server/build.rs Co-authored-by: yuroitaki <25913766+yuroitaki@users.noreply.github.com> --- crates/notary/server/build.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/notary/server/build.rs b/crates/notary/server/build.rs index f8181bff91..15833ad972 100644 --- a/crates/notary/server/build.rs +++ b/crates/notary/server/build.rs @@ -2,7 +2,6 @@ use std::{env, process::Command}; fn main() { if env::var("GIT_COMMIT_HASH").is_err() || env::var("GIT_COMMIT_TIMESTAMP").is_err() { - } else { // Used to extract latest HEAD commit hash and timestamp for the /info endpoint let output = Command::new("git") .args(["show", "HEAD", "-s", "--format=%H,%cI"]) From a2a976dcc16ea057fd6bcbbb7985cc2e586bcb8e Mon Sep 17 00:00:00 2001 From: mac Date: Wed, 9 Oct 2024 15:00:36 +0200 Subject: [PATCH 05/12] Update Dockerfile --- crates/notary/server/tee/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/notary/server/tee/Dockerfile b/crates/notary/server/tee/Dockerfile index 8b64731647..bc76076387 100644 --- a/crates/notary/server/tee/Dockerfile +++ b/crates/notary/server/tee/Dockerfile @@ -13,7 +13,7 @@ RUN set -eux \ RUN apt update && apt install -y libssl-dev libclang-dev -ARG TLSN_TAG=quote-presentation +ARG TLSN_TAG=dev ARG TLSN_FT=tee_quote RUN git clone --depth 1 -b $TLSN_TAG https://github.com/tlsnotary/tlsn /tlsn && \ cargo build --release --bin notary-server --features $TLSN_FT --color always --manifest-path /tlsn/Cargo.toml From 0c9767a95b4135d90a715c408255a882f74d1846 Mon Sep 17 00:00:00 2001 From: maceipo Date: Fri, 11 Oct 2024 15:12:03 +0000 Subject: [PATCH 06/12] unsuss --- crates/notary/server/src/tee.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs index 9cca82b44b..9ef6f81468 100644 --- a/crates/notary/server/src/tee.rs +++ b/crates/notary/server/src/tee.rs @@ -1,5 +1,5 @@ use base64::{engine::general_purpose::STANDARD, Engine}; -use mc_sgx_dcap_types::QlError; +use mc_sgx_dcap_types::{QlError, Quote3}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; use std::fs; @@ -101,17 +101,11 @@ async fn gramine_quote() -> Result { //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote` let mut quote_file = File::open("/dev/attestation/quote")?; let mut quote = Vec::new(); - quote_file.read_to_end(&mut quote)?; - - if quote.len() < 432 { - error!("Quote data is too short, expected at least 432 bytes"); - return Err(QuoteError::IntelQuoteLibrary(QlError::InvalidReport)); - } - - //// Extract mrenclave: enclave image, and mrsigner: identity key bound to - //// enclave https://github.com/intel/linux-sgx/blob/main/common/inc/sgx_quote.h - let mrenclave = hex::encode("e[112..144]); - let mrsigner = hex::encode("e[176..208]); + let _ = quote_file.read_to_end(&mut quote); + //// todo: wire up Qlerror and drop .expect() + let quote3 = Quote3::try_from(quote.as_ref()).expect("quote3 error"); + let mrenclave = quote3.app_report_body().mr_enclave().to_string(); + let mrsigner = quote3.app_report_body().mr_signer().to_string(); debug!("mrenclave: {}", mrenclave); debug!("mrsigner: {}", mrsigner); From 8930529ae0da9b190a794d93632ac7ca79c81b09 Mon Sep 17 00:00:00 2001 From: maceip Date: Fri, 18 Oct 2024 16:27:38 +0000 Subject: [PATCH 07/12] feat: tsukino => proper pem export of pubkey --- crates/notary/server/Cargo.toml | 15 +++++++++++++-- crates/notary/server/src/tee.rs | 34 +++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/crates/notary/server/Cargo.toml b/crates/notary/server/Cargo.toml index 5dc2a0cb68..7cd2015884 100644 --- a/crates/notary/server/Cargo.toml +++ b/crates/notary/server/Cargo.toml @@ -4,7 +4,15 @@ version = "0.1.0-alpha.7" edition = "2021" [features] -tee_quote = ["dep:mc-sgx-dcap-types", "dep:hex", "dep:rand_chacha", "dep:once_cell"] +tee_quote = [ + "dep:mc-sgx-dcap-types", + "dep:hex", + "dep:rand_chacha", + "dep:once_cell", + "dep:simple_asn1", + "dep:pem", + "dep:lazy_static", +] [dependencies] tlsn-core = { workspace = true } @@ -57,4 +65,7 @@ zeroize = { workspace = true } mc-sgx-dcap-types = { version = "0.11.0", optional = true } hex = { workspace = true, optional = true } rand_chacha = { workspace = true, optional = true } -once_cell = { workspace = true, optional =true } \ No newline at end of file +once_cell = { workspace = true, optional =true } +simple_asn1 = {version = "0.6.2", optional = true } +pem = { version = "1.1.0", optional = true } +lazy_static = { version = "1.4", optional = true } diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs index 9ef6f81468..c2ed572401 100644 --- a/crates/notary/server/src/tee.rs +++ b/crates/notary/server/src/tee.rs @@ -1,4 +1,3 @@ -use base64::{engine::general_purpose::STANDARD, Engine}; use mc_sgx_dcap_types::{QlError, Quote3}; use once_cell::sync::OnceCell; use serde::{Deserialize, Serialize}; @@ -18,6 +17,11 @@ use std::{ }; use tracing::{debug, error, instrument}; +lazy_static::lazy_static! { + static ref SECP256K1_OID: simple_asn1::OID = simple_asn1::oid!(1, 3, 132, 0, 10); + static ref ECDSA_OID: simple_asn1::OID = simple_asn1::oid!(1, 2, 840, 10045, 2, 1); +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Quote { @@ -68,6 +72,25 @@ impl From for QuoteError { static PUBLIC_KEY: OnceCell = OnceCell::new(); +fn pem_der_encode_with_asn1(public_point: &[u8]) -> String { + use simple_asn1::*; + + let ecdsa_oid = ASN1Block::ObjectIdentifier(0, ECDSA_OID.clone()); + let secp256k1_oid = ASN1Block::ObjectIdentifier(0, SECP256K1_OID.clone()); + let alg_id = ASN1Block::Sequence(0, vec![ecdsa_oid, secp256k1_oid]); + let key_bytes = ASN1Block::BitString(0, public_point.len() * 8, public_point.to_vec()); + + let blocks = vec![alg_id, key_bytes]; + + let der_out = simple_asn1::to_der(&ASN1Block::Sequence(0, blocks)) + .expect("Failed to encode ECDSA private key as DER"); + + pem::encode(&pem::Pem { + tag: "PUBLIC KEY".to_string(), + contents: der_out, + }) +} + #[instrument(level = "debug", skip_all)] async fn gramine_quote() -> Result { //// Check if the the gramine pseudo-hardware exists @@ -131,16 +154,11 @@ pub fn ephemeral_keypair() -> (AttestationKey, String) { .verifying_key() .to_encoded_point(true) .to_bytes(); - let b64k = STANDARD.encode(derk.as_ref()); - let pem = format!( - "-----BEGIN PUBLIC KEY-----\n{}\n-----END PUBLIC KEY-----\n", - b64k - ); - + let pem_spki_pub = pem_der_encode_with_asn1(&derk); let _ = PUBLIC_KEY .set(*signing_key.verifying_key()) .map_err(|_| "Public key has already been set"); - (attkey, pem) + (attkey, pem_spki_pub) } pub async fn quote() -> Quote { From c03449dfb108bf523061811b5ad34e2da1f868fc Mon Sep 17 00:00:00 2001 From: maceipo Date: Thu, 24 Oct 2024 23:46:56 +0000 Subject: [PATCH 08/12] chore: kebab->snake --- crates/notary/server/tee/config/config.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/notary/server/tee/config/config.yaml b/crates/notary/server/tee/config/config.yaml index 49a505c470..65a29cde7b 100644 --- a/crates/notary/server/tee/config/config.yaml +++ b/crates/notary/server/tee/config/config.yaml @@ -2,7 +2,7 @@ server: name: "notary.codes" host: "0.0.0.0" port: 7047 - html-info: | + html_info: | @@ -25,21 +25,21 @@ server: notarization: - max-sent-data: 4096 - max-recv-data: 16384 + max_sent_data: 4096 + max_recv_data: 16384 tls: enabled: false - private-key-pem-path: "." - certificate-pem-path: "." + private_key_pem_path: "." + certificate_pem_path: "." -notary-key: - private-key-pem-path: "." - public-key-pem-path: "." +notary_key: + private_key_pem_path: "." + public_key_pem_path: "." logging: level: INFO authorization: enabled: false - whitelist-csv-path: "." + whitelist_csv_path: "." From 2a4f0a386cace62f6e477968b91458ed8223a183 Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Mon, 28 Oct 2024 10:58:30 -0700 Subject: [PATCH 09/12] consolidate config --- crates/notary/server/src/server.rs | 13 ++++--------- crates/notary/server/src/tee.rs | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs index 89646f432f..636530f1e3 100644 --- a/crates/notary/server/src/server.rs +++ b/crates/notary/server/src/server.rs @@ -47,18 +47,12 @@ use crate::{ }; #[cfg(feature = "tee_quote")] -use crate::tee::{ephemeral_keypair, quote}; +use crate::tee::{generate_ephemeral_keypair, quote}; /// Start a TCP server (with or without TLS) to accept notarization request for /// both TCP and WebSocket clients #[tracing::instrument(skip(config))] pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> { - // tee uses ephemeral key - #[cfg(feature = "tee_quote")] - let (attestation_key, public_key) = ephemeral_keypair(); - - // Load the private key for notarized transcript signing - #[cfg(not(feature = "tee_quote"))] let attestation_key = load_attestation_key(&config.notary_key).await?; let crypto_provider = build_crypto_provider(attestation_key); @@ -115,7 +109,6 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer ); // Parameters needed for the info endpoint - #[cfg(not(feature = "tee_quote"))] let public_key = std::fs::read_to_string(&config.notary_key.public_key_pem_path) .map_err(|err| eyre!("Failed to load notary public signing key for notarization: {err}"))?; @@ -243,8 +236,10 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider { } /// Load notary signing key for attestations from static file -#[allow(dead_code)] async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result { + #[cfg(feature = "tee_quote")] + generate_ephemeral_keypair(&config.private_key_pem_path, &config.public_key_pem_path); + debug!("Loading notary server's signing key"); let mut file = File::open(&config.private_key_pem_path).await?; diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs index c2ed572401..4a4c419afb 100644 --- a/crates/notary/server/src/tee.rs +++ b/crates/notary/server/src/tee.rs @@ -1,16 +1,14 @@ +use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey}; use mc_sgx_dcap_types::{QlError, Quote3}; use once_cell::sync::OnceCell; -use serde::{Deserialize, Serialize}; -use std::fs; - -use crate::signing::AttestationKey; -use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey}; -use pkcs8::{DecodePrivateKey, EncodePrivateKey, LineEnding}; +use pkcs8::{EncodePrivateKey, LineEnding}; use rand_chacha::{ rand_core::{OsRng, SeedableRng}, ChaCha20Rng, }; +use serde::{Deserialize, Serialize}; use std::{ + fs, fs::File, io::{self, Read}, path::Path, @@ -142,23 +140,25 @@ async fn gramine_quote() -> Result { }) } -pub fn ephemeral_keypair() -> (AttestationKey, String) { +pub fn generate_ephemeral_keypair(notary_private: &str, notary_public: &str) { let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!"); let signing_key = SigningKey::random(&mut rng); let pem_string = signing_key .clone() .to_pkcs8_pem(LineEnding::LF) .expect("to pem"); - let attkey = AttestationKey::from_pkcs8_pem(&pem_string).expect("from pem"); - let derk = signing_key + + std::fs::write(notary_private, pem_string).expect("fs::write"); + + let der = signing_key .verifying_key() .to_encoded_point(true) .to_bytes(); - let pem_spki_pub = pem_der_encode_with_asn1(&derk); + let pem_spki_pub = pem_der_encode_with_asn1(&der); + std::fs::write(notary_public, pem_spki_pub).expect("fs::write"); let _ = PUBLIC_KEY .set(*signing_key.verifying_key()) .map_err(|_| "Public key has already been set"); - (attkey, pem_spki_pub) } pub async fn quote() -> Quote { From c5c72f00d023a5b258723c2707bf3dcc097608e4 Mon Sep 17 00:00:00 2001 From: Ryan MacArthur Date: Mon, 28 Oct 2024 11:00:11 -0700 Subject: [PATCH 10/12] config interop --- crates/notary/server/tee/config/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/notary/server/tee/config/config.yaml b/crates/notary/server/tee/config/config.yaml index 65a29cde7b..2565cb7d49 100644 --- a/crates/notary/server/tee/config/config.yaml +++ b/crates/notary/server/tee/config/config.yaml @@ -34,8 +34,8 @@ tls: certificate_pem_path: "." notary_key: - private_key_pem_path: "." - public_key_pem_path: "." + private_key_pem_path: "notary.key" + public_key_pem_path: "notary.pub" logging: level: INFO From 66f5c863964046bb7e4964e58f9f93ff2791f2fc Mon Sep 17 00:00:00 2001 From: maceipo Date: Mon, 28 Oct 2024 20:21:23 +0000 Subject: [PATCH 11/12] gramine config updates to sync with server config --- crates/notary/server/tee/config/config.yaml | 4 ++-- crates/notary/server/tee/notary-server.manifest.template | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/notary/server/tee/config/config.yaml b/crates/notary/server/tee/config/config.yaml index 2565cb7d49..d85859fed4 100644 --- a/crates/notary/server/tee/config/config.yaml +++ b/crates/notary/server/tee/config/config.yaml @@ -34,8 +34,8 @@ tls: certificate_pem_path: "." notary_key: - private_key_pem_path: "notary.key" - public_key_pem_path: "notary.pub" + private_key_pem_path: "/ephemeral/notary.key" + public_key_pem_path: "/ephemeral/notary.pub" logging: level: INFO diff --git a/crates/notary/server/tee/notary-server.manifest.template b/crates/notary/server/tee/notary-server.manifest.template index db57ea47bf..40ab8a3545 100644 --- a/crates/notary/server/tee/notary-server.manifest.template +++ b/crates/notary/server/tee/notary-server.manifest.template @@ -10,6 +10,7 @@ loader.env.MALLOC_ARENA_MAX = "1" fs.mounts = [ { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { type = "tmpfs", path = "/ephemeral" }, { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" }, ] From 3d928437a748a3d30ab915fd16d85d935b3ddeb8 Mon Sep 17 00:00:00 2001 From: maceipo Date: Mon, 28 Oct 2024 20:50:16 +0000 Subject: [PATCH 12/12] bad merge -- rm git_commit_timestamp --- crates/notary/server/src/domain.rs | 2 -- crates/notary/server/src/server.rs | 2 -- 2 files changed, 4 deletions(-) diff --git a/crates/notary/server/src/domain.rs b/crates/notary/server/src/domain.rs index a269540507..58a5dc8b1c 100644 --- a/crates/notary/server/src/domain.rs +++ b/crates/notary/server/src/domain.rs @@ -15,8 +15,6 @@ pub struct InfoResponse { pub public_key: String, /// Current git commit hash of notary-server pub git_commit_hash: String, - /// Current git commit timestamp of notary-server - pub git_commit_timestamp: String, /// Hardware attestation #[cfg(feature = "tee_quote")] pub quote: Quote, diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs index e279036168..5a93463a1b 100644 --- a/crates/notary/server/src/server.rs +++ b/crates/notary/server/src/server.rs @@ -111,7 +111,6 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer // Parameters needed for the info endpoint let public_key = std::fs::read_to_string(&config.notary_key.public_key_pem_path) .map_err(|err| eyre!("Failed to load notary public signing key for notarization: {err}"))?; - let version = env!("CARGO_PKG_VERSION").to_string(); let git_commit_hash = env!("GIT_COMMIT_HASH").to_string(); @@ -142,7 +141,6 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer version, public_key, git_commit_hash, - git_commit_timestamp, #[cfg(feature = "tee_quote")] quote: quote().await, }),