From d74b7b04bb2200b3a118fcda7585b1c2c280dfcf Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Mon, 14 Sep 2020 16:35:55 -0700 Subject: [PATCH 01/19] Cancel queue --- intel-sgx/enclave-runner/src/usercalls/abi.rs | 4 +- .../enclave-runner/src/usercalls/interface.rs | 7 +- intel-sgx/enclave-runner/src/usercalls/mod.rs | 146 +++++++++++++++--- ipc-queue/src/fifo.rs | 32 ++-- ipc-queue/src/interface_async.rs | 72 ++++++++- ipc-queue/src/interface_sync.rs | 4 +- ipc-queue/src/lib.rs | 21 ++- ipc-queue/src/position.rs | 48 ++++++ 8 files changed, 296 insertions(+), 38 deletions(-) create mode 100644 ipc-queue/src/position.rs diff --git a/intel-sgx/enclave-runner/src/usercalls/abi.rs b/intel-sgx/enclave-runner/src/usercalls/abi.rs index 8345aa85..0fcc0c88 100644 --- a/intel-sgx/enclave-runner/src/usercalls/abi.rs +++ b/intel-sgx/enclave-runner/src/usercalls/abi.rs @@ -19,7 +19,7 @@ use futures::future::Future; type Register = u64; -trait RegisterArgument { +pub(super) trait RegisterArgument { fn from_register(_: Register) -> Self; fn into_register(self) -> Register; } @@ -29,7 +29,7 @@ type EnclaveAbort = super::EnclaveAbort; pub(crate) type UsercallResult = ::std::result::Result; pub(crate) type DispatchResult = UsercallResult<(Register, Register)>; -trait ReturnValue { +pub(super) trait ReturnValue { fn into_registers(self) -> DispatchResult; } diff --git a/intel-sgx/enclave-runner/src/usercalls/interface.rs b/intel-sgx/enclave-runner/src/usercalls/interface.rs index c5ec9ca1..c8731bcc 100644 --- a/intel-sgx/enclave-runner/src/usercalls/interface.rs +++ b/intel-sgx/enclave-runner/src/usercalls/interface.rs @@ -252,12 +252,13 @@ impl<'future, 'ioinput: 'future, 'tcs: 'ioinput> Usercalls<'future> for Handler< self, usercall_queue: *mut FifoDescriptor, return_queue: *mut FifoDescriptor, + cancel_queue: *mut FifoDescriptor, ) -> std::pin::Pin)> + 'future>> { async move { unsafe { let ret = match (usercall_queue.as_mut(), return_queue.as_mut()) { (Some(usercall_queue), Some(return_queue)) => { - self.0.async_queues(usercall_queue, return_queue).await.map(Ok) + self.0.async_queues(usercall_queue, return_queue, cancel_queue.as_mut()).await.map(Ok) }, _ => { Ok(Err(IoErrorKind::InvalidInput.into())) @@ -321,13 +322,13 @@ fn result_from_io_error(err: IoError) -> Result { ret as _ } -trait ToSgxResult { +pub(super) trait ToSgxResult { type Return; fn to_sgx_result(self) -> Self::Return; } -trait SgxReturn { +pub(super) trait SgxReturn { fn on_error() -> Self; } diff --git a/intel-sgx/enclave-runner/src/usercalls/mod.rs b/intel-sgx/enclave-runner/src/usercalls/mod.rs index 51dd389d..9a9d0967 100644 --- a/intel-sgx/enclave-runner/src/usercalls/mod.rs +++ b/intel-sgx/enclave-runner/src/usercalls/mod.rs @@ -6,7 +6,7 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::cell::RefCell; -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::io::{self, ErrorKind as IoErrorKind, Read, Result as IoResult}; use std::pin::Pin; use std::result::Result as StdResult; @@ -34,12 +34,12 @@ use tokio::sync::mpsc as async_mpsc; use tokio::sync::Semaphore; use fortanix_sgx_abi::*; -use ipc_queue::{self, DescriptorGuard, Identified, QueueEvent}; +use ipc_queue::{self, DescriptorGuard, Identified, QueueEvent, WritePosition}; use sgxs::loader::Tcs as SgxsTcs; use crate::loader::{EnclavePanic, ErasedTcs}; use crate::tcs::{self, CoResult, ThreadResult}; -use self::abi::dispatch; +use self::abi::{dispatch, UsercallList}; use self::interface::{Handler, OutputBuffer}; pub(crate) mod abi; @@ -49,10 +49,11 @@ lazy_static! { static ref DEBUGGER_TOGGLE_SYNC: Mutex<()> = Mutex::new(()); } -const EV_ABORT: u64 = 0b0000_0000_0000_1000; +const EV_ABORT: u64 = 0b0000_0000_0001_0000; const USERCALL_QUEUE_SIZE: usize = 16; const RETURN_QUEUE_SIZE: usize = 1024; +const CANCEL_QUEUE_SIZE: usize = USERCALL_QUEUE_SIZE * 2; enum UsercallSendData { Sync(ThreadResult, RunningTcs, RefCell<[u8; 1024]>), @@ -62,7 +63,7 @@ enum UsercallSendData { // This is the same as UsercallSendData except that it can't be Sync(CoResult::Return(...), ...) enum UsercallHandleData { Sync(tcs::Usercall, RunningTcs, RefCell<[u8; 1024]>), - Async(Identified), + Async(Identified, Option>), } type EnclaveResult = StdResult<(u64, u64), EnclaveAbort>>; @@ -515,7 +516,7 @@ struct PendingEvents { impl PendingEvents { // Will error if it doesn't fit in a `u64` - const EV_MAX_U64: u64 = (EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK) + 1; + const EV_MAX_U64: u64 = (EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL) + 1; const EV_MAX: usize = Self::EV_MAX_U64 as _; // Will error if it doesn't fit in a `usize` const _ERROR_IF_USIZE_TOO_SMALL: u64 = u64::MAX + (Self::EV_MAX_U64 - (Self::EV_MAX as u64)); @@ -528,6 +529,8 @@ impl PendingEvents { counts: [ Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), + Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), + Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), Semaphore::new(0), ], abort: Semaphore::new(0), } @@ -639,6 +642,7 @@ impl EnclaveKind { struct FifoGuards { usercall_queue: DescriptorGuard, return_queue: DescriptorGuard, + cancel_queue: DescriptorGuard, async_queues_called: bool, } @@ -684,6 +688,27 @@ impl Work { } } +enum UsercallEvent { + Started(u64, tokio::sync::oneshot::Sender<()>), + Finished(u64), + Cancelled(u64, WritePosition), +} + +trait IgnoreCancel { + fn ignore_cancel(&self) -> bool; +} + +impl IgnoreCancel for Identified { + fn ignore_cancel(&self) -> bool { + self.data.0 != UsercallList::read as u64 && + self.data.0 != UsercallList::read_alloc as u64 && + self.data.0 != UsercallList::write as u64 && + self.data.0 != UsercallList::accept_stream as u64 && + self.data.0 != UsercallList::connect_stream as u64 && + self.data.0 != UsercallList::wait as u64 + } +} + impl EnclaveState { fn event_queue_add_tcs( event_queues: &mut FnvHashMap, @@ -754,15 +779,41 @@ impl EnclaveState { tx_return_channel: tokio::sync::mpsc::UnboundedSender<(EnclaveResult, ReturnSource)>, mut handle_data: UsercallHandleData, ) { + let notifier_rx = match handle_data { + UsercallHandleData::Async(ref usercall, Some(ref usercall_event_tx)) => { + let (notifier_tx, notifier_rx) = tokio::sync::oneshot::channel(); + usercall_event_tx.send(UsercallEvent::Started(usercall.id, notifier_tx)).ok() + .expect("failed to send usercall event"); + Some(notifier_rx) + }, + _ => None, + }; let (parameters, mode, tcs) = match handle_data { UsercallHandleData::Sync(ref usercall, ref mut tcs, _) => (usercall.parameters(), tcs.mode.into(), Some(tcs)), - UsercallHandleData::Async(ref usercall) => (usercall.data.into(), ReturnSource::AsyncUsercall, None), + UsercallHandleData::Async(ref usercall, _) => (usercall.data.into(), ReturnSource::AsyncUsercall, None), }; let mut input = IOHandlerInput { enclave: enclave.clone(), tcs, work_sender: &work_sender }; let handler = Handler(&mut input); - let (_handler, result) = { + let result = { + use self::interface::ToSgxResult; + use self::abi::ReturnValue; + let (p1, p2, p3, p4, p5) = parameters; - dispatch(handler, p1, p2, p3, p4, p5).await + match notifier_rx { + None => dispatch(handler, p1, p2, p3, p4, p5).await.1, + Some(notifier_rx) => { + let a = dispatch(handler, p1, p2, p3, p4, p5).boxed_local(); + let b = notifier_rx; + match futures::future::select(a, b).await { + Either::Left((ret, _)) => ret.1, + Either::Right((Ok(()), _)) => { + let result: IoResult = Err(IoErrorKind::Interrupted.into()); + ReturnValue::into_registers(Ok(result.to_sgx_result())) + }, + Either::Right((Err(_), _)) => panic!("notifier channel closed unexpectedly"), + } + }, + } }; let ret = match result { Ok(ret) => { @@ -773,7 +824,11 @@ impl EnclaveState { entry: CoEntry::Resume(usercall, ret), }).expect("Work sender couldn't send data to receiver"); } - UsercallHandleData::Async(usercall) => { + UsercallHandleData::Async(usercall, usercall_event_tx) => { + if let Some(usercall_event_tx) = usercall_event_tx { + usercall_event_tx.send(UsercallEvent::Finished(usercall.id)).ok() + .expect("failed to send usercall event"); + } let return_queue_tx = enclave.return_queue_tx.lock().await.clone().expect("return_queue_tx not initialized"); let ret = Identified { id: usercall.id, @@ -794,7 +849,7 @@ impl EnclaveState { } EnclavePanic::from(debug_buf) } - UsercallHandleData::Async(_) => { + UsercallHandleData::Async(_, _) => { // TODO: https://github.com/fortanix/rust-sgx/issues/235#issuecomment-641811437 EnclavePanic::DebugStr("async exit with a panic".to_owned()) } @@ -872,26 +927,61 @@ impl EnclaveState { }; let enclave_clone = enclave.clone(); let io_future = async move { - let (usercall_queue_synchronizer, return_queue_synchronizer, sync_usercall_tx) = QueueSynchronizer::new(enclave_clone.clone()); + let (uqs, rqs, cqs, sync_usercall_tx) = QueueSynchronizer::new(enclave_clone.clone()); - let (usercall_queue_tx, usercall_queue_rx) = ipc_queue::bounded_async(USERCALL_QUEUE_SIZE, usercall_queue_synchronizer); - let (return_queue_tx, return_queue_rx) = ipc_queue::bounded_async(RETURN_QUEUE_SIZE, return_queue_synchronizer); + let (usercall_queue_tx, usercall_queue_rx) = ipc_queue::bounded_async(USERCALL_QUEUE_SIZE, uqs); + let (return_queue_tx, return_queue_rx) = ipc_queue::bounded_async(RETURN_QUEUE_SIZE, rqs); + let (cancel_queue_tx, cancel_queue_rx) = ipc_queue::bounded_async(CANCEL_QUEUE_SIZE, cqs); let fifo_guards = FifoGuards { usercall_queue: usercall_queue_tx.into_descriptor_guard(), return_queue: return_queue_rx.into_descriptor_guard(), + cancel_queue: cancel_queue_tx.into_descriptor_guard(), async_queues_called: false, }; *enclave_clone.fifo_guards.lock().await = Some(fifo_guards); *enclave_clone.return_queue_tx.lock().await = Some(return_queue_tx); + let usercall_queue_monitor = usercall_queue_rx.position_monitor(); + tokio::task::spawn_local(async move { while let Ok(usercall) = usercall_queue_rx.recv().await { let _ = io_queue_send.send(UsercallSendData::Async(usercall)); } }); + let (usercall_event_tx, mut usercall_event_rx) = async_mpsc::unbounded_channel(); + let usercall_event_tx_clone = usercall_event_tx.clone(); + let usercall_queue_monitor_clone = usercall_queue_monitor.clone(); + tokio::task::spawn_local(async move { + while let Ok(c) = cancel_queue_rx.recv().await { + let write_position = usercall_queue_monitor_clone.write_position(); + let _ = usercall_event_tx_clone.send(UsercallEvent::Cancelled(c.id, write_position)); + } + }); + + tokio::task::spawn_local(async move { + let mut notifiers = HashMap::new(); + let mut cancels: HashMap = HashMap::new(); + loop { + match usercall_event_rx.recv().await.expect("usercall_event channel closed unexpectedly") { + UsercallEvent::Started(id, notifier) => match cancels.remove(&id) { + Some(_) => { let _ = notifier.send(()); }, + _ => { notifiers.insert(id, notifier); }, + }, + UsercallEvent::Finished(id) => { notifiers.remove(&id); }, + UsercallEvent::Cancelled(id, wp) => match notifiers.remove(&id) { + Some(notifier) => { let _ = notifier.send(()); }, + None => { cancels.insert(id, wp); }, + }, + } + // cleanup old cancels + let read_position = usercall_queue_monitor.read_position(); + cancels.retain(|_id, wp| !read_position.is_past(wp)); + } + }); + let mut recv_queue = io_queue_receive.into_future(); while let (Some(work), stream) = recv_queue.await { recv_queue = stream.into_future(); @@ -899,7 +989,8 @@ impl EnclaveState { let tx_return_channel = tx_return_channel.clone(); match work { UsercallSendData::Async(usercall) => { - let uchd = UsercallHandleData::Async(usercall); + let usercall_event_tx = if usercall.ignore_cancel() { None } else { Some(usercall_event_tx.clone()) }; + let uchd = UsercallHandleData::Async(usercall, usercall_event_tx); let fut = Self::handle_usercall(enclave_clone, work_sender.clone(), tx_return_channel, uchd); tokio::task::spawn_local(fut); } @@ -1483,7 +1574,7 @@ impl<'tcs> IOHandlerInput<'tcs> { } fn check_event_set(set: u64) -> IoResult<()> { - const EV_ALL: u64 = EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK; + const EV_ALL: u64 = EV_USERCALLQ_NOT_FULL | EV_RETURNQ_NOT_EMPTY | EV_UNPARK | EV_CANCELQ_NOT_FULL; if (set & !EV_ALL) != 0 { return Err(IoErrorKind::InvalidInput.into()); } @@ -1593,12 +1684,16 @@ impl<'tcs> IOHandlerInput<'tcs> { &mut self, usercall_queue: &mut FifoDescriptor, return_queue: &mut FifoDescriptor, + cancel_queue: Option<&mut FifoDescriptor>, ) -> StdResult<(), EnclaveAbort> { let mut fifo_guards = self.enclave.fifo_guards.lock().await; match &mut *fifo_guards { Some(ref mut fifo_guards) if !fifo_guards.async_queues_called => { *usercall_queue = fifo_guards.usercall_queue.fifo_descriptor(); *return_queue = fifo_guards.return_queue.fifo_descriptor(); + if let Some(cancel_queue) = cancel_queue { + *cancel_queue = fifo_guards.cancel_queue.fifo_descriptor(); + } fifo_guards.async_queues_called = true; Ok(()) } @@ -1617,6 +1712,7 @@ impl<'tcs> IOHandlerInput<'tcs> { enum Queue { Usercall, Return, + Cancel, } struct QueueSynchronizer { @@ -1629,7 +1725,7 @@ struct QueueSynchronizer { } impl QueueSynchronizer { - fn new(enclave: Arc) -> (Self, Self, broadcast::Sender<()>) { + fn new(enclave: Arc) -> (Self, Self, Self, broadcast::Sender<()>) { // This broadcast channel is used to notify enclave-runner of any // synchronous usercalls made by the enclave for the purpose of // synchronizing access to usercall and return queues. @@ -1637,6 +1733,7 @@ impl QueueSynchronizer { // return RecvError::Lagged. let (tx, rx1) = broadcast::channel(1); let rx2 = tx.subscribe(); + let rx3 = tx.subscribe(); let usercall_queue_synchronizer = QueueSynchronizer { queue: Queue::Usercall, enclave: enclave.clone(), @@ -1645,11 +1742,17 @@ impl QueueSynchronizer { }; let return_queue_synchronizer = QueueSynchronizer { queue: Queue::Return, - enclave, + enclave: enclave.clone(), subscription: Mutex::new(rx2), subscription_maker: tx.clone(), }; - (usercall_queue_synchronizer, return_queue_synchronizer, tx) + let cancel_queue_synchronizer = QueueSynchronizer { + queue: Queue::Cancel, + enclave, + subscription: Mutex::new(rx3), + subscription_maker: tx.clone(), + }; + (usercall_queue_synchronizer, return_queue_synchronizer, cancel_queue_synchronizer, tx) } } @@ -1668,6 +1771,7 @@ impl ipc_queue::AsyncSynchronizer for QueueSynchronizer { fn wait(&self, event: QueueEvent) -> Pin> + '_>> { match (self.queue, event) { (Queue::Usercall, QueueEvent::NotFull) => panic!("enclave runner should not send on the usercall queue"), + (Queue::Cancel, QueueEvent::NotFull) => panic!("enclave runner should not send on the cancel queue"), (Queue::Return, QueueEvent::NotEmpty) => panic!("enclave runner should not receive on the return queue"), _ => {} } @@ -1686,12 +1790,14 @@ impl ipc_queue::AsyncSynchronizer for QueueSynchronizer { fn notify(&self, event: QueueEvent) { let ev = match (self.queue, event) { (Queue::Usercall, QueueEvent::NotEmpty) => panic!("enclave runner should not send on the usercall queue"), + (Queue::Cancel, QueueEvent::NotEmpty) => panic!("enclave runner should not send on the cancel queue"), (Queue::Return, QueueEvent::NotFull) => panic!("enclave runner should not receive on the return queue"), (Queue::Usercall, QueueEvent::NotFull) => EV_USERCALLQ_NOT_FULL, + (Queue::Cancel, QueueEvent::NotFull) => EV_CANCELQ_NOT_FULL, (Queue::Return, QueueEvent::NotEmpty) => EV_RETURNQ_NOT_EMPTY, }; // When the enclave needs to wait on a queue, it executes the wait() usercall synchronously, - // specifying EV_USERCALLQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY, or both in the event_mask. + // specifying EV_USERCALLQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY or EV_CANCELQ_NOT_FULL in the event_mask. // Userspace will wake any or all threads waiting on the appropriate event when it is triggered. for pending_events in self.enclave.event_queues.values() { pending_events.push(ev as _); diff --git a/ipc-queue/src/fifo.rs b/ipc-queue/src/fifo.rs index b000562d..d9ca36ba 100644 --- a/ipc-queue/src/fifo.rs +++ b/ipc-queue/src/fifo.rs @@ -9,7 +9,7 @@ use std::marker::PhantomData; use std::mem; #[cfg(not(target_env = "sgx"))] use { - std::sync::atomic::AtomicU64, + std::sync::atomic::{AtomicU32, AtomicU64}, std::sync::Arc, }; use std::sync::atomic::{AtomicUsize, Ordering, Ordering::SeqCst}; @@ -68,7 +68,7 @@ where let arc = Arc::new(FifoBuffer::new(len)); let inner = Fifo::from_arc(arc); let tx = AsyncSender { inner: inner.clone(), synchronizer: s.clone() }; - let rx = AsyncReceiver { inner, synchronizer: s }; + let rx = AsyncReceiver { inner, synchronizer: s, read_epoch: Arc::new(AtomicU32::new(0)) }; (tx, rx) } @@ -156,6 +156,12 @@ impl Clone for Fifo { } } +impl Fifo { + pub(crate) fn current_offsets(&self, ordering: Ordering) -> Offsets { + Offsets::new(self.offsets.load(ordering), self.data.len() as u32) + } +} + impl Fifo { pub(crate) unsafe fn from_descriptor(descriptor: FifoDescriptor) -> Self { assert!( @@ -209,7 +215,7 @@ impl Fifo { pub(crate) fn try_send_impl(&self, val: Identified) -> Result { let (new, was_empty) = loop { // 1. Load the current offsets. - let current = Offsets::new(self.offsets.load(SeqCst), self.data.len() as u32); + let current = self.current_offsets(Ordering::SeqCst); let was_empty = current.is_empty(); // 2. If the queue is full, wait, then go to step 1. @@ -218,7 +224,7 @@ impl Fifo { } // 3. Add 1 to the write offset and do an atomic compare-and-swap (CAS) - // with the current offsets. If the CAS was not succesful, go to step 1. + // with the current offsets. If the CAS was not successful, go to step 1. let new = current.increment_write_offset(); let current = current.as_usize(); if self.offsets.compare_exchange(current, new.as_usize(), SeqCst, SeqCst).is_ok() { @@ -237,9 +243,9 @@ impl Fifo { Ok(was_empty) } - pub(crate) fn try_recv_impl(&self) -> Result<(Identified, /*wake up writer:*/ bool), TryRecvError> { + pub(crate) fn try_recv_impl(&self) -> Result<(Identified, /*wake up writer:*/ bool, /*read offset wrapped around:*/bool), TryRecvError> { // 1. Load the current offsets. - let current = Offsets::new(self.offsets.load(SeqCst), self.data.len() as u32); + let current = self.current_offsets(Ordering::SeqCst); // 2. If the queue is empty, wait, then go to step 1. if current.is_empty() { @@ -275,7 +281,7 @@ impl Fifo { // 8. If the queue was full before step 7, signal the writer to wake up. let was_full = Offsets::new(before, self.data.len() as u32).is_full(); - Ok((val, was_full)) + Ok((val, was_full, new.read_offset() == 0)) } } @@ -341,6 +347,14 @@ impl Offsets { ..*self } } + + pub(crate) fn read_high_bit(&self) -> bool { + self.read & self.len == self.len + } + + pub(crate) fn write_high_bit(&self) -> bool { + self.write & self.len == self.len + } } #[cfg(test)] @@ -366,7 +380,7 @@ mod tests { } for i in 1..=7 { - let (v, wake) = inner.try_recv_impl().unwrap(); + let (v, wake, _) = inner.try_recv_impl().unwrap(); assert!(!wake); assert_eq!(v.id, i); assert_eq!(v.data.0, i); @@ -385,7 +399,7 @@ mod tests { assert!(inner.try_send_impl(Identified { id: 9, data: TestValue(9) }).is_err()); for i in 1..=8 { - let (v, wake) = inner.try_recv_impl().unwrap(); + let (v, wake, _) = inner.try_recv_impl().unwrap(); assert!(if i == 1 { wake } else { !wake }); assert_eq!(v.id, i); assert_eq!(v.data.0, i); diff --git a/ipc-queue/src/interface_async.rs b/ipc-queue/src/interface_async.rs index 9478e93e..68fd63c3 100644 --- a/ipc-queue/src/interface_async.rs +++ b/ipc-queue/src/interface_async.rs @@ -5,6 +5,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ use super::*; +use std::sync::atomic::Ordering; unsafe impl Send for AsyncSender {} unsafe impl Sync for AsyncSender {} @@ -53,10 +54,13 @@ impl AsyncReceiver { pub async fn recv(&self) -> Result, RecvError> { loop { match self.inner.try_recv_impl() { - Ok((val, wake_sender)) => { + Ok((val, wake_sender, read_wrapped_around)) => { if wake_sender { self.synchronizer.notify(QueueEvent::NotFull); } + if read_wrapped_around { + self.read_epoch.fetch_add(1, Ordering::Relaxed); + } return Ok(val); } Err(TryRecvError::QueueEmpty) => { @@ -69,6 +73,13 @@ impl AsyncReceiver { } } + pub fn position_monitor(&self) -> PositionMonitor { + PositionMonitor { + read_epoch: self.read_epoch.clone(), + fifo: self.inner.clone(), + } + } + /// Consumes `self` and returns a DescriptorGuard. /// The returned guard can be used to make `FifoDescriptor`s that remain /// valid as long as the guard is not dropped. @@ -155,6 +166,65 @@ mod tests { do_multi_sender(1024, 30, 100).await; } + #[tokio::test] + async fn positions() { + const LEN: usize = 16; + let s = TestAsyncSynchronizer::new(); + let (tx, rx) = bounded_async(LEN, s); + let monitor = rx.position_monitor(); + let mut id = 1; + + let p0 = monitor.write_position(); + tx.send(Identified { id, data: TestValue(1) }).await.unwrap(); + let p1 = monitor.write_position(); + tx.send(Identified { id: id + 1, data: TestValue(2) }).await.unwrap(); + let p2 = monitor.write_position(); + tx.send(Identified { id: id + 2, data: TestValue(3) }).await.unwrap(); + let p3 = monitor.write_position(); + id += 3; + assert!(monitor.read_position().is_past(&p0) == false); + assert!(monitor.read_position().is_past(&p1) == false); + assert!(monitor.read_position().is_past(&p2) == false); + assert!(monitor.read_position().is_past(&p3) == false); + + rx.recv().await.unwrap(); + assert!(monitor.read_position().is_past(&p0) == true); + assert!(monitor.read_position().is_past(&p1) == false); + assert!(monitor.read_position().is_past(&p2) == false); + assert!(monitor.read_position().is_past(&p3) == false); + + rx.recv().await.unwrap(); + assert!(monitor.read_position().is_past(&p0) == true); + assert!(monitor.read_position().is_past(&p1) == true); + assert!(monitor.read_position().is_past(&p2) == false); + assert!(monitor.read_position().is_past(&p3) == false); + + rx.recv().await.unwrap(); + assert!(monitor.read_position().is_past(&p0) == true); + assert!(monitor.read_position().is_past(&p1) == true); + assert!(monitor.read_position().is_past(&p2) == true); + assert!(monitor.read_position().is_past(&p3) == false); + + for i in 0..1000 { + let n = 1 + (i % LEN); + let p4 = monitor.write_position(); + for _ in 0..n { + tx.send(Identified { id, data: TestValue(id) }).await.unwrap(); + id += 1; + } + let p5 = monitor.write_position(); + for _ in 0..n { + rx.recv().await.unwrap(); + assert!(monitor.read_position().is_past(&p0) == true); + assert!(monitor.read_position().is_past(&p1) == true); + assert!(monitor.read_position().is_past(&p2) == true); + assert!(monitor.read_position().is_past(&p3) == true); + assert!(monitor.read_position().is_past(&p4) == true); + assert!(monitor.read_position().is_past(&p5) == false); + } + } + } + struct Subscription { tx: broadcast::Sender, rx: Mutex>, diff --git a/ipc-queue/src/interface_sync.rs b/ipc-queue/src/interface_sync.rs index 2096c3c6..1e07cafa 100644 --- a/ipc-queue/src/interface_sync.rs +++ b/ipc-queue/src/interface_sync.rs @@ -112,7 +112,7 @@ impl Receiver { } pub fn try_recv(&self) -> Result, TryRecvError> { - self.inner.try_recv_impl().map(|(val, wake_sender)| { + self.inner.try_recv_impl().map(|(val, wake_sender, _)| { if wake_sender { self.synchronizer.notify(QueueEvent::NotFull); } @@ -127,7 +127,7 @@ impl Receiver { pub fn recv(&self) -> Result, RecvError> { loop { match self.inner.try_recv_impl() { - Ok((val, wake_sender)) => { + Ok((val, wake_sender, _)) => { if wake_sender { self.synchronizer.notify(QueueEvent::NotFull); } diff --git a/ipc-queue/src/lib.rs b/ipc-queue/src/lib.rs index cbada6fe..4cc31117 100644 --- a/ipc-queue/src/lib.rs +++ b/ipc-queue/src/lib.rs @@ -10,6 +10,8 @@ use std::future::Future; use std::pin::Pin; +use std::sync::atomic::AtomicU32; +use std::sync::Arc; use fortanix_sgx_abi::FifoDescriptor; @@ -21,13 +23,13 @@ use std::os::fortanix_sgx::usercalls::alloc::{UserRef, UserSafeSized}; #[cfg(not(target_env = "sgx"))] use { std::ptr, - std::sync::Arc, self::fifo::FifoBuffer, }; mod fifo; mod interface_sync; mod interface_async; +mod position; #[cfg(test)] mod test_support; @@ -152,6 +154,7 @@ pub struct AsyncSender { pub struct AsyncReceiver { inner: Fifo, synchronizer: S, + read_epoch: Arc, } /// `DescriptorGuard` can produce a `FifoDescriptor` that is guaranteed @@ -167,3 +170,19 @@ impl DescriptorGuard { self.descriptor } } + +/// `PositionMonitor` can be used to record the current read/write positions +/// of a queue. Even though a queue is comprised of a limited number of slots +/// arranged as a ring buffer, we can assign a position to each value written/ +/// read to/from the queue. This is useful in case we want to know whether or +/// not a particular value written to the queue has been read. +pub struct PositionMonitor { + read_epoch: Arc, + fifo: Fifo, +} + +/// A read position in a queue. +pub struct ReadPosition(u64); + +/// A write position in a queue. +pub struct WritePosition(u64); diff --git a/ipc-queue/src/position.rs b/ipc-queue/src/position.rs new file mode 100644 index 00000000..22c30e5d --- /dev/null +++ b/ipc-queue/src/position.rs @@ -0,0 +1,48 @@ +/* Copyright (c) Fortanix, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use super::*; +use std::sync::atomic::Ordering; + +impl PositionMonitor { + pub fn read_position(&self) -> ReadPosition { + let current = self.fifo.current_offsets(Ordering::Relaxed); + let read_epoch = self.read_epoch.load(Ordering::Relaxed); + ReadPosition(((read_epoch as u64) << 32) | (current.read_offset() as u64)) + } + + pub fn write_position(&self) -> WritePosition { + let current = self.fifo.current_offsets(Ordering::Relaxed); + let mut write_epoch = self.read_epoch.load(Ordering::Relaxed); + if current.read_high_bit() != current.write_high_bit() { + write_epoch += 1; + } + WritePosition(((write_epoch as u64) << 32) | (current.write_offset() as u64)) + } +} + +impl Clone for PositionMonitor { + fn clone(&self) -> Self { + Self { + read_epoch: self.read_epoch.clone(), + fifo: self.fifo.clone(), + } + } +} + +impl ReadPosition { + /// A `WritePosition` can be compared to a `ReadPosition` **correctly** if + /// at most 2³¹ writes have occured since the write position was recorded. + pub fn is_past(&self, write: &WritePosition) -> bool { + let (read, write) = (self.0, write.0); + let hr = read & (1 << 63); + let hw = write & (1 << 63); + if hr == hw { + return read > write; + } + true + } +} From 3001201b698306ed0a2e13bfc4b5f54469306aee Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Fri, 2 Oct 2020 11:33:31 -0700 Subject: [PATCH 02/19] Async usercall interface for SGX enclaves --- .travis.yml | 3 +- Cargo.lock | 11 + Cargo.toml | 1 + async-usercalls/Cargo.toml | 30 ++ async-usercalls/src/batch_drop.rs | 127 ++++++ async-usercalls/src/callback.rs | 65 +++ async-usercalls/src/duplicated.rs | 168 ++++++++ async-usercalls/src/hacks/async_queues.rs | 50 +++ async-usercalls/src/hacks/mod.rs | 59 +++ async-usercalls/src/hacks/unsafe_typecasts.rs | 95 +++++ async-usercalls/src/io_bufs.rs | 324 ++++++++++++++ async-usercalls/src/lib.rs | 398 ++++++++++++++++++ async-usercalls/src/provider_api.rs | 274 ++++++++++++ async-usercalls/src/provider_core.rs | 66 +++ async-usercalls/src/queues.rs | 188 +++++++++ async-usercalls/src/raw.rs | 244 +++++++++++ async-usercalls/src/test_support.rs | 47 +++ async-usercalls/test.sh | 14 + 18 files changed, 2163 insertions(+), 1 deletion(-) create mode 100644 async-usercalls/Cargo.toml create mode 100644 async-usercalls/src/batch_drop.rs create mode 100644 async-usercalls/src/callback.rs create mode 100644 async-usercalls/src/duplicated.rs create mode 100644 async-usercalls/src/hacks/async_queues.rs create mode 100644 async-usercalls/src/hacks/mod.rs create mode 100644 async-usercalls/src/hacks/unsafe_typecasts.rs create mode 100644 async-usercalls/src/io_bufs.rs create mode 100644 async-usercalls/src/lib.rs create mode 100644 async-usercalls/src/provider_api.rs create mode 100644 async-usercalls/src/provider_core.rs create mode 100644 async-usercalls/src/queues.rs create mode 100644 async-usercalls/src/raw.rs create mode 100644 async-usercalls/src/test_support.rs create mode 100755 async-usercalls/test.sh diff --git a/.travis.yml b/.travis.yml index fd78e0b7..68df86a3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,7 +44,8 @@ matrix: - rustup toolchain add nightly - rustup target add x86_64-fortanix-unknown-sgx --toolchain nightly script: - - cargo test --verbose --locked --all --exclude sgxs-loaders && [ "$(echo $(nm -D target/debug/sgx-detect|grep __vdso_sgx_enter_enclave))" = "w __vdso_sgx_enter_enclave" ] + - cargo test --verbose --locked --all --exclude sgxs-loaders --exclude async-usercalls && [ "$(echo $(nm -D target/debug/sgx-detect|grep __vdso_sgx_enter_enclave))" = "w __vdso_sgx_enter_enclave" ] + - cargo test --verbose --locked -p async-usercalls --target x86_64-fortanix-unknown-sgx --no-run - cargo test --verbose --locked -p dcap-ql --features link - cargo test --verbose --locked -p dcap-ql --features verify - cargo test --verbose --locked -p ias --features mbedtls diff --git a/Cargo.lock b/Cargo.lock index 0e766e81..a908ab9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,17 @@ version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d9ff5d688f1c13395289f67db01d4826b46dd694e7580accdc3e8430f2d98e" +[[package]] +name = "async-usercalls" +version = "0.1.0" +dependencies = [ + "crossbeam-channel", + "fnv", + "fortanix-sgx-abi", + "ipc-queue", + "lazy_static 1.4.0", +] + [[package]] name = "atty" version = "0.2.14" diff --git a/Cargo.toml b/Cargo.toml index 5195a54b..90acffde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "fortanix-vme/tests/iron", "fortanix-vme/vme-pkix", "intel-sgx/aesm-client", + "intel-sgx/async-usercalls", "intel-sgx/dcap-provider", "intel-sgx/dcap-ql-sys", "intel-sgx/dcap-ql", diff --git a/async-usercalls/Cargo.toml b/async-usercalls/Cargo.toml new file mode 100644 index 00000000..c0cd25a0 --- /dev/null +++ b/async-usercalls/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "async-usercalls" +version = "0.1.0" +authors = ["Fortanix, Inc."] +license = "MPL-2.0" +edition = "2018" +description = """ +An interface for asynchronous usercalls in SGX enclaves. + +This is an SGX-only crate, you should compile it with the `x86_64-fortanix-unknown-sgx` target. +""" +repository = "https://github.com/fortanix/rust-sgx" +documentation = "https://edp.fortanix.com/docs/api/async_usercalls/" +homepage = "https://edp.fortanix.com/" +keywords = ["sgx", "async", "usercall"] +categories = ["asynchronous"] + +[dependencies] +# Project dependencies +ipc-queue = { version = "0.1", path = "../ipc-queue" } +fortanix-sgx-abi = { version = "0.4", path = "../fortanix-sgx-abi" } + +# External dependencies +lazy_static = "1.4.0" # MIT/Apache-2.0 +crossbeam-channel = "0.4" # MIT/Apache-2.0 +fnv = "1.0" # MIT/Apache-2.0 + +# For cargo test --target x86_64-fortanix-unknown-sgx +[package.metadata.fortanix-sgx] +threads = 128 diff --git a/async-usercalls/src/batch_drop.rs b/async-usercalls/src/batch_drop.rs new file mode 100644 index 00000000..62435460 --- /dev/null +++ b/async-usercalls/src/batch_drop.rs @@ -0,0 +1,127 @@ +use crate::hacks::Usercall; +use crate::provider_core::ProviderCore; +use ipc_queue::Identified; +use std::cell::RefCell; +use std::mem; +use std::os::fortanix_sgx::usercalls::alloc::{User, UserSafe}; +use std::os::fortanix_sgx::usercalls::raw::UsercallNrs; + +pub trait BatchDroppable: private::BatchDroppable {} +impl BatchDroppable for T {} + +/// Drop the given value at some point in the future (no rush!). This is useful +/// for freeing userspace memory when we don't particularly care about when the +/// buffer is freed. Multiple `free` usercalls are batched together and sent to +/// userspace asynchronously. It is also guaranteed that the memory is freed if +/// the current thread exits before there is a large enough batch. +/// +/// This is mainly an optimization to avoid exitting the enclave for each +/// usercall. Note that even when sending usercalls asynchronously, if the +/// usercall queue is empty we still need to exit the enclave to signal the +/// userspace that the queue is not empty anymore. The batch send would send +/// multiple usercalls and notify the userspace at most once. +pub fn batch_drop(t: T) { + t.batch_drop(); +} + +mod private { + use super::*; + + const BATCH_SIZE: usize = 8; + + struct BatchDropProvider { + core: ProviderCore, + deferred: Vec>, + } + + impl BatchDropProvider { + pub fn new() -> Self { + Self { + core: ProviderCore::new(None), + deferred: Vec::with_capacity(BATCH_SIZE), + } + } + + fn make_progress(&self, deferred: &[Identified]) -> usize { + let sent = self.core.try_send_multiple_usercalls(deferred); + if sent == 0 { + self.core.send_usercall(deferred[0]); + return 1; + } + sent + } + + fn maybe_send_usercall(&mut self, u: Usercall) { + self.deferred.push(self.core.assign_id(u)); + if self.deferred.len() < BATCH_SIZE { + return; + } + let sent = self.make_progress(&self.deferred); + let mut not_sent = self.deferred.split_off(sent); + self.deferred.clear(); + self.deferred.append(&mut not_sent); + } + + pub fn free(&mut self, buf: User) { + let ptr = buf.into_raw(); + let size = unsafe { mem::size_of_val(&mut *ptr) }; + let alignment = T::align_of(); + let ptr = ptr as *mut u8; + let u = Usercall(UsercallNrs::free as _, ptr as _, size as _, alignment as _, 0); + self.maybe_send_usercall(u); + } + } + + impl Drop for BatchDropProvider { + fn drop(&mut self) { + let mut sent = 0; + while sent < self.deferred.len() { + sent += self.make_progress(&self.deferred[sent..]); + } + } + } + + std::thread_local! { + static PROVIDER: RefCell = RefCell::new(BatchDropProvider::new()); + } + + pub trait BatchDroppable { + fn batch_drop(self); + } + + impl BatchDroppable for User { + fn batch_drop(self) { + PROVIDER.with(|p| p.borrow_mut().free(self)); + } + } +} + +#[cfg(test)] +mod tests { + use super::batch_drop; + use std::os::fortanix_sgx::usercalls::alloc::User; + use std::thread; + + #[test] + fn basic() { + for _ in 0..100 { + batch_drop(User::<[u8]>::uninitialized(100)); + } + } + + #[test] + fn multiple_threads() { + const THREADS: usize = 16; + let mut handles = Vec::with_capacity(THREADS); + for _ in 0..THREADS { + handles.push(thread::spawn(move || { + for _ in 0..1000 { + batch_drop(User::<[u8]>::uninitialized(100)); + } + })); + } + for h in handles { + h.join().unwrap(); + } + } +} diff --git a/async-usercalls/src/callback.rs b/async-usercalls/src/callback.rs new file mode 100644 index 00000000..7586ebb0 --- /dev/null +++ b/async-usercalls/src/callback.rs @@ -0,0 +1,65 @@ +use crate::duplicated::{FromSgxResult, ReturnValue}; +use crate::hacks::Return; +use fortanix_sgx_abi::{invoke_with_usercalls, Fd, Result}; +use std::io; + +pub struct CbFn(Box); + +impl CbFn { + fn call(self, t: T) { + (self.0)(t); + } +} + +impl From for CbFn +where + F: FnOnce(T) + Send + 'static, +{ + fn from(f: F) -> Self { + Self(Box::new(f)) + } +} + +macro_rules! cbfn_type { + ( ) => { CbFn<()> }; + ( -> ! ) => { () }; + ( -> u64 ) => { CbFn }; + ( -> (Result, usize) ) => { CbFn> }; + ( -> (Result, u64) ) => { CbFn> }; + ( -> (Result, Fd) ) => { CbFn> }; + ( -> (Result, *mut u8) ) => { CbFn> }; + ( -> Result ) => { CbFn> }; +} + +macro_rules! call_cbfn { + ( $cb:ident, $rv:expr, ) => { let x: () = $rv; $cb.call(x); }; + ( $cb:ident, $rv:expr, -> ! ) => { let _: ! = $rv; }; + ( $cb:ident, $rv:expr, -> u64 ) => { let x: u64 = $rv; $cb.call(x); }; + ( $cb:ident, $rv:expr, -> $t:ty ) => { let x: $t = $rv; $cb.call(x.from_sgx_result()); }; +} + +macro_rules! define_callback { + ($(fn $name:ident($($n:ident: $t:ty),*) $(-> $r:tt)*; )*) => { + #[allow(unused)] + #[allow(non_camel_case_types)] + pub(crate) enum Callback { + $( $name(cbfn_type! { $(-> $r)* }), )* + } + + impl Callback { + pub(crate) fn call(self, ret: Return) { + match self {$( + Callback::$name(_cb) => { + call_cbfn!( + _cb, + ReturnValue::from_registers(stringify!($name), (ret.0, ret.1)), + $(-> $r)* + ); + } + )*} + } + } + }; +} + +invoke_with_usercalls!(define_callback); diff --git a/async-usercalls/src/duplicated.rs b/async-usercalls/src/duplicated.rs new file mode 100644 index 00000000..0a39e5a1 --- /dev/null +++ b/async-usercalls/src/duplicated.rs @@ -0,0 +1,168 @@ +//! this file contains code duplicated from libstd's sys/sgx +use fortanix_sgx_abi::{Error, Result, RESULT_SUCCESS}; +use std::io; +use std::ptr::NonNull; + +fn check_os_error(err: Result) -> i32 { + // FIXME: not sure how to make sure all variants of Error are covered + if err == Error::NotFound as _ + || err == Error::PermissionDenied as _ + || err == Error::ConnectionRefused as _ + || err == Error::ConnectionReset as _ + || err == Error::ConnectionAborted as _ + || err == Error::NotConnected as _ + || err == Error::AddrInUse as _ + || err == Error::AddrNotAvailable as _ + || err == Error::BrokenPipe as _ + || err == Error::AlreadyExists as _ + || err == Error::WouldBlock as _ + || err == Error::InvalidInput as _ + || err == Error::InvalidData as _ + || err == Error::TimedOut as _ + || err == Error::WriteZero as _ + || err == Error::Interrupted as _ + || err == Error::Other as _ + || err == Error::UnexpectedEof as _ + || ((Error::UserRangeStart as _)..=(Error::UserRangeEnd as _)).contains(&err) + { + err + } else { + panic!("Usercall: returned invalid error value {}", err) + } +} + +pub trait FromSgxResult { + type Return; + + fn from_sgx_result(self) -> io::Result; +} + +impl FromSgxResult for (Result, T) { + type Return = T; + + fn from_sgx_result(self) -> io::Result { + if self.0 == RESULT_SUCCESS { + Ok(self.1) + } else { + Err(io::Error::from_raw_os_error(check_os_error(self.0))) + } + } +} + +impl FromSgxResult for Result { + type Return = (); + + fn from_sgx_result(self) -> io::Result { + if self == RESULT_SUCCESS { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(check_os_error(self))) + } + } +} + +type Register = u64; + +pub trait RegisterArgument { + fn from_register(_: Register) -> Self; + fn into_register(self) -> Register; +} + +pub trait ReturnValue { + fn from_registers(call: &'static str, regs: (Register, Register)) -> Self; +} + +macro_rules! define_ra { + (< $i:ident > $t:ty) => { + impl<$i> RegisterArgument for $t { + fn from_register(a: Register) -> Self { + a as _ + } + fn into_register(self) -> Register { + self as _ + } + } + }; + ($i:ty as $t:ty) => { + impl RegisterArgument for $t { + fn from_register(a: Register) -> Self { + a as $i as _ + } + fn into_register(self) -> Register { + self as $i as _ + } + } + }; + ($t:ty) => { + impl RegisterArgument for $t { + fn from_register(a: Register) -> Self { + a as _ + } + fn into_register(self) -> Register { + self as _ + } + } + }; +} + +define_ra!(Register); +define_ra!(i64); +define_ra!(u32); +define_ra!(u32 as i32); +define_ra!(u16); +define_ra!(u16 as i16); +define_ra!(u8); +define_ra!(u8 as i8); +define_ra!(usize); +define_ra!(usize as isize); +define_ra!( *const T); +define_ra!( *mut T); + +impl RegisterArgument for bool { + fn from_register(a: Register) -> bool { + if a != 0 { + true + } else { + false + } + } + fn into_register(self) -> Register { + self as _ + } +} + +impl RegisterArgument for Option> { + fn from_register(a: Register) -> Option> { + NonNull::new(a as _) + } + fn into_register(self) -> Register { + self.map_or(0 as _, NonNull::as_ptr) as _ + } +} + +impl ReturnValue for ! { + fn from_registers(call: &'static str, _regs: (Register, Register)) -> Self { + panic!("Usercall {}: did not expect to be re-entered", call); + } +} + +impl ReturnValue for () { + fn from_registers(_call: &'static str, usercall_retval: (Register, Register)) -> Self { + assert!(usercall_retval.0 == 0); + assert!(usercall_retval.1 == 0); + () + } +} + +impl ReturnValue for T { + fn from_registers(_call: &'static str, usercall_retval: (Register, Register)) -> Self { + assert!(usercall_retval.1 == 0); + T::from_register(usercall_retval.0) + } +} + +impl ReturnValue for (T, U) { + fn from_registers(_call: &'static str, regs: (Register, Register)) -> Self { + (T::from_register(regs.0), U::from_register(regs.1)) + } +} diff --git a/async-usercalls/src/hacks/async_queues.rs b/async-usercalls/src/hacks/async_queues.rs new file mode 100644 index 00000000..a325a28f --- /dev/null +++ b/async-usercalls/src/hacks/async_queues.rs @@ -0,0 +1,50 @@ +use super::{Cancel, Return, Usercall}; +use crate::duplicated::ReturnValue; +use fortanix_sgx_abi::FifoDescriptor; +use std::num::NonZeroU64; +use std::os::fortanix_sgx::usercalls; +use std::os::fortanix_sgx::usercalls::raw; +use std::{mem, ptr}; + +// TODO: remove these once support for cancel queue is added in `std::os::fortanix_sgx` + +pub unsafe fn async_queues( + usercall_queue: *mut FifoDescriptor, + return_queue: *mut FifoDescriptor, + cancel_queue: *mut FifoDescriptor, +) -> raw::Result { + ReturnValue::from_registers( + "async_queues", + raw::do_usercall( + NonZeroU64::new(raw::UsercallNrs::async_queues as _).unwrap(), + usercall_queue as _, + return_queue as _, + cancel_queue as _, + 0, + false, + ), + ) +} + +pub unsafe fn alloc_descriptor() -> *mut FifoDescriptor { + usercalls::alloc( + mem::size_of::>(), + mem::align_of::>(), + ) + .expect("failed to allocate userspace memory") as _ +} + +pub unsafe fn to_enclave(ptr: *mut FifoDescriptor) -> FifoDescriptor { + let mut dest: FifoDescriptor = mem::zeroed(); + ptr::copy( + ptr as *const u8, + (&mut dest) as *mut FifoDescriptor as *mut u8, + mem::size_of_val(&mut dest), + ); + usercalls::free( + ptr as _, + mem::size_of::>(), + mem::align_of::>(), + ); + dest +} diff --git a/async-usercalls/src/hacks/mod.rs b/async-usercalls/src/hacks/mod.rs new file mode 100644 index 00000000..9011c63e --- /dev/null +++ b/async-usercalls/src/hacks/mod.rs @@ -0,0 +1,59 @@ +use std::ops::{Deref, DerefMut}; +use std::os::fortanix_sgx::usercalls::alloc::{User, UserSafeSized}; +use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; + +mod async_queues; +mod unsafe_typecasts; + +pub use self::async_queues::{alloc_descriptor, async_queues, to_enclave}; +pub use self::unsafe_typecasts::{new_std_listener, new_std_stream}; + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct Usercall(pub u64, pub u64, pub u64, pub u64, pub u64); + +unsafe impl UserSafeSized for Usercall {} + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct Return(pub u64, pub u64); + +unsafe impl UserSafeSized for Return {} + +#[repr(C)] +#[derive(Copy, Clone, Default)] +pub struct Cancel; + +unsafe impl UserSafeSized for Cancel {} + +// Interim solution until we mark the target types appropriately +pub(crate) struct MakeSend(T); + +impl MakeSend { + pub fn new(t: T) -> Self { + Self(t) + } + + #[allow(unused)] + pub fn into_inner(self) -> T { + self.0 + } +} + +impl Deref for MakeSend { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for MakeSend { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +unsafe impl Send for MakeSend {} +unsafe impl Send for MakeSend> {} +unsafe impl Send for MakeSend> {} diff --git a/async-usercalls/src/hacks/unsafe_typecasts.rs b/async-usercalls/src/hacks/unsafe_typecasts.rs new file mode 100644 index 00000000..1e3d67c5 --- /dev/null +++ b/async-usercalls/src/hacks/unsafe_typecasts.rs @@ -0,0 +1,95 @@ +//! The incredibly unsafe code in this module allows us to create +//! `std::net::TcpStream` and `std::net::TcpListener` types from their raw +//! components in SGX. +//! +//! This is obviously very unsafe and not maintainable and is only intended as +//! an iterim solution until we add similar functionality as extension traits +//! in `std::os::fortanix_sgx`. +use fortanix_sgx_abi::Fd; + +mod sgx { + use fortanix_sgx_abi::Fd; + use std::sync::Arc; + + #[derive(Debug)] + pub struct FileDesc { + fd: Fd, + } + + #[derive(Debug, Clone)] + pub struct Socket { + inner: Arc, + local_addr: Option, + } + + #[derive(Clone)] + pub struct TcpStream { + inner: Socket, + peer_addr: Option, + } + + impl TcpStream { + pub fn new(fd: Fd, local_addr: Option, peer_addr: Option) -> TcpStream { + TcpStream { + inner: Socket { + inner: Arc::new(FileDesc { fd }), + local_addr, + }, + peer_addr, + } + } + } + + #[derive(Clone)] + pub struct TcpListener { + inner: Socket, + } + + impl TcpListener { + pub fn new(fd: Fd, local_addr: Option) -> TcpListener { + TcpListener { + inner: Socket { + inner: Arc::new(FileDesc { fd }), + local_addr, + }, + } + } + } +} + +struct TcpStream(self::sgx::TcpStream); +struct TcpListener(self::sgx::TcpListener); + +pub unsafe fn new_std_stream(fd: Fd, local_addr: Option, peer_addr: Option) -> std::net::TcpStream { + let stream = TcpStream(sgx::TcpStream::new(fd, local_addr, peer_addr)); + std::mem::transmute(stream) +} + +pub unsafe fn new_std_listener(fd: Fd, local_addr: Option) -> std::net::TcpListener { + let listener = TcpListener(sgx::TcpListener::new(fd, local_addr)); + std::mem::transmute(listener) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::mem; + use std::os::fortanix_sgx::io::AsRawFd; + + #[test] + fn sanity_check() { + let fd = 42; + let local = "1.2.3.4:1234"; + let peer = "5.6.7.8:443"; + let stream = unsafe { new_std_stream(fd, Some(local.to_owned()), Some(peer.to_owned())) }; + assert_eq!(stream.as_raw_fd(), fd); + assert_eq!(stream.local_addr().unwrap().to_string(), local); + assert_eq!(stream.peer_addr().unwrap().to_string(), peer); + mem::forget(stream); // not a real stream... + + let listener = unsafe { new_std_listener(fd, Some(local.to_owned())) }; + assert_eq!(listener.as_raw_fd(), fd); + assert_eq!(listener.local_addr().unwrap().to_string(), local); + mem::forget(listener); // not a real listener... + } +} diff --git a/async-usercalls/src/io_bufs.rs b/async-usercalls/src/io_bufs.rs new file mode 100644 index 00000000..a8ede0de --- /dev/null +++ b/async-usercalls/src/io_bufs.rs @@ -0,0 +1,324 @@ +use std::cell::UnsafeCell; +use std::cmp; +use std::io::IoSlice; +use std::ops::{Deref, DerefMut, Range}; +use std::os::fortanix_sgx::usercalls::alloc::{User, UserRef}; +use std::sync::Arc; + +pub struct UserBuf(UserBufKind); + +enum UserBufKind { + Owned { + user: User<[u8]>, + range: Range, + }, + Shared { + user: Arc>>, + range: Range, + }, +} + +impl UserBuf { + pub fn into_user(self) -> Result, Self> { + match self.0 { + UserBufKind::Owned { user, .. } => Ok(user), + UserBufKind::Shared { user, range } => Err(Self(UserBufKind::Shared { user, range })), + } + } + + fn into_shared(self) -> Option>>> { + match self.0 { + UserBufKind::Owned { .. } => None, + UserBufKind::Shared { user, .. } => Some(user), + } + } +} + +unsafe impl Send for UserBuf {} + +impl Deref for UserBuf { + type Target = UserRef<[u8]>; + + fn deref(&self) -> &Self::Target { + match self.0 { + UserBufKind::Owned { ref user, ref range } => &user[range.start..range.end], + UserBufKind::Shared { ref user, ref range } => { + let user = unsafe { &*user.get() }; + &user[range.start..range.end] + } + } + } +} + +impl DerefMut for UserBuf { + fn deref_mut(&mut self) -> &mut Self::Target { + match self.0 { + UserBufKind::Owned { + ref mut user, + ref range, + } => &mut user[range.start..range.end], + UserBufKind::Shared { ref user, ref range } => { + let user = unsafe { &mut *user.get() }; + &mut user[range.start..range.end] + } + } + } +} + +impl From> for UserBuf { + fn from(user: User<[u8]>) -> Self { + UserBuf(UserBufKind::Owned { + range: 0..user.len(), + user, + }) + } +} + +impl From<(User<[u8]>, Range)> for UserBuf { + fn from(pair: (User<[u8]>, Range)) -> Self { + UserBuf(UserBufKind::Owned { + user: pair.0, + range: pair.1, + }) + } +} + +/// `WriteBuffer` provides a ring buffer that can be written to by the code +/// running in the enclave while a portion of it can be passed to a `write` +/// usercall running concurrently. It ensures that enclave code does not write +/// to the portion sent to userspace. +pub struct WriteBuffer { + userbuf: Arc>>, + buf_len: usize, + read: u32, + write: u32, +} + +unsafe impl Send for WriteBuffer {} + +impl WriteBuffer { + pub fn new(userbuf: User<[u8]>) -> Self { + Self { + buf_len: userbuf.len(), + userbuf: Arc::new(UnsafeCell::new(userbuf)), + read: 0, + write: 0, + } + } + + pub fn write_vectored(&mut self, bufs: &[IoSlice<'_>]) -> usize { + if self.is_full() { + return 0; + } + let mut wrote = 0; + for buf in bufs { + wrote += self.write(buf); + } + wrote + } + + pub fn write(&mut self, buf: &[u8]) -> usize { + let (_, write_offset) = self.offsets(); + let rem = self.remaining_capacity(); + let can_write = cmp::min(buf.len(), rem); + let end = cmp::min(self.buf_len, write_offset + can_write); + let n = end - write_offset; + unsafe { + let userbuf = &mut *self.userbuf.get(); + userbuf[write_offset..write_offset + n].copy_from_enclave(&buf[..n]); + } + self.advance_write(n); + n + if n < can_write { self.write(&buf[n..]) } else { 0 } + } + + /// This function returns a slice of bytes appropriate for writing to a socket. + /// Once some or all of these bytes are successfully written to the socket, + /// `self.consume()` must be called to actually consume those bytes. + /// + /// Returns None if the buffer is empty. + /// + /// Panics if called more than once in a row without either calling `consume()` + /// or dropping the previously returned buffer. + pub fn consumable_chunk(&mut self) -> Option { + assert!( + Arc::strong_count(&self.userbuf) == 1, + "called consumable_chunk() more than once in a row" + ); + let range = match self.offsets() { + (_, _) if self.read == self.write => return None, // empty + (r, w) if r < w => r..w, + (r, _) => r..self.buf_len, + }; + Some(UserBuf(UserBufKind::Shared { + user: self.userbuf.clone(), + range, + })) + } + + /// Mark `n` bytes as consumed. `buf` must have been produced by a call + /// to `self.consumable_chunk()`. + /// Panics if: + /// - `n > buf.len()` + /// - `buf` was not produced by `self.consumable_chunk()` + /// + /// This function is supposed to be used in conjunction with `consumable_chunk()`. + pub fn consume(&mut self, buf: UserBuf, n: usize) { + assert!(n <= buf.len()); + const PANIC_MESSAGE: &'static str = "`buf` not produced by self.consumable_chunk()"; + let buf = buf.into_shared().expect(PANIC_MESSAGE); + assert!(Arc::ptr_eq(&self.userbuf, &buf), PANIC_MESSAGE); + drop(buf); + assert!(Arc::strong_count(&self.userbuf) == 1, PANIC_MESSAGE); + self.advance_read(n); + } + + fn len(&self) -> usize { + match self.offsets() { + (_, _) if self.read == self.write => 0, // empty + (r, w) if r == w && self.read != self.write => self.buf_len, // full + (r, w) if r < w => w - r, + (r, w) => w + self.buf_len - r, + } + } + + fn remaining_capacity(&self) -> usize { + let len = self.len(); + debug_assert!(len <= self.buf_len); + self.buf_len - len + } + + fn offsets(&self) -> (usize, usize) { + (self.read as usize % self.buf_len, self.write as usize % self.buf_len) + } + + pub fn is_empty(&self) -> bool { + self.read == self.write + } + + fn is_full(&self) -> bool { + let (read_offset, write_offset) = self.offsets(); + read_offset == write_offset && self.read != self.write + } + + fn advance_read(&mut self, by: usize) { + debug_assert!(by <= self.len()); + self.read = ((self.read as usize + by) % (self.buf_len * 2)) as _; + } + + fn advance_write(&mut self, by: usize) { + debug_assert!(by <= self.remaining_capacity()); + self.write = ((self.write as usize + by) % (self.buf_len * 2)) as _; + } +} + +pub struct ReadBuffer { + userbuf: User<[u8]>, + position: usize, + len: usize, +} + +impl ReadBuffer { + /// Constructs a new `ReadBuffer`, assuming `len` bytes of `userbuf` have + /// meaningful data. Panics if `len > userbuf.len()`. + pub fn new(userbuf: User<[u8]>, len: usize) -> ReadBuffer { + assert!(len <= userbuf.len()); + ReadBuffer { + userbuf, + position: 0, + len, + } + } + + pub fn read(&mut self, buf: &mut [u8]) -> usize { + debug_assert!(self.position <= self.len); + if self.position == self.len { + return 0; + } + let n = cmp::min(buf.len(), self.len - self.position); + self.userbuf[self.position..self.position + n].copy_to_enclave(&mut buf[..n]); + self.position += n; + n + } + + /// Returns the number of bytes that have not been read yet. + pub fn remaining_bytes(&self) -> usize { + debug_assert!(self.position <= self.len); + self.len - self.position + } + + pub fn len(&self) -> usize { + self.len + } + + /// Consumes self and returns the internal userspace buffer. + /// It's the caller's responsibility to ensure all bytes have been read + /// before calling this function. + pub fn into_inner(self) -> User<[u8]> { + self.userbuf + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::fortanix_sgx::usercalls::alloc::User; + + #[test] + fn write_buffer_basic() { + const LENGTH: usize = 1024; + let mut write_buffer = WriteBuffer::new(User::<[u8]>::uninitialized(1024)); + + let buf = vec![0u8; LENGTH]; + assert_eq!(write_buffer.write(&buf), LENGTH); + assert_eq!(write_buffer.write(&buf), 0); + + let chunk = write_buffer.consumable_chunk().unwrap(); + write_buffer.consume(chunk, 200); + assert_eq!(write_buffer.write(&buf), 200); + assert_eq!(write_buffer.write(&buf), 0); + } + + #[test] + #[should_panic] + fn call_consumable_chunk_twice() { + const LENGTH: usize = 1024; + let mut write_buffer = WriteBuffer::new(User::<[u8]>::uninitialized(1024)); + + let buf = vec![0u8; LENGTH]; + assert_eq!(write_buffer.write(&buf), LENGTH); + assert_eq!(write_buffer.write(&buf), 0); + + let chunk1 = write_buffer.consumable_chunk().unwrap(); + let _ = write_buffer.consumable_chunk().unwrap(); + drop(chunk1); + } + + #[test] + #[should_panic] + fn consume_wrong_buf() { + const LENGTH: usize = 1024; + let mut write_buffer = WriteBuffer::new(User::<[u8]>::uninitialized(1024)); + + let buf = vec![0u8; LENGTH]; + assert_eq!(write_buffer.write(&buf), LENGTH); + assert_eq!(write_buffer.write(&buf), 0); + + let unrelated_buf: UserBuf = User::<[u8]>::uninitialized(512).into(); + write_buffer.consume(unrelated_buf, 100); + } + + #[test] + fn read_buffer_basic() { + let mut buf = User::<[u8]>::uninitialized(64); + const DATA: &'static [u8] = b"hello"; + buf[0..DATA.len()].copy_from_enclave(DATA); + + let mut read_buffer = ReadBuffer::new(buf, DATA.len()); + assert_eq!(read_buffer.len(), DATA.len()); + assert_eq!(read_buffer.remaining_bytes(), DATA.len()); + let mut buf = [0u8; 8]; + assert_eq!(read_buffer.read(&mut buf), DATA.len()); + assert_eq!(read_buffer.remaining_bytes(), 0); + assert_eq!(&buf, b"hello\0\0\0"); + } +} diff --git a/async-usercalls/src/lib.rs b/async-usercalls/src/lib.rs new file mode 100644 index 00000000..5ac00326 --- /dev/null +++ b/async-usercalls/src/lib.rs @@ -0,0 +1,398 @@ +#![feature(sgx_platform)] +#![feature(never_type)] +#![cfg_attr(test, feature(unboxed_closures))] +#![cfg_attr(test, feature(fn_traits))] + +use crossbeam_channel as mpmc; +use ipc_queue::Identified; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::Duration; + +mod batch_drop; +mod callback; +mod duplicated; +mod hacks; +mod io_bufs; +mod provider_api; +mod provider_core; +mod queues; +mod raw; +#[cfg(test)] +mod test_support; + +pub use self::batch_drop::batch_drop; +pub use self::callback::CbFn; +pub use self::io_bufs::{ReadBuffer, UserBuf, WriteBuffer}; +pub use self::raw::RawApi; + +use self::callback::*; +use self::hacks::{Cancel, Return, Usercall}; +use self::provider_core::ProviderCore; +use self::queues::*; + +pub struct CancelHandle(Identified); + +impl CancelHandle { + pub fn cancel(self) { + PROVIDERS + .cancel_sender() + .send(self.0) + .expect("failed to send cancellation"); + } + + pub(crate) fn new(c: Identified) -> Self { + CancelHandle(c) + } +} + +/// This type provides a mechanism for submitting usercalls asynchronously. +/// Usercalls are sent to the enclave runner through a queue. The results are +/// retrieved when `CallbackHandler::poll` is called. Users are notified of the +/// results through callback functions. +/// +/// Users of this type should take care not to block execution in callbacks. +/// Certain usercalls can be cancelled through a handle, but note that it is +/// still possible to receive successful results for cancelled usercalls. +pub struct AsyncUsercallProvider { + core: ProviderCore, + callback_tx: mpmc::Sender<(u64, Callback)>, +} + +impl AsyncUsercallProvider { + pub fn new() -> (Self, CallbackHandler) { + let (return_tx, return_rx) = mpmc::unbounded(); + let core = ProviderCore::new(Some(return_tx)); + let callbacks = Mutex::new(HashMap::new()); + let (callback_tx, callback_rx) = mpmc::unbounded(); + let provider = Self { core, callback_tx }; + let waker = CallbackHandlerWaker::new(); + let handler = CallbackHandler { + return_rx, + callbacks, + callback_rx, + waker, + }; + (provider, handler) + } + + #[cfg(test)] + pub(crate) fn provider_id(&self) -> u32 { + self.core.provider_id() + } + + fn send_usercall(&self, usercall: Usercall, callback: Option) -> CancelHandle { + let usercall = self.core.assign_id(usercall); + if let Some(callback) = callback { + self.callback_tx + .send((usercall.id, callback)) + .expect("failed to send callback"); + } + self.core.send_usercall(usercall) + } +} + +#[derive(Clone)] +pub struct CallbackHandlerWaker { + rx: mpmc::Receiver<()>, + tx: mpmc::Sender<()>, +} + +impl CallbackHandlerWaker { + fn new() -> Self { + let (tx, rx) = mpmc::bounded(1); + Self { tx, rx } + } + + /// Interrupts the currently running or a future call to the related + /// CallbackHandler's `poll()`. + pub fn wake(&self) { + let _ = self.tx.try_send(()); + } + + /// Clears the effect of a previous call to `self.wake()` that is not yet + /// observed by `CallbackHandler::poll()`. + pub fn clear(&self) { + let _ = self.rx.try_recv(); + } +} + +pub struct CallbackHandler { + return_rx: mpmc::Receiver>, + callbacks: Mutex>, + // This is used so that threads sending usercalls don't have to take the lock. + callback_rx: mpmc::Receiver<(u64, Callback)>, + waker: CallbackHandlerWaker, +} + +impl CallbackHandler { + const RECV_BATCH_SIZE: usize = 128; + + // Returns an object that can be used to interrupt a blocked `self.poll()`. + pub fn waker(&self) -> CallbackHandlerWaker { + self.waker.clone() + } + + #[inline] + fn recv_returns(&self, timeout: Option, returns: &mut [Identified]) -> usize { + let first = match timeout { + None => mpmc::select! { + recv(self.return_rx) -> res => res.ok(), + recv(self.waker.rx) -> _res => return 0, + }, + Some(timeout) => mpmc::select! { + recv(self.return_rx) -> res => res.ok(), + recv(self.waker.rx) -> _res => return 0, + default(timeout) => return 0, + }, + } + .expect("return channel closed unexpectedly"); + let mut count = 0; + for ret in std::iter::once(first).chain(self.return_rx.try_iter().take(returns.len() - 1)) { + returns[count] = ret; + count += 1; + } + count + } + + /// Poll for returned usercalls and execute their respective callback + /// functions. If `timeout` is `None`, it will block execution until at + /// least one return is received, otherwise it will block until there is a + /// return or timeout is elapsed. Returns the number of executed callbacks. + /// This can be interrupted using `CallbackHandlerWaker::wake()`. + pub fn poll(&self, timeout: Option) -> usize { + // 1. wait for returns + let mut returns = [Identified::default(); Self::RECV_BATCH_SIZE]; + let returns = match self.recv_returns(timeout, &mut returns) { + 0 => return 0, + n => &returns[..n], + }; + // 2. try to lock the mutex, if successful, receive all pending callbacks and put them in the hash map + let mut guard = match self.callbacks.try_lock() { + Ok(mut callbacks) => { + for (id, cb) in self.callback_rx.try_iter() { + callbacks.insert(id, cb); + } + callbacks + } + _ => self.callbacks.lock().unwrap(), + }; + // 3. remove callbacks for returns received in step 1 from the hash map + let mut ret_callbacks = Vec::with_capacity(returns.len()); + for ret in returns { + let cb = guard.remove(&ret.id); + ret_callbacks.push((ret, cb)); + } + drop(guard); + // 4. execute the callbacks without hugging the mutex + let mut count = 0; + for (ret, cb) in ret_callbacks { + if let Some(cb) = cb { + cb.call(ret.data); + count += 1; + } + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::hacks::MakeSend; + use crate::test_support::*; + use crossbeam_channel as mpmc; + use std::io; + use std::net::{TcpListener, TcpStream}; + use std::os::fortanix_sgx::io::AsRawFd; + use std::os::fortanix_sgx::usercalls::alloc::User; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use std::thread; + use std::time::Duration; + + #[test] + fn cancel_accept() { + let provider = AutoPollingProvider::new(); + let port = 6688; + let addr = format!("0.0.0.0:{}", port); + let (tx, rx) = mpmc::bounded(1); + provider.bind_stream(&addr, move |res| { + tx.send(res).unwrap(); + }); + let bind_res = rx.recv().unwrap(); + let listener = bind_res.unwrap(); + let fd = listener.as_raw_fd(); + let accept_count = Arc::new(AtomicUsize::new(0)); + let accept_count1 = Arc::clone(&accept_count); + let (tx, rx) = mpmc::bounded(1); + let accept = provider.accept_stream(fd, move |res| { + if let Ok(_) = res { + accept_count1.fetch_add(1, Ordering::Relaxed); + } + tx.send(()).unwrap(); + }); + accept.cancel(); + thread::sleep(Duration::from_millis(10)); + let _ = TcpStream::connect(&addr); + let _ = rx.recv(); + assert_eq!(accept_count.load(Ordering::Relaxed), 0); + } + + #[test] + fn connect() { + let listener = TcpListener::bind("0.0.0.0:0").unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let provider = AutoPollingProvider::new(); + let (tx, rx) = mpmc::bounded(1); + provider.connect_stream(&addr, move |res| { + tx.send(res).unwrap(); + }); + let res = rx.recv().unwrap(); + assert!(res.is_ok()); + } + + #[test] + fn safe_alloc_free() { + let provider = AutoPollingProvider::new(); + + const LEN: usize = 64 * 1024; + let (tx, rx) = mpmc::bounded(1); + provider.alloc_slice::(LEN, move |res| { + let buf = res.expect("failed to allocate memory"); + tx.send(MakeSend::new(buf)).unwrap(); + }); + let user_buf = rx.recv().unwrap().into_inner(); + assert_eq!(user_buf.len(), LEN); + + let (tx, rx) = mpmc::bounded(1); + let cb = move || { + tx.send(()).unwrap(); + }; + provider.free(user_buf, Some(cb)); + rx.recv().unwrap(); + } + + #[test] + fn callback_handler_waker() { + let (_provider, handler) = AsyncUsercallProvider::new(); + let waker = handler.waker(); + let (tx, rx) = mpmc::bounded(1); + let h = thread::spawn(move || { + let n1 = handler.poll(None); + tx.send(()).unwrap(); + let n2 = handler.poll(Some(Duration::from_secs(3))); + tx.send(()).unwrap(); + n1 + n2 + }); + for _ in 0..2 { + waker.wake(); + rx.recv().unwrap(); + } + assert_eq!(h.join().unwrap(), 0); + } + + #[test] + #[ignore] + fn echo() { + println!(); + let provider = Arc::new(AutoPollingProvider::new()); + const ADDR: &'static str = "0.0.0.0:7799"; + let (tx, rx) = mpmc::bounded(1); + provider.bind_stream(ADDR, move |res| { + tx.send(res).unwrap(); + }); + let bind_res = rx.recv().unwrap(); + let listener = bind_res.unwrap(); + println!("bind done: {:?}", listener); + let fd = listener.as_raw_fd(); + let cb = KeepAccepting { + listener, + provider: Arc::clone(&provider), + }; + provider.accept_stream(fd, cb); + thread::sleep(Duration::from_secs(60)); + } + + struct KeepAccepting { + listener: TcpListener, + provider: Arc, + } + + impl FnOnce<(io::Result,)> for KeepAccepting { + type Output = (); + + extern "rust-call" fn call_once(self, args: (io::Result,)) -> Self::Output { + let res = args.0; + println!("accept result: {:?}", res); + if let Ok(stream) = res { + let fd = stream.as_raw_fd(); + let cb = Echo { + stream, + read: true, + provider: self.provider.clone(), + }; + self.provider + .read(fd, User::<[u8]>::uninitialized(Echo::READ_BUF_SIZE), cb); + } + let provider = Arc::clone(&self.provider); + provider.accept_stream(self.listener.as_raw_fd(), self); + } + } + + struct Echo { + stream: TcpStream, + read: bool, + provider: Arc, + } + + impl Echo { + const READ_BUF_SIZE: usize = 1024; + + fn close(self) { + let fd = self.stream.as_raw_fd(); + println!("connection closed, fd = {}", fd); + self.provider.close(fd, None::>); + } + } + + // read callback + impl FnOnce<(io::Result, User<[u8]>)> for Echo { + type Output = (); + + extern "rust-call" fn call_once(mut self, args: (io::Result, User<[u8]>)) -> Self::Output { + let (res, user) = args; + assert!(self.read); + match res { + Ok(len) if len > 0 => { + self.read = false; + let provider = Arc::clone(&self.provider); + provider.write(self.stream.as_raw_fd(), (user, 0..len).into(), self); + } + _ => self.close(), + } + } + } + + // write callback + impl FnOnce<(io::Result, UserBuf)> for Echo { + type Output = (); + + extern "rust-call" fn call_once(mut self, args: (io::Result, UserBuf)) -> Self::Output { + let (res, _) = args; + assert!(!self.read); + match res { + Ok(len) if len > 0 => { + self.read = true; + let provider = Arc::clone(&self.provider); + provider.read( + self.stream.as_raw_fd(), + User::<[u8]>::uninitialized(Echo::READ_BUF_SIZE), + self, + ); + } + _ => self.close(), + } + } + } +} diff --git a/async-usercalls/src/provider_api.rs b/async-usercalls/src/provider_api.rs new file mode 100644 index 00000000..234dbddb --- /dev/null +++ b/async-usercalls/src/provider_api.rs @@ -0,0 +1,274 @@ +use crate::batch_drop; +use crate::hacks::{new_std_listener, new_std_stream, MakeSend}; +use crate::io_bufs::UserBuf; +use crate::raw::RawApi; +use crate::{AsyncUsercallProvider, CancelHandle}; +use fortanix_sgx_abi::Fd; +use std::io; +use std::mem::{self, ManuallyDrop}; +use std::net::{TcpListener, TcpStream}; +use std::os::fortanix_sgx::usercalls::alloc::{User, UserRef, UserSafe}; +use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +impl AsyncUsercallProvider { + /// Sends an asynchronous `read` usercall. `callback` is called when a + /// return value is received from userspace. `read_buf` is returned as an + /// argument to `callback` along with the result of the `read` usercall. + /// + /// Returns a handle that can be used to cancel the usercall if desired. + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn read(&self, fd: Fd, read_buf: User<[u8]>, callback: F) -> CancelHandle + where + F: FnOnce(io::Result, User<[u8]>) + Send + 'static, + { + let mut read_buf = ManuallyDrop::new(MakeSend::new(read_buf)); + let ptr = read_buf.as_mut_ptr(); + let len = read_buf.len(); + let cb = move |res: io::Result| { + let read_buf = ManuallyDrop::into_inner(read_buf).into_inner(); + callback(res, read_buf); + }; + unsafe { self.raw_read(fd, ptr, len, Some(cb.into())) } + } + + /// Sends an asynchronous `write` usercall. `callback` is called when a + /// return value is received from userspace. `write_buf` is returned as an + /// argument to `callback` along with the result of the `write` usercall. + /// + /// Returns a handle that can be used to cancel the usercall if desired. + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn write(&self, fd: Fd, write_buf: UserBuf, callback: F) -> CancelHandle + where + F: FnOnce(io::Result, UserBuf) + Send + 'static, + { + let mut write_buf = ManuallyDrop::new(write_buf); + let ptr = write_buf.as_mut_ptr(); + let len = write_buf.len(); + let cb = move |res| { + let write_buf = ManuallyDrop::into_inner(write_buf); + callback(res, write_buf); + }; + unsafe { self.raw_write(fd, ptr, len, Some(cb.into())) } + } + + /// Sends an asynchronous `flush` usercall. `callback` is called when a + /// return value is received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn flush(&self, fd: Fd, callback: F) + where + F: FnOnce(io::Result<()>) + Send + 'static, + { + unsafe { + self.raw_flush(fd, Some(callback.into())); + } + } + + /// Sends an asynchronous `close` usercall. If specified, `callback` is + /// called when a return is received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn close(&self, fd: Fd, callback: Option) + where + F: FnOnce() + Send + 'static, + { + let cb = callback.map(|callback| move |()| callback()); + unsafe { + self.raw_close(fd, cb.map(Into::into)); + } + } + + /// Sends an asynchronous `bind_stream` usercall. `callback` is called when + /// a return value is received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn bind_stream(&self, addr: &str, callback: F) + where + F: FnOnce(io::Result) + Send + 'static, + { + let mut addr_buf = ManuallyDrop::new(MakeSend::new(User::<[u8]>::uninitialized(addr.len()))); + let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + + addr_buf[0..addr.len()].copy_from_enclave(addr.as_bytes()); + let addr_buf_ptr = addr_buf.as_raw_mut_ptr() as *mut u8; + let local_addr_ptr = local_addr.as_raw_mut_ptr(); + + let cb = move |res: io::Result| { + let _addr_buf = ManuallyDrop::into_inner(addr_buf); + let local_addr = ManuallyDrop::into_inner(local_addr); + + let local = string_from_bytebuffer(&local_addr, "bind_stream", "local_addr"); + let res = res.map(|fd| unsafe { new_std_listener(fd, Some(local)) }); + callback(res); + }; + unsafe { self.raw_bind_stream(addr_buf_ptr, addr.len(), local_addr_ptr, Some(cb.into())) } + } + + /// Sends an asynchronous `accept_stream` usercall. `callback` is called + /// when a return value is received from userspace. + /// + /// Returns a handle that can be used to cancel the usercall if desired. + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn accept_stream(&self, fd: Fd, callback: F) -> CancelHandle + where + F: FnOnce(io::Result) + Send + 'static, + { + let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut peer_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + + let local_addr_ptr = local_addr.as_raw_mut_ptr(); + let peer_addr_ptr = peer_addr.as_raw_mut_ptr(); + + let cb = move |res: io::Result| { + let local_addr = ManuallyDrop::into_inner(local_addr); + let peer_addr = ManuallyDrop::into_inner(peer_addr); + + let local = string_from_bytebuffer(&*local_addr, "accept_stream", "local_addr"); + let peer = string_from_bytebuffer(&*peer_addr, "accept_stream", "peer_addr"); + let res = res.map(|fd| unsafe { new_std_stream(fd, Some(local), Some(peer)) }); + callback(res); + }; + unsafe { self.raw_accept_stream(fd, local_addr_ptr, peer_addr_ptr, Some(cb.into())) } + } + + /// Sends an asynchronous `connect_stream` usercall. `callback` is called + /// when a return value is received from userspace. + /// + /// Returns a handle that can be used to cancel the usercall if desired. + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn connect_stream(&self, addr: &str, callback: F) -> CancelHandle + where + F: FnOnce(io::Result) + Send + 'static, + { + let mut addr_buf = ManuallyDrop::new(MakeSend::new(User::<[u8]>::uninitialized(addr.len()))); + let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut peer_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + + addr_buf[0..addr.len()].copy_from_enclave(addr.as_bytes()); + let addr_buf_ptr = addr_buf.as_raw_mut_ptr() as *mut u8; + let local_addr_ptr = local_addr.as_raw_mut_ptr(); + let peer_addr_ptr = peer_addr.as_raw_mut_ptr(); + + let cb = move |res: io::Result| { + let _addr_buf = ManuallyDrop::into_inner(addr_buf); + let local_addr = ManuallyDrop::into_inner(local_addr); + let peer_addr = ManuallyDrop::into_inner(peer_addr); + + let local = string_from_bytebuffer(&local_addr, "connect_stream", "local_addr"); + let peer = string_from_bytebuffer(&peer_addr, "connect_stream", "peer_addr"); + let res = res.map(|fd| unsafe { new_std_stream(fd, Some(local), Some(peer)) }); + callback(res); + }; + unsafe { self.raw_connect_stream(addr_buf_ptr, addr.len(), local_addr_ptr, peer_addr_ptr, Some(cb.into())) } + } + + /// Sends an asynchronous `alloc` usercall to allocate one instance of `T` + /// in userspace. `callback` is called when a return value is received from + /// userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn alloc(&self, callback: F) + where + T: UserSafe, + F: FnOnce(io::Result>) + Send + 'static, + { + let cb = move |res: io::Result<*mut u8>| { + let res = res.map(|ptr| unsafe { User::::from_raw(ptr as _) }); + callback(res); + }; + unsafe { + self.raw_alloc(mem::size_of::(), T::align_of(), Some(cb.into())); + } + } + + /// Sends an asynchronous `alloc` usercall to allocate a slice of `T` in + /// userspace with the specified `len`. `callback` is called when a return + /// value is received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn alloc_slice(&self, len: usize, callback: F) + where + [T]: UserSafe, + F: FnOnce(io::Result>) + Send + 'static, + { + let cb = move |res: io::Result<*mut u8>| { + let res = res.map(|ptr| unsafe { User::<[T]>::from_raw_parts(ptr as _, len) }); + callback(res); + }; + unsafe { + self.raw_alloc(len * mem::size_of::(), <[T]>::align_of(), Some(cb.into())); + } + } + + /// Sends an asynchronous `free` usercall to deallocate the userspace + /// buffer `buf`. If specified, `callback` is called when a return is + /// received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn free(&self, mut buf: User, callback: Option) + where + T: ?Sized + UserSafe, + F: FnOnce() + Send + 'static, + { + let ptr = buf.as_raw_mut_ptr(); + let cb = callback.map(|callback| move |()| callback()); + unsafe { + self.raw_free( + buf.into_raw() as _, + mem::size_of_val(&mut *ptr), + T::align_of(), + cb.map(Into::into), + ); + } + } + + /// Sends an asynchronous `insecure_time` usercall. `callback` is called + /// when a return value is received from userspace. + /// + /// Please refer to the type-level documentation for general notes about + /// callbacks. + pub fn insecure_time(&self, callback: F) + where + F: FnOnce(SystemTime) + Send + 'static, + { + let cb = move |nanos_since_epoch| { + let t = UNIX_EPOCH + Duration::from_nanos(nanos_since_epoch); + callback(t); + }; + unsafe { + self.raw_insecure_time(Some(cb.into())); + } + } +} + +fn string_from_bytebuffer(buf: &UserRef, usercall: &str, arg: &str) -> String { + String::from_utf8(copy_user_buffer(buf)) + .unwrap_or_else(|_| panic!("Usercall {}: expected {} to be valid UTF-8", usercall, arg)) +} + +// adapted from libstd sys/sgx/abi/usercalls/alloc.rs +fn copy_user_buffer(buf: &UserRef) -> Vec { + unsafe { + let buf = buf.to_enclave(); + if buf.len > 0 { + let user = User::from_raw_parts(buf.data as _, buf.len); + let v = user.to_enclave(); + batch_drop(user); + v + } else { + // Mustn't look at `data` or call `free` if `len` is `0`. + Vec::new() + } + } +} diff --git a/async-usercalls/src/provider_core.rs b/async-usercalls/src/provider_core.rs new file mode 100644 index 00000000..5b027c16 --- /dev/null +++ b/async-usercalls/src/provider_core.rs @@ -0,0 +1,66 @@ +use crate::hacks::{Cancel, Return, Usercall}; +use crate::queues::*; +use crate::CancelHandle; +use crossbeam_channel as mpmc; +use ipc_queue::Identified; +use std::sync::atomic::{AtomicU32, Ordering}; + +pub(crate) struct ProviderCore { + provider_id: u32, + next_id: AtomicU32, +} + +impl ProviderCore { + pub fn new(return_tx: Option>>) -> ProviderCore { + let provider_id = PROVIDERS.new_provider(return_tx); + ProviderCore { + provider_id, + next_id: AtomicU32::new(1), + } + } + + #[cfg(test)] + pub fn provider_id(&self) -> u32 { + self.provider_id + } + + fn next_id(&self) -> u32 { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + match id { + 0 => self.next_id(), + _ => id, + } + } + + pub fn assign_id(&self, usercall: Usercall) -> Identified { + let id = self.next_id(); + Identified { + id: ((self.provider_id as u64) << 32) | id as u64, + data: usercall, + } + } + + pub fn send_usercall(&self, usercall: Identified) -> CancelHandle { + assert!(usercall.id != 0); + let cancel = Identified { + id: usercall.id, + data: Cancel, + }; + PROVIDERS + .usercall_sender() + .send(usercall) + .expect("failed to send async usercall"); + CancelHandle::new(cancel) + } + + // returns the number of usercalls successfully sent. + pub fn try_send_multiple_usercalls(&self, usercalls: &[Identified]) -> usize { + PROVIDERS.usercall_sender().try_send_multiple(usercalls).unwrap_or(0) + } +} + +impl Drop for ProviderCore { + fn drop(&mut self) { + PROVIDERS.remove_provider(self.provider_id); + } +} diff --git a/async-usercalls/src/queues.rs b/async-usercalls/src/queues.rs new file mode 100644 index 00000000..fc7bbd07 --- /dev/null +++ b/async-usercalls/src/queues.rs @@ -0,0 +1,188 @@ +use crate::hacks::{alloc_descriptor, async_queues, to_enclave, Cancel, Return, Usercall}; +use crossbeam_channel as mpmc; +use fortanix_sgx_abi::{EV_CANCELQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY, EV_USERCALLQ_NOT_FULL}; +use ipc_queue::{self, Identified, QueueEvent, RecvError, SynchronizationError, Synchronizer}; +use lazy_static::lazy_static; +use std::os::fortanix_sgx::usercalls::raw; +use std::sync::{Arc, Mutex}; +use std::{io, iter, thread}; + +pub(crate) type Sender = ipc_queue::Sender; +pub(crate) type Receiver = ipc_queue::Receiver; + +pub(crate) struct Providers { + usercall_queue_tx: Sender, + cancel_queue_tx: Sender, + provider_map: Arc>>>>>, +} + +impl Providers { + pub(crate) fn new_provider(&self, return_tx: Option>>) -> u32 { + self.provider_map.lock().unwrap().insert(return_tx) + } + + pub(crate) fn remove_provider(&self, id: u32) { + let entry = self.provider_map.lock().unwrap().remove(id); + assert!(entry.is_some()); + } + + pub(crate) fn usercall_sender(&self) -> &Sender { + &self.usercall_queue_tx + } + + pub(crate) fn cancel_sender(&self) -> &Sender { + &self.cancel_queue_tx + } +} + +lazy_static! { + pub(crate) static ref PROVIDERS: Providers = { + let (utx, ctx, rx) = init_async_queues().expect("Failed to initialize async queues"); + let provider_map = Arc::new(Mutex::new(Map::new())); + let return_handler = ReturnHandler { + return_queue_rx: rx, + provider_map: Arc::clone(&provider_map), + }; + thread::spawn(move || return_handler.run()); + Providers { + usercall_queue_tx: utx, + cancel_queue_tx: ctx, + provider_map, + } + }; +} + +fn init_async_queues() -> io::Result<(Sender, Sender, Receiver)> { + // FIXME: this is just a hack. Replace these with `User::>::uninitialized().into_raw()` + let usercall_q = unsafe { alloc_descriptor::() }; + let cancel_q = unsafe { alloc_descriptor::() }; + let return_q = unsafe { alloc_descriptor::() }; + + let r = unsafe { async_queues(usercall_q, return_q, cancel_q) }; + if r != 0 { + return Err(io::Error::from_raw_os_error(r)); + } + + // FIXME: this is another hack, replace with `unsafe { User::>::from_raw(q) }.to_enclave()` + let usercall_queue = unsafe { to_enclave(usercall_q) }; + let cancel_queue = unsafe { to_enclave(cancel_q) }; + let return_queue = unsafe { to_enclave(return_q) }; + + let utx = unsafe { Sender::from_descriptor(usercall_queue, QueueSynchronizer { queue: Queue::Usercall }) }; + let ctx = unsafe { Sender::from_descriptor(cancel_queue, QueueSynchronizer { queue: Queue::Cancel }) }; + let rx = unsafe { Receiver::from_descriptor(return_queue, QueueSynchronizer { queue: Queue::Return }) }; + Ok((utx, ctx, rx)) +} + +struct ReturnHandler { + return_queue_rx: Receiver, + provider_map: Arc>>>>>, +} + +impl ReturnHandler { + const RECV_BATCH_SIZE: usize = 1024; + + fn send(&self, returns: &[Identified]) { + // This should hold the lock only for a short amount of time + // since mpmc::Sender::send() will not block (unbounded channel). + // Also note that the lock is uncontested most of the time, so + // taking the lock should be fast. + let provider_map = self.provider_map.lock().unwrap(); + for ret in returns { + let provider_id = (ret.id >> 32) as u32; + // NOTE: some providers might decide not to receive results of usercalls they send + // because the results are not interesting, e.g. BatchDropProvider. + if let Some(sender) = provider_map.get(provider_id).and_then(|entry| entry.as_ref()) { + let _ = sender.send(*ret); + } + } + } + + fn run(self) { + let mut returns = [Identified::default(); Self::RECV_BATCH_SIZE]; + loop { + let first = match self.return_queue_rx.recv() { + Ok(ret) => ret, + Err(RecvError::Closed) => break, + }; + let mut count = 0; + for ret in iter::once(first).chain(self.return_queue_rx.try_iter().take(Self::RECV_BATCH_SIZE - 1)) { + assert!(ret.id != 0); + returns[count] = ret; + count += 1; + } + self.send(&returns[..count]); + } + } +} + +#[derive(Clone, Copy, Debug)] +enum Queue { + Usercall, + Return, + Cancel, +} + +#[derive(Clone, Debug)] +pub(crate) struct QueueSynchronizer { + queue: Queue, +} + +impl Synchronizer for QueueSynchronizer { + fn wait(&self, event: QueueEvent) -> Result<(), SynchronizationError> { + let ev = match (self.queue, event) { + (Queue::Usercall, QueueEvent::NotEmpty) => panic!("enclave should not recv on usercall queue"), + (Queue::Cancel, QueueEvent::NotEmpty) => panic!("enclave should not recv on cancel queue"), + (Queue::Return, QueueEvent::NotFull) => panic!("enclave should not send on return queue"), + (Queue::Usercall, QueueEvent::NotFull) => EV_USERCALLQ_NOT_FULL, + (Queue::Cancel, QueueEvent::NotFull) => EV_CANCELQ_NOT_FULL, + (Queue::Return, QueueEvent::NotEmpty) => EV_RETURNQ_NOT_EMPTY, + }; + unsafe { + raw::wait(ev, raw::WAIT_INDEFINITE); + } + Ok(()) + } + + fn notify(&self, _event: QueueEvent) { + // any synchronous usercall would do + unsafe { + raw::wait(0, raw::WAIT_NO); + } + } +} + +use self::map::Map; +mod map { + use fnv::FnvHashMap; + + pub struct Map { + map: FnvHashMap, + next_id: u32, + } + + impl Map { + pub fn new() -> Self { + Self { + map: FnvHashMap::with_capacity_and_hasher(16, Default::default()), + next_id: 0, + } + } + + pub fn insert(&mut self, value: T) -> u32 { + let id = self.next_id; + self.next_id += 1; + let old = self.map.insert(id, value); + debug_assert!(old.is_none()); + id + } + + pub fn get(&self, id: u32) -> Option<&T> { + self.map.get(&id) + } + + pub fn remove(&mut self, id: u32) -> Option { + self.map.remove(&id) + } + } +} diff --git a/async-usercalls/src/raw.rs b/async-usercalls/src/raw.rs new file mode 100644 index 00000000..7edaa7eb --- /dev/null +++ b/async-usercalls/src/raw.rs @@ -0,0 +1,244 @@ +use crate::callback::*; +use crate::hacks::Usercall; +use crate::{AsyncUsercallProvider, CancelHandle}; +use fortanix_sgx_abi::Fd; +use std::io; +use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; +use std::os::fortanix_sgx::usercalls::raw::UsercallNrs; + +pub trait RawApi { + unsafe fn raw_read( + &self, + fd: Fd, + buf: *mut u8, + len: usize, + callback: Option>>, + ) -> CancelHandle; + + unsafe fn raw_write( + &self, + fd: Fd, + buf: *const u8, + len: usize, + callback: Option>>, + ) -> CancelHandle; + + unsafe fn raw_flush(&self, fd: Fd, callback: Option>>); + + unsafe fn raw_close(&self, fd: Fd, callback: Option>); + + unsafe fn raw_bind_stream( + &self, + addr: *const u8, + len: usize, + local_addr: *mut ByteBuffer, + callback: Option>>, + ); + + unsafe fn raw_accept_stream( + &self, + fd: Fd, + local_addr: *mut ByteBuffer, + peer_addr: *mut ByteBuffer, + callback: Option>>, + ) -> CancelHandle; + + unsafe fn raw_connect_stream( + &self, + addr: *const u8, + len: usize, + local_addr: *mut ByteBuffer, + peer_addr: *mut ByteBuffer, + callback: Option>>, + ) -> CancelHandle; + + unsafe fn raw_insecure_time(&self, callback: Option>); + + unsafe fn raw_alloc(&self, size: usize, alignment: usize, callback: Option>>); + + unsafe fn raw_free(&self, ptr: *mut u8, size: usize, alignment: usize, callback: Option>); +} + +impl RawApi for AsyncUsercallProvider { + unsafe fn raw_read( + &self, + fd: Fd, + buf: *mut u8, + len: usize, + callback: Option>>, + ) -> CancelHandle { + let u = Usercall(UsercallNrs::read as _, fd as _, buf as _, len as _, 0); + self.send_usercall(u, callback.map(|cb| Callback::read(cb))) + } + + unsafe fn raw_write( + &self, + fd: Fd, + buf: *const u8, + len: usize, + callback: Option>>, + ) -> CancelHandle { + let u = Usercall(UsercallNrs::write as _, fd as _, buf as _, len as _, 0); + self.send_usercall(u, callback.map(|cb| Callback::write(cb))) + } + + unsafe fn raw_flush(&self, fd: Fd, callback: Option>>) { + let u = Usercall(UsercallNrs::flush as _, fd as _, 0, 0, 0); + self.send_usercall(u, callback.map(|cb| Callback::flush(cb))); + } + + unsafe fn raw_close(&self, fd: Fd, callback: Option>) { + let u = Usercall(UsercallNrs::close as _, fd as _, 0, 0, 0); + self.send_usercall(u, callback.map(|cb| Callback::close(cb))); + } + + unsafe fn raw_bind_stream( + &self, + addr: *const u8, + len: usize, + local_addr: *mut ByteBuffer, + callback: Option>>, + ) { + let u = Usercall(UsercallNrs::bind_stream as _, addr as _, len as _, local_addr as _, 0); + self.send_usercall(u, callback.map(|cb| Callback::bind_stream(cb))); + } + + unsafe fn raw_accept_stream( + &self, + fd: Fd, + local_addr: *mut ByteBuffer, + peer_addr: *mut ByteBuffer, + callback: Option>>, + ) -> CancelHandle { + let u = Usercall( + UsercallNrs::accept_stream as _, + fd as _, + local_addr as _, + peer_addr as _, + 0, + ); + self.send_usercall(u, callback.map(|cb| Callback::accept_stream(cb))) + } + + unsafe fn raw_connect_stream( + &self, + addr: *const u8, + len: usize, + local_addr: *mut ByteBuffer, + peer_addr: *mut ByteBuffer, + callback: Option>>, + ) -> CancelHandle { + let u = Usercall( + UsercallNrs::connect_stream as _, + addr as _, + len as _, + local_addr as _, + peer_addr as _, + ); + self.send_usercall(u, callback.map(|cb| Callback::connect_stream(cb))) + } + + unsafe fn raw_insecure_time(&self, callback: Option>) { + let u = Usercall(UsercallNrs::insecure_time as _, 0, 0, 0, 0); + self.send_usercall(u, callback.map(|cb| Callback::insecure_time(cb))); + } + + unsafe fn raw_alloc(&self, size: usize, alignment: usize, callback: Option>>) { + let u = Usercall(UsercallNrs::alloc as _, size as _, alignment as _, 0, 0); + self.send_usercall(u, callback.map(|cb| Callback::alloc(cb))); + } + + unsafe fn raw_free(&self, ptr: *mut u8, size: usize, alignment: usize, callback: Option>) { + let u = Usercall(UsercallNrs::free as _, ptr as _, size as _, alignment as _, 0); + self.send_usercall(u, callback.map(|cb| Callback::free(cb))); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::*; + use crossbeam_channel as mpmc; + use std::io; + use std::sync::atomic::{AtomicPtr, Ordering}; + use std::sync::Arc; + use std::thread; + use std::time::{Duration, UNIX_EPOCH}; + + #[test] + fn get_time_async_raw() { + fn run(tid: u32, provider: AutoPollingProvider) -> (u32, u32, Duration) { + let pid = provider.provider_id(); + const N: usize = 500; + let (tx, rx) = mpmc::bounded(N); + for _ in 0..N { + let tx = tx.clone(); + let cb = move |d| { + let system_time = UNIX_EPOCH + Duration::from_nanos(d); + tx.send(system_time).unwrap(); + }; + unsafe { + provider.raw_insecure_time(Some(cb.into())); + } + } + let mut all = Vec::with_capacity(N); + for _ in 0..N { + all.push(rx.recv().unwrap()); + } + + assert_eq!(all.len(), N); + // The results are returned in arbitrary order + all.sort(); + let t0 = *all.first().unwrap(); + let tn = *all.last().unwrap(); + let total = tn.duration_since(t0).unwrap(); + (tid, pid, total / N as u32) + } + + println!(); + const THREADS: usize = 4; + let mut providers = Vec::with_capacity(THREADS); + for _ in 0..THREADS { + providers.push(AutoPollingProvider::new()); + } + let mut handles = Vec::with_capacity(THREADS); + for (i, provider) in providers.into_iter().enumerate() { + handles.push(thread::spawn(move || run(i as u32, provider))); + } + for h in handles { + let res = h.join().unwrap(); + println!("[{}/{}] (Tn - T0) / N = {:?}", res.0, res.1, res.2); + } + } + + #[test] + fn raw_alloc_free() { + let provider = AutoPollingProvider::new(); + let ptr: Arc> = Arc::new(AtomicPtr::new(0 as _)); + let ptr2 = Arc::clone(&ptr); + const SIZE: usize = 1024; + const ALIGN: usize = 8; + + let (tx, rx) = mpmc::bounded(1); + let cb_alloc = move |p: io::Result<*mut u8>| { + let p = p.unwrap(); + ptr2.store(p, Ordering::Relaxed); + tx.send(()).unwrap(); + }; + unsafe { + provider.raw_alloc(SIZE, ALIGN, Some(cb_alloc.into())); + } + rx.recv().unwrap(); + let p = ptr.load(Ordering::Relaxed); + assert!(!p.is_null()); + + let (tx, rx) = mpmc::bounded(1); + let cb_free = move |()| { + tx.send(()).unwrap(); + }; + unsafe { + provider.raw_free(p, SIZE, ALIGN, Some(cb_free.into())); + } + rx.recv().unwrap(); + } +} diff --git a/async-usercalls/src/test_support.rs b/async-usercalls/src/test_support.rs new file mode 100644 index 00000000..fa3b75bd --- /dev/null +++ b/async-usercalls/src/test_support.rs @@ -0,0 +1,47 @@ +use crate::AsyncUsercallProvider; +use std::ops::Deref; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::thread; + +pub(crate) struct AutoPollingProvider { + provider: AsyncUsercallProvider, + shutdown: Arc, + join_handle: Option>, +} + +impl AutoPollingProvider { + pub fn new() -> Self { + let (provider, handler) = AsyncUsercallProvider::new(); + let shutdown = Arc::new(AtomicBool::new(false)); + let shutdown1 = shutdown.clone(); + let join_handle = Some(thread::spawn(move || loop { + handler.poll(None); + if shutdown1.load(Ordering::Relaxed) { + break; + } + })); + Self { + provider, + shutdown, + join_handle, + } + } +} + +impl Deref for AutoPollingProvider { + type Target = AsyncUsercallProvider; + + fn deref(&self) -> &Self::Target { + &self.provider + } +} + +impl Drop for AutoPollingProvider { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::Relaxed); + // send a usercall to ensure thread wakes up + self.provider.insecure_time(|_| {}); + self.join_handle.take().unwrap().join().unwrap(); + } +} diff --git a/async-usercalls/test.sh b/async-usercalls/test.sh new file mode 100755 index 00000000..cdb85673 --- /dev/null +++ b/async-usercalls/test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Run this in parallel with: +# $ cargo test --target x86_64-fortanix-unknown-sgx --release -- --nocapture --ignored echo + +for i in $(seq 1 100); do + echo $i + telnet localhost 7799 < /dev/zero &> /dev/null & + sleep 0.01 +done + +sleep 10s +kill $(jobs -p) +wait From be9fd6483687039af21b41f2b1349ddc42452e03 Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Mon, 18 Oct 2021 18:37:58 -0700 Subject: [PATCH 03/19] Fix race condition when cancel is received before usercall `UsercallEvent::Start` was being sent in `fn handle_usercall`, which is too late. It needs to be sent before we receive the next usercall from the enclave so we can maintain the invariant that "we only need to keep track of cancels received before the actual usercall if the read position has not moved past the write position when cancel was received." --- intel-sgx/enclave-runner/src/usercalls/mod.rs | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/intel-sgx/enclave-runner/src/usercalls/mod.rs b/intel-sgx/enclave-runner/src/usercalls/mod.rs index 9a9d0967..3ffcb787 100644 --- a/intel-sgx/enclave-runner/src/usercalls/mod.rs +++ b/intel-sgx/enclave-runner/src/usercalls/mod.rs @@ -29,9 +29,7 @@ use libc::*; use nix::sys::signal; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::stream::Stream as TokioStream; -use tokio::sync::broadcast; -use tokio::sync::mpsc as async_mpsc; -use tokio::sync::Semaphore; +use tokio::sync::{broadcast, mpsc as async_mpsc, oneshot, Semaphore}; use fortanix_sgx_abi::*; use ipc_queue::{self, DescriptorGuard, Identified, QueueEvent, WritePosition}; @@ -57,13 +55,13 @@ const CANCEL_QUEUE_SIZE: usize = USERCALL_QUEUE_SIZE * 2; enum UsercallSendData { Sync(ThreadResult, RunningTcs, RefCell<[u8; 1024]>), - Async(Identified), + Async(Identified, Option>), } // This is the same as UsercallSendData except that it can't be Sync(CoResult::Return(...), ...) enum UsercallHandleData { Sync(tcs::Usercall, RunningTcs, RefCell<[u8; 1024]>), - Async(Identified, Option>), + Async(Identified, Option>, Option>), } type EnclaveResult = StdResult<(u64, u64), EnclaveAbort>>; @@ -689,7 +687,7 @@ impl Work { } enum UsercallEvent { - Started(u64, tokio::sync::oneshot::Sender<()>), + Started(u64, oneshot::Sender<()>), Finished(u64), Cancelled(u64, WritePosition), } @@ -780,17 +778,12 @@ impl EnclaveState { mut handle_data: UsercallHandleData, ) { let notifier_rx = match handle_data { - UsercallHandleData::Async(ref usercall, Some(ref usercall_event_tx)) => { - let (notifier_tx, notifier_rx) = tokio::sync::oneshot::channel(); - usercall_event_tx.send(UsercallEvent::Started(usercall.id, notifier_tx)).ok() - .expect("failed to send usercall event"); - Some(notifier_rx) - }, + UsercallHandleData::Async(_, ref mut notifier_rx, _) => notifier_rx.take(), _ => None, }; let (parameters, mode, tcs) = match handle_data { UsercallHandleData::Sync(ref usercall, ref mut tcs, _) => (usercall.parameters(), tcs.mode.into(), Some(tcs)), - UsercallHandleData::Async(ref usercall, _) => (usercall.data.into(), ReturnSource::AsyncUsercall, None), + UsercallHandleData::Async(ref usercall, _, _) => (usercall.data.into(), ReturnSource::AsyncUsercall, None), }; let mut input = IOHandlerInput { enclave: enclave.clone(), tcs, work_sender: &work_sender }; let handler = Handler(&mut input); @@ -824,7 +817,7 @@ impl EnclaveState { entry: CoEntry::Resume(usercall, ret), }).expect("Work sender couldn't send data to receiver"); } - UsercallHandleData::Async(usercall, usercall_event_tx) => { + UsercallHandleData::Async(usercall, _, usercall_event_tx) => { if let Some(usercall_event_tx) = usercall_event_tx { usercall_event_tx.send(UsercallEvent::Finished(usercall.id)).ok() .expect("failed to send usercall event"); @@ -849,7 +842,7 @@ impl EnclaveState { } EnclavePanic::from(debug_buf) } - UsercallHandleData::Async(_, _) => { + UsercallHandleData::Async(_, _, _) => { // TODO: https://github.com/fortanix/rust-sgx/issues/235#issuecomment-641811437 EnclavePanic::DebugStr("async exit with a panic".to_owned()) } @@ -945,13 +938,21 @@ impl EnclaveState { let usercall_queue_monitor = usercall_queue_rx.position_monitor(); + let (usercall_event_tx, mut usercall_event_rx) = async_mpsc::unbounded_channel(); + let usercall_event_tx_clone = usercall_event_tx.clone(); tokio::task::spawn_local(async move { while let Ok(usercall) = usercall_queue_rx.recv().await { - let _ = io_queue_send.send(UsercallSendData::Async(usercall)); + let notifier_rx = if usercall.ignore_cancel() { + None + } else { + let (notifier_tx, notifier_rx) = oneshot::channel(); + usercall_event_tx_clone.send(UsercallEvent::Started(usercall.id, notifier_tx)).ok().expect("failed to send usercall event"); + Some(notifier_rx) + }; + let _ = io_queue_send.send(UsercallSendData::Async(usercall, notifier_rx)); } }); - let (usercall_event_tx, mut usercall_event_rx) = async_mpsc::unbounded_channel(); let usercall_event_tx_clone = usercall_event_tx.clone(); let usercall_queue_monitor_clone = usercall_queue_monitor.clone(); tokio::task::spawn_local(async move { @@ -988,9 +989,9 @@ impl EnclaveState { let enclave_clone = enclave_clone.clone(); let tx_return_channel = tx_return_channel.clone(); match work { - UsercallSendData::Async(usercall) => { + UsercallSendData::Async(usercall, notifier_rx) => { let usercall_event_tx = if usercall.ignore_cancel() { None } else { Some(usercall_event_tx.clone()) }; - let uchd = UsercallHandleData::Async(usercall, usercall_event_tx); + let uchd = UsercallHandleData::Async(usercall, notifier_rx, usercall_event_tx); let fut = Self::handle_usercall(enclave_clone, work_sender.clone(), tx_return_channel, uchd); tokio::task::spawn_local(fut); } From 104d151f8c7eb95b3ab8609854137f011a1fbc8a Mon Sep 17 00:00:00 2001 From: Raoul Strackx Date: Mon, 9 May 2022 10:02:03 +0200 Subject: [PATCH 04/19] Verify allocation `FifoDescriptor` --- async-usercalls/src/hacks/async_queues.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/async-usercalls/src/hacks/async_queues.rs b/async-usercalls/src/hacks/async_queues.rs index a325a28f..5e7a9252 100644 --- a/async-usercalls/src/hacks/async_queues.rs +++ b/async-usercalls/src/hacks/async_queues.rs @@ -4,6 +4,7 @@ use fortanix_sgx_abi::FifoDescriptor; use std::num::NonZeroU64; use std::os::fortanix_sgx::usercalls; use std::os::fortanix_sgx::usercalls::raw; +use std::os::fortanix_sgx::usercalls::alloc::{UserSafeSized, User}; use std::{mem, ptr}; // TODO: remove these once support for cancel queue is added in `std::os::fortanix_sgx` @@ -26,12 +27,13 @@ pub unsafe fn async_queues( ) } -pub unsafe fn alloc_descriptor() -> *mut FifoDescriptor { - usercalls::alloc( - mem::size_of::>(), - mem::align_of::>(), - ) - .expect("failed to allocate userspace memory") as _ +pub unsafe fn alloc_descriptor() -> *mut FifoDescriptor { + #[repr(transparent)] + #[derive(Copy, Clone)] + struct WrappedFifoDescriptor(FifoDescriptor); + unsafe impl UserSafeSized for WrappedFifoDescriptor{} + + User::>::uninitialized().into_raw() as _ } pub unsafe fn to_enclave(ptr: *mut FifoDescriptor) -> FifoDescriptor { From c8c68d9e436843a920ede00d336209295918dda2 Mon Sep 17 00:00:00 2001 From: Raoul Strackx Date: Wed, 1 Jun 2022 13:56:59 +0200 Subject: [PATCH 05/19] Making `read_epoch` `AtomicU64` --- ipc-queue/src/fifo.rs | 2 +- ipc-queue/src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ipc-queue/src/fifo.rs b/ipc-queue/src/fifo.rs index d9ca36ba..d834673a 100644 --- a/ipc-queue/src/fifo.rs +++ b/ipc-queue/src/fifo.rs @@ -68,7 +68,7 @@ where let arc = Arc::new(FifoBuffer::new(len)); let inner = Fifo::from_arc(arc); let tx = AsyncSender { inner: inner.clone(), synchronizer: s.clone() }; - let rx = AsyncReceiver { inner, synchronizer: s, read_epoch: Arc::new(AtomicU32::new(0)) }; + let rx = AsyncReceiver { inner, synchronizer: s, read_epoch: Arc::new(AtomicU64::new(0)) }; (tx, rx) } diff --git a/ipc-queue/src/lib.rs b/ipc-queue/src/lib.rs index 4cc31117..42fd7915 100644 --- a/ipc-queue/src/lib.rs +++ b/ipc-queue/src/lib.rs @@ -10,7 +10,7 @@ use std::future::Future; use std::pin::Pin; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::AtomicU64; use std::sync::Arc; use fortanix_sgx_abi::FifoDescriptor; @@ -154,7 +154,7 @@ pub struct AsyncSender { pub struct AsyncReceiver { inner: Fifo, synchronizer: S, - read_epoch: Arc, + read_epoch: Arc, } /// `DescriptorGuard` can produce a `FifoDescriptor` that is guaranteed @@ -177,7 +177,7 @@ impl DescriptorGuard { /// read to/from the queue. This is useful in case we want to know whether or /// not a particular value written to the queue has been read. pub struct PositionMonitor { - read_epoch: Arc, + read_epoch: Arc, fifo: Fifo, } From 6bd6f343d8704638e71e3f9b7127525de819bdfd Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Thu, 28 Jul 2022 11:50:18 -0700 Subject: [PATCH 06/19] Post rebase fixes - Move async-usercalls to intel-sgx directory - Remove hacks/unsafe_typecasts.rs - Fix some typos in docs - Use marker trait for MakeSend to avoid warnings about issue #93367 - Update crossbeam and crossbeam-channel dependencies - Use nightly Rust in CI - Use SGX target for generating docs when crate has `feature(sgx_platform)` --- .travis.yml | 2 +- Cargo.lock | 113 ++++++++++++------ async-usercalls/src/hacks/unsafe_typecasts.rs | 95 --------------- doc/generate-api-docs.sh | 3 + .../async-usercalls}/Cargo.toml | 4 +- .../async-usercalls}/src/batch_drop.rs | 0 .../async-usercalls}/src/callback.rs | 0 .../async-usercalls}/src/duplicated.rs | 0 .../src/hacks/async_queues.rs | 0 .../async-usercalls}/src/hacks/mod.rs | 20 ++-- .../async-usercalls}/src/io_bufs.rs | 4 +- .../async-usercalls}/src/lib.rs | 0 .../async-usercalls}/src/provider_api.rs | 49 ++++---- .../async-usercalls}/src/provider_core.rs | 0 .../async-usercalls}/src/queues.rs | 0 .../async-usercalls}/src/raw.rs | 0 .../async-usercalls}/src/test_support.rs | 0 .../async-usercalls}/test.sh | 0 intel-sgx/enclave-runner/Cargo.toml | 2 +- intel-sgx/enclave-runner/src/usercalls/mod.rs | 16 +-- ipc-queue/src/fifo.rs | 2 +- 21 files changed, 132 insertions(+), 178 deletions(-) delete mode 100644 async-usercalls/src/hacks/unsafe_typecasts.rs rename {async-usercalls => intel-sgx/async-usercalls}/Cargo.toml (89%) rename {async-usercalls => intel-sgx/async-usercalls}/src/batch_drop.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/callback.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/duplicated.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/hacks/async_queues.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/hacks/mod.rs (71%) rename {async-usercalls => intel-sgx/async-usercalls}/src/io_bufs.rs (98%) rename {async-usercalls => intel-sgx/async-usercalls}/src/lib.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/provider_api.rs (80%) rename {async-usercalls => intel-sgx/async-usercalls}/src/provider_core.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/queues.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/raw.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/src/test_support.rs (100%) rename {async-usercalls => intel-sgx/async-usercalls}/test.sh (100%) diff --git a/.travis.yml b/.travis.yml index 68df86a3..52693a75 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: - clang-11 - musl-tools rust: - - stable + - nightly env: - RUST_BACKTRACE=1 - CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" diff --git a/Cargo.lock b/Cargo.lock index a908ab9f..284c8156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,7 +68,7 @@ dependencies = [ "fnv", "fortanix-sgx-abi", "ipc-queue", - "lazy_static 1.4.0", + "lazy_static", ] [[package]] @@ -517,26 +517,26 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.7.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "crossbeam-channel", - "crossbeam-deque", - "crossbeam-epoch", - "crossbeam-queue", - "crossbeam-utils", + "crossbeam-deque 0.8.2", + "crossbeam-epoch 0.9.10", + "crossbeam-queue 0.3.6", + "crossbeam-utils 0.8.11", ] [[package]] name = "crossbeam-channel" -version = "0.4.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" +checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" dependencies = [ - "crossbeam-utils", - "maybe-uninit", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", ] [[package]] @@ -545,11 +545,22 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", "maybe-uninit", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch 0.9.10", + "crossbeam-utils 0.8.11", +] + [[package]] name = "crossbeam-epoch" version = "0.8.2" @@ -558,13 +569,27 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg 1.0.1", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", "memoffset 0.5.6", "scopeguard", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", + "memoffset 0.6.4", + "once_cell", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.2.3" @@ -572,10 +597,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "maybe-uninit", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.11", +] + [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -587,6 +622,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", +] + [[package]] name = "crypto-hash" version = "0.3.4" @@ -918,8 +963,8 @@ dependencies = [ "sha2 0.9.8", "shiplift", "tempfile", - "tokio 1.14.0", - "url 2.2.2", + "tokio 1.16.1", + "url 2.1.1", ] [[package]] @@ -1597,7 +1642,7 @@ dependencies = [ "itoa 0.4.6", "pin-project-lite 0.2.7", "socket2", - "tokio 1.14.0", + "tokio 1.16.1", "tower-service", "tracing", "want 0.3.0", @@ -1616,7 +1661,7 @@ dependencies = [ "openssl", "openssl-sys", "parking_lot 0.11.2", - "tokio 1.14.0", + "tokio 1.16.1", "tokio-openssl", "tower-layer", ] @@ -1657,7 +1702,7 @@ dependencies = [ "hex 0.4.3", "hyper 0.14.15", "pin-project 1.0.8", - "tokio 1.14.0", + "tokio 1.16.1", ] [[package]] @@ -1741,7 +1786,7 @@ dependencies = [ "futures-core", "inotify-sys", "libc", - "tokio 1.14.0", + "tokio 1.16.1", ] [[package]] @@ -3637,8 +3682,8 @@ dependencies = [ "serde", "serde_json", "tar", - "tokio 1.14.0", - "url 2.2.2", + "tokio 1.16.1", + "url 2.1.1", ] [[package]] @@ -3917,12 +3962,10 @@ dependencies = [ [[package]] name = "tokio" -version = "1.14.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e992e41e0d2fb9f755b37446f20900f64446ef54874f40a60c78f021ac6144" +checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" dependencies = [ - "autocfg 1.0.1", - "bytes 1.1.0", "libc", "memchr", "mio 0.7.14", @@ -3959,7 +4002,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", ] @@ -4015,7 +4058,7 @@ dependencies = [ "futures-util", "openssl", "openssl-sys", - "tokio 1.14.0", + "tokio 1.16.1", ] [[package]] @@ -4024,7 +4067,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", "lazy_static", "log 0.4.14", @@ -4067,9 +4110,9 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" dependencies = [ - "crossbeam-deque", - "crossbeam-queue", - "crossbeam-utils", + "crossbeam-deque 0.7.3", + "crossbeam-queue 0.2.3", + "crossbeam-utils 0.7.2", "futures 0.1.30", "lazy_static", "log 0.4.14", @@ -4084,7 +4127,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" dependencies = [ - "crossbeam-utils", + "crossbeam-utils 0.7.2", "futures 0.1.30", "slab", "tokio-executor", diff --git a/async-usercalls/src/hacks/unsafe_typecasts.rs b/async-usercalls/src/hacks/unsafe_typecasts.rs deleted file mode 100644 index 1e3d67c5..00000000 --- a/async-usercalls/src/hacks/unsafe_typecasts.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! The incredibly unsafe code in this module allows us to create -//! `std::net::TcpStream` and `std::net::TcpListener` types from their raw -//! components in SGX. -//! -//! This is obviously very unsafe and not maintainable and is only intended as -//! an iterim solution until we add similar functionality as extension traits -//! in `std::os::fortanix_sgx`. -use fortanix_sgx_abi::Fd; - -mod sgx { - use fortanix_sgx_abi::Fd; - use std::sync::Arc; - - #[derive(Debug)] - pub struct FileDesc { - fd: Fd, - } - - #[derive(Debug, Clone)] - pub struct Socket { - inner: Arc, - local_addr: Option, - } - - #[derive(Clone)] - pub struct TcpStream { - inner: Socket, - peer_addr: Option, - } - - impl TcpStream { - pub fn new(fd: Fd, local_addr: Option, peer_addr: Option) -> TcpStream { - TcpStream { - inner: Socket { - inner: Arc::new(FileDesc { fd }), - local_addr, - }, - peer_addr, - } - } - } - - #[derive(Clone)] - pub struct TcpListener { - inner: Socket, - } - - impl TcpListener { - pub fn new(fd: Fd, local_addr: Option) -> TcpListener { - TcpListener { - inner: Socket { - inner: Arc::new(FileDesc { fd }), - local_addr, - }, - } - } - } -} - -struct TcpStream(self::sgx::TcpStream); -struct TcpListener(self::sgx::TcpListener); - -pub unsafe fn new_std_stream(fd: Fd, local_addr: Option, peer_addr: Option) -> std::net::TcpStream { - let stream = TcpStream(sgx::TcpStream::new(fd, local_addr, peer_addr)); - std::mem::transmute(stream) -} - -pub unsafe fn new_std_listener(fd: Fd, local_addr: Option) -> std::net::TcpListener { - let listener = TcpListener(sgx::TcpListener::new(fd, local_addr)); - std::mem::transmute(listener) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::mem; - use std::os::fortanix_sgx::io::AsRawFd; - - #[test] - fn sanity_check() { - let fd = 42; - let local = "1.2.3.4:1234"; - let peer = "5.6.7.8:443"; - let stream = unsafe { new_std_stream(fd, Some(local.to_owned()), Some(peer.to_owned())) }; - assert_eq!(stream.as_raw_fd(), fd); - assert_eq!(stream.local_addr().unwrap().to_string(), local); - assert_eq!(stream.peer_addr().unwrap().to_string(), peer); - mem::forget(stream); // not a real stream... - - let listener = unsafe { new_std_listener(fd, Some(local.to_owned())) }; - assert_eq!(listener.as_raw_fd(), fd); - assert_eq!(listener.local_addr().unwrap().to_string(), local); - mem::forget(listener); // not a real listener... - } -} diff --git a/doc/generate-api-docs.sh b/doc/generate-api-docs.sh index b25386ff..dcdcebb9 100755 --- a/doc/generate-api-docs.sh +++ b/doc/generate-api-docs.sh @@ -58,6 +58,9 @@ for LIB in $LIBS_SORTED; do if FEATURES="$(cargo read-manifest|jq -r '.metadata.docs.rs.features | join(",")' 2> /dev/null)"; then ARGS="--features $FEATURES" fi + if grep -q 'feature(sgx_platform)' ./src/lib.rs; then + ARGS="$ARGS --target x86_64-fortanix-unknown-sgx" + fi cargo doc --no-deps --lib $ARGS popd fi diff --git a/async-usercalls/Cargo.toml b/intel-sgx/async-usercalls/Cargo.toml similarity index 89% rename from async-usercalls/Cargo.toml rename to intel-sgx/async-usercalls/Cargo.toml index c0cd25a0..1efbc98f 100644 --- a/async-usercalls/Cargo.toml +++ b/intel-sgx/async-usercalls/Cargo.toml @@ -17,12 +17,12 @@ categories = ["asynchronous"] [dependencies] # Project dependencies -ipc-queue = { version = "0.1", path = "../ipc-queue" } +ipc-queue = { version = "0.2", path = "../../ipc-queue" } fortanix-sgx-abi = { version = "0.4", path = "../fortanix-sgx-abi" } # External dependencies lazy_static = "1.4.0" # MIT/Apache-2.0 -crossbeam-channel = "0.4" # MIT/Apache-2.0 +crossbeam-channel = "0.5" # MIT/Apache-2.0 fnv = "1.0" # MIT/Apache-2.0 # For cargo test --target x86_64-fortanix-unknown-sgx diff --git a/async-usercalls/src/batch_drop.rs b/intel-sgx/async-usercalls/src/batch_drop.rs similarity index 100% rename from async-usercalls/src/batch_drop.rs rename to intel-sgx/async-usercalls/src/batch_drop.rs diff --git a/async-usercalls/src/callback.rs b/intel-sgx/async-usercalls/src/callback.rs similarity index 100% rename from async-usercalls/src/callback.rs rename to intel-sgx/async-usercalls/src/callback.rs diff --git a/async-usercalls/src/duplicated.rs b/intel-sgx/async-usercalls/src/duplicated.rs similarity index 100% rename from async-usercalls/src/duplicated.rs rename to intel-sgx/async-usercalls/src/duplicated.rs diff --git a/async-usercalls/src/hacks/async_queues.rs b/intel-sgx/async-usercalls/src/hacks/async_queues.rs similarity index 100% rename from async-usercalls/src/hacks/async_queues.rs rename to intel-sgx/async-usercalls/src/hacks/async_queues.rs diff --git a/async-usercalls/src/hacks/mod.rs b/intel-sgx/async-usercalls/src/hacks/mod.rs similarity index 71% rename from async-usercalls/src/hacks/mod.rs rename to intel-sgx/async-usercalls/src/hacks/mod.rs index 9011c63e..5fe7a2df 100644 --- a/async-usercalls/src/hacks/mod.rs +++ b/intel-sgx/async-usercalls/src/hacks/mod.rs @@ -3,10 +3,8 @@ use std::os::fortanix_sgx::usercalls::alloc::{User, UserSafeSized}; use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; mod async_queues; -mod unsafe_typecasts; pub use self::async_queues::{alloc_descriptor, async_queues, to_enclave}; -pub use self::unsafe_typecasts::{new_std_listener, new_std_stream}; #[repr(C)] #[derive(Copy, Clone, Default)] @@ -26,10 +24,12 @@ pub struct Cancel; unsafe impl UserSafeSized for Cancel {} +pub(crate) trait MakeSendMarker {} + // Interim solution until we mark the target types appropriately -pub(crate) struct MakeSend(T); +pub(crate) struct MakeSend(T); -impl MakeSend { +impl MakeSend { pub fn new(t: T) -> Self { Self(t) } @@ -40,7 +40,7 @@ impl MakeSend { } } -impl Deref for MakeSend { +impl Deref for MakeSend { type Target = T; fn deref(&self) -> &Self::Target { @@ -48,12 +48,14 @@ impl Deref for MakeSend { } } -impl DerefMut for MakeSend { +impl DerefMut for MakeSend { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 } } -unsafe impl Send for MakeSend {} -unsafe impl Send for MakeSend> {} -unsafe impl Send for MakeSend> {} +unsafe impl Send for MakeSend {} + +impl MakeSendMarker for ByteBuffer {} +impl MakeSendMarker for User {} +impl MakeSendMarker for User<[u8]> {} diff --git a/async-usercalls/src/io_bufs.rs b/intel-sgx/async-usercalls/src/io_bufs.rs similarity index 98% rename from async-usercalls/src/io_bufs.rs rename to intel-sgx/async-usercalls/src/io_bufs.rs index a8ede0de..c86df75f 100644 --- a/async-usercalls/src/io_bufs.rs +++ b/intel-sgx/async-usercalls/src/io_bufs.rs @@ -166,9 +166,9 @@ impl WriteBuffer { assert!(n <= buf.len()); const PANIC_MESSAGE: &'static str = "`buf` not produced by self.consumable_chunk()"; let buf = buf.into_shared().expect(PANIC_MESSAGE); - assert!(Arc::ptr_eq(&self.userbuf, &buf), PANIC_MESSAGE); + assert!(Arc::ptr_eq(&self.userbuf, &buf), "{}", PANIC_MESSAGE); drop(buf); - assert!(Arc::strong_count(&self.userbuf) == 1, PANIC_MESSAGE); + assert!(Arc::strong_count(&self.userbuf) == 1, "{}", PANIC_MESSAGE); self.advance_read(n); } diff --git a/async-usercalls/src/lib.rs b/intel-sgx/async-usercalls/src/lib.rs similarity index 100% rename from async-usercalls/src/lib.rs rename to intel-sgx/async-usercalls/src/lib.rs diff --git a/async-usercalls/src/provider_api.rs b/intel-sgx/async-usercalls/src/provider_api.rs similarity index 80% rename from async-usercalls/src/provider_api.rs rename to intel-sgx/async-usercalls/src/provider_api.rs index 234dbddb..ca307b17 100644 --- a/async-usercalls/src/provider_api.rs +++ b/intel-sgx/async-usercalls/src/provider_api.rs @@ -1,5 +1,5 @@ use crate::batch_drop; -use crate::hacks::{new_std_listener, new_std_stream, MakeSend}; +use crate::hacks::MakeSend; use crate::io_bufs::UserBuf; use crate::raw::RawApi; use crate::{AsyncUsercallProvider, CancelHandle}; @@ -7,6 +7,7 @@ use fortanix_sgx_abi::Fd; use std::io; use std::mem::{self, ManuallyDrop}; use std::net::{TcpListener, TcpStream}; +use std::os::fortanix_sgx::io::{FromRawFd, TcpListenerMetadata, TcpStreamMetadata}; use std::os::fortanix_sgx::usercalls::alloc::{User, UserRef, UserSafe}; use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -93,18 +94,18 @@ impl AsyncUsercallProvider { F: FnOnce(io::Result) + Send + 'static, { let mut addr_buf = ManuallyDrop::new(MakeSend::new(User::<[u8]>::uninitialized(addr.len()))); - let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut local_addr_buf = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); addr_buf[0..addr.len()].copy_from_enclave(addr.as_bytes()); let addr_buf_ptr = addr_buf.as_raw_mut_ptr() as *mut u8; - let local_addr_ptr = local_addr.as_raw_mut_ptr(); + let local_addr_ptr = local_addr_buf.as_raw_mut_ptr(); let cb = move |res: io::Result| { let _addr_buf = ManuallyDrop::into_inner(addr_buf); - let local_addr = ManuallyDrop::into_inner(local_addr); + let local_addr_buf = ManuallyDrop::into_inner(local_addr_buf); - let local = string_from_bytebuffer(&local_addr, "bind_stream", "local_addr"); - let res = res.map(|fd| unsafe { new_std_listener(fd, Some(local)) }); + let local_addr = Some(string_from_bytebuffer(&local_addr_buf, "bind_stream", "local_addr")); + let res = res.map(|fd| unsafe { TcpListener::from_raw_fd(fd, TcpListenerMetadata { local_addr }) }); callback(res); }; unsafe { self.raw_bind_stream(addr_buf_ptr, addr.len(), local_addr_ptr, Some(cb.into())) } @@ -120,19 +121,19 @@ impl AsyncUsercallProvider { where F: FnOnce(io::Result) + Send + 'static, { - let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); - let mut peer_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut local_addr_buf = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut peer_addr_buf = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); - let local_addr_ptr = local_addr.as_raw_mut_ptr(); - let peer_addr_ptr = peer_addr.as_raw_mut_ptr(); + let local_addr_ptr = local_addr_buf.as_raw_mut_ptr(); + let peer_addr_ptr = peer_addr_buf.as_raw_mut_ptr(); let cb = move |res: io::Result| { - let local_addr = ManuallyDrop::into_inner(local_addr); - let peer_addr = ManuallyDrop::into_inner(peer_addr); + let local_addr_buf = ManuallyDrop::into_inner(local_addr_buf); + let peer_addr_buf = ManuallyDrop::into_inner(peer_addr_buf); - let local = string_from_bytebuffer(&*local_addr, "accept_stream", "local_addr"); - let peer = string_from_bytebuffer(&*peer_addr, "accept_stream", "peer_addr"); - let res = res.map(|fd| unsafe { new_std_stream(fd, Some(local), Some(peer)) }); + let local_addr = Some(string_from_bytebuffer(&*local_addr_buf, "accept_stream", "local_addr")); + let peer_addr = Some(string_from_bytebuffer(&*peer_addr_buf, "accept_stream", "peer_addr")); + let res = res.map(|fd| unsafe { TcpStream::from_raw_fd(fd, TcpStreamMetadata { local_addr, peer_addr }) }); callback(res); }; unsafe { self.raw_accept_stream(fd, local_addr_ptr, peer_addr_ptr, Some(cb.into())) } @@ -149,22 +150,22 @@ impl AsyncUsercallProvider { F: FnOnce(io::Result) + Send + 'static, { let mut addr_buf = ManuallyDrop::new(MakeSend::new(User::<[u8]>::uninitialized(addr.len()))); - let mut local_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); - let mut peer_addr = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut local_addr_buf = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); + let mut peer_addr_buf = ManuallyDrop::new(MakeSend::new(User::::uninitialized())); addr_buf[0..addr.len()].copy_from_enclave(addr.as_bytes()); let addr_buf_ptr = addr_buf.as_raw_mut_ptr() as *mut u8; - let local_addr_ptr = local_addr.as_raw_mut_ptr(); - let peer_addr_ptr = peer_addr.as_raw_mut_ptr(); + let local_addr_ptr = local_addr_buf.as_raw_mut_ptr(); + let peer_addr_ptr = peer_addr_buf.as_raw_mut_ptr(); let cb = move |res: io::Result| { let _addr_buf = ManuallyDrop::into_inner(addr_buf); - let local_addr = ManuallyDrop::into_inner(local_addr); - let peer_addr = ManuallyDrop::into_inner(peer_addr); + let local_addr_buf = ManuallyDrop::into_inner(local_addr_buf); + let peer_addr_buf = ManuallyDrop::into_inner(peer_addr_buf); - let local = string_from_bytebuffer(&local_addr, "connect_stream", "local_addr"); - let peer = string_from_bytebuffer(&peer_addr, "connect_stream", "peer_addr"); - let res = res.map(|fd| unsafe { new_std_stream(fd, Some(local), Some(peer)) }); + let local_addr = Some(string_from_bytebuffer(&local_addr_buf, "connect_stream", "local_addr")); + let peer_addr = Some(string_from_bytebuffer(&peer_addr_buf, "connect_stream", "peer_addr")); + let res = res.map(|fd| unsafe { TcpStream::from_raw_fd(fd, TcpStreamMetadata { local_addr, peer_addr }) }); callback(res); }; unsafe { self.raw_connect_stream(addr_buf_ptr, addr.len(), local_addr_ptr, peer_addr_ptr, Some(cb.into())) } diff --git a/async-usercalls/src/provider_core.rs b/intel-sgx/async-usercalls/src/provider_core.rs similarity index 100% rename from async-usercalls/src/provider_core.rs rename to intel-sgx/async-usercalls/src/provider_core.rs diff --git a/async-usercalls/src/queues.rs b/intel-sgx/async-usercalls/src/queues.rs similarity index 100% rename from async-usercalls/src/queues.rs rename to intel-sgx/async-usercalls/src/queues.rs diff --git a/async-usercalls/src/raw.rs b/intel-sgx/async-usercalls/src/raw.rs similarity index 100% rename from async-usercalls/src/raw.rs rename to intel-sgx/async-usercalls/src/raw.rs diff --git a/async-usercalls/src/test_support.rs b/intel-sgx/async-usercalls/src/test_support.rs similarity index 100% rename from async-usercalls/src/test_support.rs rename to intel-sgx/async-usercalls/src/test_support.rs diff --git a/async-usercalls/test.sh b/intel-sgx/async-usercalls/test.sh similarity index 100% rename from async-usercalls/test.sh rename to intel-sgx/async-usercalls/test.sh diff --git a/intel-sgx/enclave-runner/Cargo.toml b/intel-sgx/enclave-runner/Cargo.toml index 377b0317..0002469e 100644 --- a/intel-sgx/enclave-runner/Cargo.toml +++ b/intel-sgx/enclave-runner/Cargo.toml @@ -33,7 +33,7 @@ lazy_static = "1.2.0" # MIT/Apache-2.0 libc = "0.2.48" # MIT/Apache-2.0 nix = "0.13.0" # MIT openssl = { version = "0.10", optional = true } # Apache-2.0 -crossbeam = "0.7.1" # MIT/Apache-2.0 +crossbeam = "0.8.0" # MIT/Apache-2.0 num_cpus = "1.10.0" # MIT/Apache-2.0 tokio = { version = "0.2", features = ["full"] } # MIT futures = { version = "0.3", features = ["compat", "io-compat"] } # MIT/Apache-2.0 diff --git a/intel-sgx/enclave-runner/src/usercalls/mod.rs b/intel-sgx/enclave-runner/src/usercalls/mod.rs index 3ffcb787..2ed43e01 100644 --- a/intel-sgx/enclave-runner/src/usercalls/mod.rs +++ b/intel-sgx/enclave-runner/src/usercalls/mod.rs @@ -501,7 +501,7 @@ struct StoppedTcs { struct IOHandlerInput<'tcs> { tcs: Option<&'tcs mut RunningTcs>, enclave: Arc, - work_sender: &'tcs crossbeam::crossbeam_channel::Sender, + work_sender: &'tcs crossbeam::channel::Sender, } struct PendingEvents { @@ -773,7 +773,7 @@ impl EnclaveState { async fn handle_usercall( enclave: Arc, - work_sender: crossbeam::crossbeam_channel::Sender, + work_sender: crossbeam::channel::Sender, tx_return_channel: tokio::sync::mpsc::UnboundedSender<(EnclaveResult, ReturnSource)>, mut handle_data: UsercallHandleData, ) { @@ -865,7 +865,7 @@ impl EnclaveState { enclave: Arc, io_queue_receive: tokio::sync::mpsc::UnboundedReceiver, io_queue_send: tokio::sync::mpsc::UnboundedSender, - work_sender: crossbeam::crossbeam_channel::Sender, + work_sender: crossbeam::channel::Sender, ) -> EnclaveResult { let (tx_return_channel, mut rx_return_channel) = tokio::sync::mpsc::unbounded_channel(); let enclave_clone = enclave.clone(); @@ -1054,7 +1054,7 @@ impl EnclaveState { ) -> EnclaveResult { fn create_worker_threads( num_of_worker_threads: usize, - work_receiver: crossbeam::crossbeam_channel::Receiver, + work_receiver: crossbeam::channel::Receiver, io_queue_send: tokio::sync::mpsc::UnboundedSender, ) -> Vec> { let mut thread_handles = vec![]; @@ -1073,7 +1073,7 @@ impl EnclaveState { let (io_queue_send, io_queue_receive) = tokio::sync::mpsc::unbounded_channel(); - let (work_sender, work_receiver) = crossbeam::crossbeam_channel::unbounded(); + let (work_sender, work_receiver) = crossbeam::channel::unbounded(); work_sender .send(start_work) .expect("Work sender couldn't send data to receiver"); @@ -1147,7 +1147,7 @@ impl EnclaveState { rt.block_on(async move { enclave.abort_all_threads(); //clear the threads_queue - while enclave.threads_queue.pop().is_ok() {} + while enclave.threads_queue.pop().is_some() {} let cmd = enclave.kind.as_command().unwrap(); let mut cmddata = cmd.panic_reason.lock().await; @@ -1537,8 +1537,8 @@ impl<'tcs> IOHandlerInput<'tcs> { .as_command() .ok_or(IoErrorKind::InvalidInput)?; let new_tcs = match self.enclave.threads_queue.pop() { - Ok(tcs) => tcs, - Err(_) => { + Some(tcs) => tcs, + None => { return Err(IoErrorKind::WouldBlock.into()); } }; diff --git a/ipc-queue/src/fifo.rs b/ipc-queue/src/fifo.rs index d834673a..321e3043 100644 --- a/ipc-queue/src/fifo.rs +++ b/ipc-queue/src/fifo.rs @@ -9,7 +9,7 @@ use std::marker::PhantomData; use std::mem; #[cfg(not(target_env = "sgx"))] use { - std::sync::atomic::{AtomicU32, AtomicU64}, + std::sync::atomic::AtomicU64, std::sync::Arc, }; use std::sync::atomic::{AtomicUsize, Ordering, Ordering::SeqCst}; From 969a0f5c4a8e66dd709015bad0ba67d9f5f80288 Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Tue, 16 Aug 2022 13:13:23 -0700 Subject: [PATCH 07/19] Update fortanix-sgx-abi dependency and restore path dependencies --- Cargo.lock | 10 ++-------- intel-sgx/async-usercalls/Cargo.toml | 2 +- intel-sgx/enclave-runner/Cargo.toml | 2 +- ipc-queue/Cargo.toml | 2 +- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 284c8156..6f1649d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -933,7 +933,7 @@ dependencies = [ "failure", "failure_derive", "fnv", - "fortanix-sgx-abi 0.4.1", + "fortanix-sgx-abi", "futures 0.3.17", "ipc-queue", "lazy_static", @@ -1143,12 +1143,6 @@ dependencies = [ "percent-encoding 2.1.0", ] -[[package]] -name = "fortanix-sgx-abi" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816a38bd53bd5c87dd7edf4f15a2ee6b989ad7a5b5e616b75d70de64ad2a1329" - [[package]] name = "fortanix-sgx-abi" version = "0.5.0" @@ -1820,7 +1814,7 @@ dependencies = [ name = "ipc-queue" version = "0.2.0" dependencies = [ - "fortanix-sgx-abi 0.4.1", + "fortanix-sgx-abi", "futures 0.3.17", "static_assertions", "tokio 0.2.22", diff --git a/intel-sgx/async-usercalls/Cargo.toml b/intel-sgx/async-usercalls/Cargo.toml index 1efbc98f..c8f22534 100644 --- a/intel-sgx/async-usercalls/Cargo.toml +++ b/intel-sgx/async-usercalls/Cargo.toml @@ -18,7 +18,7 @@ categories = ["asynchronous"] [dependencies] # Project dependencies ipc-queue = { version = "0.2", path = "../../ipc-queue" } -fortanix-sgx-abi = { version = "0.4", path = "../fortanix-sgx-abi" } +fortanix-sgx-abi = { version = "0.5.0", path = "../fortanix-sgx-abi" } # External dependencies lazy_static = "1.4.0" # MIT/Apache-2.0 diff --git a/intel-sgx/enclave-runner/Cargo.toml b/intel-sgx/enclave-runner/Cargo.toml index 0002469e..11fe0ed4 100644 --- a/intel-sgx/enclave-runner/Cargo.toml +++ b/intel-sgx/enclave-runner/Cargo.toml @@ -21,7 +21,7 @@ exclude = ["fake-vdso/.gitignore", "fake-vdso/Makefile", "fake-vdso/main.S"] [dependencies] # Project dependencies sgxs = { version = "0.7.2", path = "../sgxs" } -fortanix-sgx-abi = { version = "0.4.0" } # TODO: add back `path = "../fortanix-sgx-abi"` +fortanix-sgx-abi = { version = "0.5.0", path = "../fortanix-sgx-abi" } sgx-isa = { version = "0.4.0", path = "../sgx-isa" } ipc-queue = { version = "0.2.0", path = "../../ipc-queue" } diff --git a/ipc-queue/Cargo.toml b/ipc-queue/Cargo.toml index a5e8c1b5..2a4d1cd7 100644 --- a/ipc-queue/Cargo.toml +++ b/ipc-queue/Cargo.toml @@ -14,7 +14,7 @@ keywords = ["sgx", "fifo", "queue", "ipc"] categories = ["asynchronous"] [dependencies] -fortanix-sgx-abi = { version = "0.4.0" } # TODO: add back `path = "../intel-sgx/fortanix-sgx-abi"` +fortanix-sgx-abi = { version = "0.5.0", path = "../intel-sgx/fortanix-sgx-abi" } [dev-dependencies] static_assertions = "1.1.0" From bdeceb9b1043ec20803b7b7891ec2587dbca9ed4 Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Wed, 17 Aug 2022 19:57:02 -0700 Subject: [PATCH 08/19] Add documentation for async-usercalls and address other minor issues --- intel-sgx/async-usercalls/src/lib.rs | 36 +++++++++++++++++++ .../async-usercalls/src/provider_core.rs | 10 ++++++ intel-sgx/async-usercalls/src/queues.rs | 6 ++-- intel-sgx/enclave-runner/src/usercalls/mod.rs | 4 +++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/intel-sgx/async-usercalls/src/lib.rs b/intel-sgx/async-usercalls/src/lib.rs index 5ac00326..13edc5e7 100644 --- a/intel-sgx/async-usercalls/src/lib.rs +++ b/intel-sgx/async-usercalls/src/lib.rs @@ -1,3 +1,39 @@ +//! This crate provides an interface for performing asynchronous usercalls in +//! SGX enclaves. The motivation behind asynchronous usercalls and ABI +//! documentation can be found +//! [here](https://edp.fortanix.com/docs/api/fortanix_sgx_abi/async/index.html). +//! The API provided here is fairly low level and is not meant for general use. +//! These APIs can be used to implement [mio] abstractions which in turn +//! allows us to use [tokio] in SGX enclaves! +//! +//! The main interface is provided through `AsyncUsercallProvider` which works +//! in tandem with `CallbackHandler`: +//! ``` +//! use async_usercalls::AsyncUsercallProvider; +//! use std::{io::Result, net::TcpStream, sync::mpsc, time::Duration}; +//! +//! let (provider, callback_handler) = AsyncUsercallProvider::new(); +//! let (tx, rx) = mpsc::sync_channel(1); +//! // The closure is called when userspace sends back the result of the +//! // usercall. +//! let cancel_handle = provider.connect_stream("www.example.com:80", move |res| { +//! tx.send(res).unwrap(); +//! }); +//! // We can cancel the connect usercall using `cancel_handle.cancel()`, but +//! // note that we may still get a successful result. +//! // We need to poll `callback_handler` to make progress. +//! loop { +//! let n = callback_handler.poll(Some(Duration::from_millis(100))); +//! if n > 0 { +//! break; // at least 1 callback function was executed! +//! } +//! } +//! let connect_result: Result = rx.recv().unwrap(); +//! ``` +//! +//! [mio]: https://docs.rs/mio/latest/mio/ +//! [tokio]: https://docs.rs/tokio/latest/tokio/ + #![feature(sgx_platform)] #![feature(never_type)] #![cfg_attr(test, feature(unboxed_closures))] diff --git a/intel-sgx/async-usercalls/src/provider_core.rs b/intel-sgx/async-usercalls/src/provider_core.rs index 5b027c16..dabd7bc5 100644 --- a/intel-sgx/async-usercalls/src/provider_core.rs +++ b/intel-sgx/async-usercalls/src/provider_core.rs @@ -64,3 +64,13 @@ impl Drop for ProviderCore { PROVIDERS.remove_provider(self.provider_id); } } + +pub trait ProviderId { + fn provider_id(&self) -> u32; +} + +impl ProviderId for Identified { + fn provider_id(&self) -> u32 { + (self.id >> 32) as u32 + } +} diff --git a/intel-sgx/async-usercalls/src/queues.rs b/intel-sgx/async-usercalls/src/queues.rs index fc7bbd07..c3a50eb1 100644 --- a/intel-sgx/async-usercalls/src/queues.rs +++ b/intel-sgx/async-usercalls/src/queues.rs @@ -1,4 +1,5 @@ use crate::hacks::{alloc_descriptor, async_queues, to_enclave, Cancel, Return, Usercall}; +use crate::provider_core::ProviderId; use crossbeam_channel as mpmc; use fortanix_sgx_abi::{EV_CANCELQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY, EV_USERCALLQ_NOT_FULL}; use ipc_queue::{self, Identified, QueueEvent, RecvError, SynchronizationError, Synchronizer}; @@ -89,10 +90,9 @@ impl ReturnHandler { // taking the lock should be fast. let provider_map = self.provider_map.lock().unwrap(); for ret in returns { - let provider_id = (ret.id >> 32) as u32; // NOTE: some providers might decide not to receive results of usercalls they send // because the results are not interesting, e.g. BatchDropProvider. - if let Some(sender) = provider_map.get(provider_id).and_then(|entry| entry.as_ref()) { + if let Some(sender) = provider_map.get(ret.provider_id()).and_then(|entry| entry.as_ref()) { let _ = sender.send(*ret); } } @@ -101,6 +101,8 @@ impl ReturnHandler { fn run(self) { let mut returns = [Identified::default(); Self::RECV_BATCH_SIZE]; loop { + // Block until there is a return. Then we receive any other values + // from the return queue **without** blocking using `try_iter()`. let first = match self.return_queue_rx.recv() { Ok(ret) => ret, Err(RecvError::Closed) => break, diff --git a/intel-sgx/enclave-runner/src/usercalls/mod.rs b/intel-sgx/enclave-runner/src/usercalls/mod.rs index 2ed43e01..258317a0 100644 --- a/intel-sgx/enclave-runner/src/usercalls/mod.rs +++ b/intel-sgx/enclave-runner/src/usercalls/mod.rs @@ -47,6 +47,10 @@ lazy_static! { static ref DEBUGGER_TOGGLE_SYNC: Mutex<()> = Mutex::new(()); } +// This is not an event in the sense that it could be passed to `send()` or +// `wait()` usercalls in enclave code. However, it's easier for the enclave +// runner implementation to lump it in with events. Also note that this constant +// is not public. const EV_ABORT: u64 = 0b0000_0000_0001_0000; const USERCALL_QUEUE_SIZE: usize = 16; From cc95cab4f9c824dfa33355dc189a1a0c7d800aeb Mon Sep 17 00:00:00 2001 From: Mohsen Zohrevandi Date: Mon, 29 Aug 2022 12:25:22 -0700 Subject: [PATCH 09/19] Remove almost all hacks --- intel-sgx/async-usercalls/src/batch_drop.rs | 3 +- intel-sgx/async-usercalls/src/callback.rs | 4 +- intel-sgx/async-usercalls/src/duplicated.rs | 168 ------------------ .../async-usercalls/src/hacks/async_queues.rs | 52 ------ intel-sgx/async-usercalls/src/lib.rs | 7 +- intel-sgx/async-usercalls/src/provider_api.rs | 2 +- .../async-usercalls/src/provider_core.rs | 2 +- intel-sgx/async-usercalls/src/queues.rs | 28 +-- intel-sgx/async-usercalls/src/raw.rs | 3 +- .../src/{hacks/mod.rs => utils.rs} | 25 +-- 10 files changed, 25 insertions(+), 269 deletions(-) delete mode 100644 intel-sgx/async-usercalls/src/duplicated.rs delete mode 100644 intel-sgx/async-usercalls/src/hacks/async_queues.rs rename intel-sgx/async-usercalls/src/{hacks/mod.rs => utils.rs} (56%) diff --git a/intel-sgx/async-usercalls/src/batch_drop.rs b/intel-sgx/async-usercalls/src/batch_drop.rs index 62435460..2f7bb698 100644 --- a/intel-sgx/async-usercalls/src/batch_drop.rs +++ b/intel-sgx/async-usercalls/src/batch_drop.rs @@ -1,10 +1,9 @@ -use crate::hacks::Usercall; use crate::provider_core::ProviderCore; use ipc_queue::Identified; use std::cell::RefCell; use std::mem; use std::os::fortanix_sgx::usercalls::alloc::{User, UserSafe}; -use std::os::fortanix_sgx::usercalls::raw::UsercallNrs; +use std::os::fortanix_sgx::usercalls::raw::{Usercall, UsercallNrs}; pub trait BatchDroppable: private::BatchDroppable {} impl BatchDroppable for T {} diff --git a/intel-sgx/async-usercalls/src/callback.rs b/intel-sgx/async-usercalls/src/callback.rs index 7586ebb0..b89fcaf9 100644 --- a/intel-sgx/async-usercalls/src/callback.rs +++ b/intel-sgx/async-usercalls/src/callback.rs @@ -1,7 +1,7 @@ -use crate::duplicated::{FromSgxResult, ReturnValue}; -use crate::hacks::Return; use fortanix_sgx_abi::{invoke_with_usercalls, Fd, Result}; use std::io; +use std::os::fortanix_sgx::usercalls::raw::{Return, ReturnValue}; +use std::os::fortanix_sgx::usercalls::FromSgxResult; pub struct CbFn(Box); diff --git a/intel-sgx/async-usercalls/src/duplicated.rs b/intel-sgx/async-usercalls/src/duplicated.rs deleted file mode 100644 index 0a39e5a1..00000000 --- a/intel-sgx/async-usercalls/src/duplicated.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! this file contains code duplicated from libstd's sys/sgx -use fortanix_sgx_abi::{Error, Result, RESULT_SUCCESS}; -use std::io; -use std::ptr::NonNull; - -fn check_os_error(err: Result) -> i32 { - // FIXME: not sure how to make sure all variants of Error are covered - if err == Error::NotFound as _ - || err == Error::PermissionDenied as _ - || err == Error::ConnectionRefused as _ - || err == Error::ConnectionReset as _ - || err == Error::ConnectionAborted as _ - || err == Error::NotConnected as _ - || err == Error::AddrInUse as _ - || err == Error::AddrNotAvailable as _ - || err == Error::BrokenPipe as _ - || err == Error::AlreadyExists as _ - || err == Error::WouldBlock as _ - || err == Error::InvalidInput as _ - || err == Error::InvalidData as _ - || err == Error::TimedOut as _ - || err == Error::WriteZero as _ - || err == Error::Interrupted as _ - || err == Error::Other as _ - || err == Error::UnexpectedEof as _ - || ((Error::UserRangeStart as _)..=(Error::UserRangeEnd as _)).contains(&err) - { - err - } else { - panic!("Usercall: returned invalid error value {}", err) - } -} - -pub trait FromSgxResult { - type Return; - - fn from_sgx_result(self) -> io::Result; -} - -impl FromSgxResult for (Result, T) { - type Return = T; - - fn from_sgx_result(self) -> io::Result { - if self.0 == RESULT_SUCCESS { - Ok(self.1) - } else { - Err(io::Error::from_raw_os_error(check_os_error(self.0))) - } - } -} - -impl FromSgxResult for Result { - type Return = (); - - fn from_sgx_result(self) -> io::Result { - if self == RESULT_SUCCESS { - Ok(()) - } else { - Err(io::Error::from_raw_os_error(check_os_error(self))) - } - } -} - -type Register = u64; - -pub trait RegisterArgument { - fn from_register(_: Register) -> Self; - fn into_register(self) -> Register; -} - -pub trait ReturnValue { - fn from_registers(call: &'static str, regs: (Register, Register)) -> Self; -} - -macro_rules! define_ra { - (< $i:ident > $t:ty) => { - impl<$i> RegisterArgument for $t { - fn from_register(a: Register) -> Self { - a as _ - } - fn into_register(self) -> Register { - self as _ - } - } - }; - ($i:ty as $t:ty) => { - impl RegisterArgument for $t { - fn from_register(a: Register) -> Self { - a as $i as _ - } - fn into_register(self) -> Register { - self as $i as _ - } - } - }; - ($t:ty) => { - impl RegisterArgument for $t { - fn from_register(a: Register) -> Self { - a as _ - } - fn into_register(self) -> Register { - self as _ - } - } - }; -} - -define_ra!(Register); -define_ra!(i64); -define_ra!(u32); -define_ra!(u32 as i32); -define_ra!(u16); -define_ra!(u16 as i16); -define_ra!(u8); -define_ra!(u8 as i8); -define_ra!(usize); -define_ra!(usize as isize); -define_ra!( *const T); -define_ra!( *mut T); - -impl RegisterArgument for bool { - fn from_register(a: Register) -> bool { - if a != 0 { - true - } else { - false - } - } - fn into_register(self) -> Register { - self as _ - } -} - -impl RegisterArgument for Option> { - fn from_register(a: Register) -> Option> { - NonNull::new(a as _) - } - fn into_register(self) -> Register { - self.map_or(0 as _, NonNull::as_ptr) as _ - } -} - -impl ReturnValue for ! { - fn from_registers(call: &'static str, _regs: (Register, Register)) -> Self { - panic!("Usercall {}: did not expect to be re-entered", call); - } -} - -impl ReturnValue for () { - fn from_registers(_call: &'static str, usercall_retval: (Register, Register)) -> Self { - assert!(usercall_retval.0 == 0); - assert!(usercall_retval.1 == 0); - () - } -} - -impl ReturnValue for T { - fn from_registers(_call: &'static str, usercall_retval: (Register, Register)) -> Self { - assert!(usercall_retval.1 == 0); - T::from_register(usercall_retval.0) - } -} - -impl ReturnValue for (T, U) { - fn from_registers(_call: &'static str, regs: (Register, Register)) -> Self { - (T::from_register(regs.0), U::from_register(regs.1)) - } -} diff --git a/intel-sgx/async-usercalls/src/hacks/async_queues.rs b/intel-sgx/async-usercalls/src/hacks/async_queues.rs deleted file mode 100644 index 5e7a9252..00000000 --- a/intel-sgx/async-usercalls/src/hacks/async_queues.rs +++ /dev/null @@ -1,52 +0,0 @@ -use super::{Cancel, Return, Usercall}; -use crate::duplicated::ReturnValue; -use fortanix_sgx_abi::FifoDescriptor; -use std::num::NonZeroU64; -use std::os::fortanix_sgx::usercalls; -use std::os::fortanix_sgx::usercalls::raw; -use std::os::fortanix_sgx::usercalls::alloc::{UserSafeSized, User}; -use std::{mem, ptr}; - -// TODO: remove these once support for cancel queue is added in `std::os::fortanix_sgx` - -pub unsafe fn async_queues( - usercall_queue: *mut FifoDescriptor, - return_queue: *mut FifoDescriptor, - cancel_queue: *mut FifoDescriptor, -) -> raw::Result { - ReturnValue::from_registers( - "async_queues", - raw::do_usercall( - NonZeroU64::new(raw::UsercallNrs::async_queues as _).unwrap(), - usercall_queue as _, - return_queue as _, - cancel_queue as _, - 0, - false, - ), - ) -} - -pub unsafe fn alloc_descriptor() -> *mut FifoDescriptor { - #[repr(transparent)] - #[derive(Copy, Clone)] - struct WrappedFifoDescriptor(FifoDescriptor); - unsafe impl UserSafeSized for WrappedFifoDescriptor{} - - User::>::uninitialized().into_raw() as _ -} - -pub unsafe fn to_enclave(ptr: *mut FifoDescriptor) -> FifoDescriptor { - let mut dest: FifoDescriptor = mem::zeroed(); - ptr::copy( - ptr as *const u8, - (&mut dest) as *mut FifoDescriptor as *mut u8, - mem::size_of_val(&mut dest), - ); - usercalls::free( - ptr as _, - mem::size_of::>(), - mem::align_of::>(), - ); - dest -} diff --git a/intel-sgx/async-usercalls/src/lib.rs b/intel-sgx/async-usercalls/src/lib.rs index 13edc5e7..f8b5b429 100644 --- a/intel-sgx/async-usercalls/src/lib.rs +++ b/intel-sgx/async-usercalls/src/lib.rs @@ -42,13 +42,12 @@ use crossbeam_channel as mpmc; use ipc_queue::Identified; use std::collections::HashMap; +use std::os::fortanix_sgx::usercalls::raw::{Cancel, Return, Usercall}; use std::sync::Mutex; use std::time::Duration; mod batch_drop; mod callback; -mod duplicated; -mod hacks; mod io_bufs; mod provider_api; mod provider_core; @@ -56,6 +55,7 @@ mod queues; mod raw; #[cfg(test)] mod test_support; +mod utils; pub use self::batch_drop::batch_drop; pub use self::callback::CbFn; @@ -63,7 +63,6 @@ pub use self::io_bufs::{ReadBuffer, UserBuf, WriteBuffer}; pub use self::raw::RawApi; use self::callback::*; -use self::hacks::{Cancel, Return, Usercall}; use self::provider_core::ProviderCore; use self::queues::*; @@ -235,8 +234,8 @@ impl CallbackHandler { #[cfg(test)] mod tests { use super::*; - use crate::hacks::MakeSend; use crate::test_support::*; + use crate::utils::MakeSend; use crossbeam_channel as mpmc; use std::io; use std::net::{TcpListener, TcpStream}; diff --git a/intel-sgx/async-usercalls/src/provider_api.rs b/intel-sgx/async-usercalls/src/provider_api.rs index ca307b17..ae795cd9 100644 --- a/intel-sgx/async-usercalls/src/provider_api.rs +++ b/intel-sgx/async-usercalls/src/provider_api.rs @@ -1,7 +1,7 @@ use crate::batch_drop; -use crate::hacks::MakeSend; use crate::io_bufs::UserBuf; use crate::raw::RawApi; +use crate::utils::MakeSend; use crate::{AsyncUsercallProvider, CancelHandle}; use fortanix_sgx_abi::Fd; use std::io; diff --git a/intel-sgx/async-usercalls/src/provider_core.rs b/intel-sgx/async-usercalls/src/provider_core.rs index dabd7bc5..5ea3a6ae 100644 --- a/intel-sgx/async-usercalls/src/provider_core.rs +++ b/intel-sgx/async-usercalls/src/provider_core.rs @@ -1,8 +1,8 @@ -use crate::hacks::{Cancel, Return, Usercall}; use crate::queues::*; use crate::CancelHandle; use crossbeam_channel as mpmc; use ipc_queue::Identified; +use std::os::fortanix_sgx::usercalls::raw::{Cancel, Return, Usercall}; use std::sync::atomic::{AtomicU32, Ordering}; pub(crate) struct ProviderCore { diff --git a/intel-sgx/async-usercalls/src/queues.rs b/intel-sgx/async-usercalls/src/queues.rs index c3a50eb1..d33b22ab 100644 --- a/intel-sgx/async-usercalls/src/queues.rs +++ b/intel-sgx/async-usercalls/src/queues.rs @@ -1,10 +1,12 @@ -use crate::hacks::{alloc_descriptor, async_queues, to_enclave, Cancel, Return, Usercall}; use crate::provider_core::ProviderId; use crossbeam_channel as mpmc; use fortanix_sgx_abi::{EV_CANCELQ_NOT_FULL, EV_RETURNQ_NOT_EMPTY, EV_USERCALLQ_NOT_FULL}; use ipc_queue::{self, Identified, QueueEvent, RecvError, SynchronizationError, Synchronizer}; use lazy_static::lazy_static; -use std::os::fortanix_sgx::usercalls::raw; +use std::os::fortanix_sgx::usercalls::alloc::User; +use std::os::fortanix_sgx::usercalls::raw::{ + self, async_queues, Cancel, FifoDescriptor, Return, Usercall, +}; use std::sync::{Arc, Mutex}; use std::{io, iter, thread}; @@ -54,24 +56,24 @@ lazy_static! { } fn init_async_queues() -> io::Result<(Sender, Sender, Receiver)> { - // FIXME: this is just a hack. Replace these with `User::>::uninitialized().into_raw()` - let usercall_q = unsafe { alloc_descriptor::() }; - let cancel_q = unsafe { alloc_descriptor::() }; - let return_q = unsafe { alloc_descriptor::() }; + let usercall_q = User::>::uninitialized().into_raw(); + let cancel_q = User::>::uninitialized().into_raw(); + let return_q = User::>::uninitialized().into_raw(); let r = unsafe { async_queues(usercall_q, return_q, cancel_q) }; if r != 0 { return Err(io::Error::from_raw_os_error(r)); } - // FIXME: this is another hack, replace with `unsafe { User::>::from_raw(q) }.to_enclave()` - let usercall_queue = unsafe { to_enclave(usercall_q) }; - let cancel_queue = unsafe { to_enclave(cancel_q) }; - let return_queue = unsafe { to_enclave(return_q) }; + let usercall_queue = unsafe { User::>::from_raw(usercall_q) }.to_enclave(); + let cancel_queue = unsafe { User::>::from_raw(cancel_q) }.to_enclave(); + let return_queue = unsafe { User::>::from_raw(return_q) }.to_enclave(); - let utx = unsafe { Sender::from_descriptor(usercall_queue, QueueSynchronizer { queue: Queue::Usercall }) }; - let ctx = unsafe { Sender::from_descriptor(cancel_queue, QueueSynchronizer { queue: Queue::Cancel }) }; - let rx = unsafe { Receiver::from_descriptor(return_queue, QueueSynchronizer { queue: Queue::Return }) }; + // FIXME: once `WithId` is exported from `std::os::fortanix_sgx::usercalls::raw`, we can remove + // `transmute` calls here and use FifoDescriptor/WithId from std everywhere including in ipc-queue. + let utx = unsafe { Sender::from_descriptor(std::mem::transmute(usercall_queue), QueueSynchronizer { queue: Queue::Usercall }) }; + let ctx = unsafe { Sender::from_descriptor(std::mem::transmute(cancel_queue), QueueSynchronizer { queue: Queue::Cancel }) }; + let rx = unsafe { Receiver::from_descriptor(std::mem::transmute(return_queue), QueueSynchronizer { queue: Queue::Return }) }; Ok((utx, ctx, rx)) } diff --git a/intel-sgx/async-usercalls/src/raw.rs b/intel-sgx/async-usercalls/src/raw.rs index 7edaa7eb..2939e0ae 100644 --- a/intel-sgx/async-usercalls/src/raw.rs +++ b/intel-sgx/async-usercalls/src/raw.rs @@ -1,10 +1,9 @@ use crate::callback::*; -use crate::hacks::Usercall; use crate::{AsyncUsercallProvider, CancelHandle}; use fortanix_sgx_abi::Fd; use std::io; use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; -use std::os::fortanix_sgx::usercalls::raw::UsercallNrs; +use std::os::fortanix_sgx::usercalls::raw::{Usercall, UsercallNrs}; pub trait RawApi { unsafe fn raw_read( diff --git a/intel-sgx/async-usercalls/src/hacks/mod.rs b/intel-sgx/async-usercalls/src/utils.rs similarity index 56% rename from intel-sgx/async-usercalls/src/hacks/mod.rs rename to intel-sgx/async-usercalls/src/utils.rs index 5fe7a2df..78f3c051 100644 --- a/intel-sgx/async-usercalls/src/hacks/mod.rs +++ b/intel-sgx/async-usercalls/src/utils.rs @@ -1,32 +1,9 @@ use std::ops::{Deref, DerefMut}; -use std::os::fortanix_sgx::usercalls::alloc::{User, UserSafeSized}; +use std::os::fortanix_sgx::usercalls::alloc::User; use std::os::fortanix_sgx::usercalls::raw::ByteBuffer; -mod async_queues; - -pub use self::async_queues::{alloc_descriptor, async_queues, to_enclave}; - -#[repr(C)] -#[derive(Copy, Clone, Default)] -pub struct Usercall(pub u64, pub u64, pub u64, pub u64, pub u64); - -unsafe impl UserSafeSized for Usercall {} - -#[repr(C)] -#[derive(Copy, Clone, Default)] -pub struct Return(pub u64, pub u64); - -unsafe impl UserSafeSized for Return {} - -#[repr(C)] -#[derive(Copy, Clone, Default)] -pub struct Cancel; - -unsafe impl UserSafeSized for Cancel {} - pub(crate) trait MakeSendMarker {} -// Interim solution until we mark the target types appropriately pub(crate) struct MakeSend(T); impl MakeSend { From 7a6387e5f5b8c6393cc33475d50d356a55958733 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Wed, 26 Apr 2023 13:10:17 -0700 Subject: [PATCH 10/19] resolve rebase conflict --- Cargo.lock | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6f1649d9..3a4dc4db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -964,7 +964,7 @@ dependencies = [ "shiplift", "tempfile", "tokio 1.16.1", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -1436,7 +1436,7 @@ dependencies = [ "http 0.2.5", "indexmap", "slab", - "tokio 1.14.0", + "tokio 1.16.1", "tokio-util", "tracing", ] @@ -1682,7 +1682,7 @@ dependencies = [ "bytes 1.1.0", "hyper 0.14.15", "native-tls", - "tokio 1.14.0", + "tokio 1.16.1", "tokio-native-tls", ] @@ -1724,7 +1724,7 @@ dependencies = [ "sgx_pkix", "sgxs", "sgxs-loaders", - "tokio 1.14.0", + "tokio 1.16.1", "url 2.2.2", ] @@ -3228,7 +3228,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded 0.7.1", - "tokio 1.14.0", + "tokio 1.16.1", "tokio-native-tls", "url 2.2.2", "wasm-bindgen", @@ -3677,7 +3677,7 @@ dependencies = [ "serde_json", "tar", "tokio 1.16.1", - "url 2.1.1", + "url 2.2.2", ] [[package]] @@ -3960,6 +3960,7 @@ version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c27a64b625de6d309e8c57716ba93021dccf1b3b5c97edd6d3dd2d2135afc0a" dependencies = [ + "bytes 1.1.0", "libc", "memchr", "mio 0.7.14", @@ -4040,7 +4041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.14.0", + "tokio 1.16.1", ] [[package]] @@ -4137,7 +4138,7 @@ dependencies = [ "futures-core", "futures-sink", "pin-project-lite 0.2.7", - "tokio 1.14.0", + "tokio 1.16.1", "tracing", ] From 8bc30477ed3dd86a51f1c1e3fdca1571d65a02bb Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 25 Apr 2023 13:40:06 -0700 Subject: [PATCH 11/19] bump version - Bump em-app,nitro-attestation-verify's dependency of `mbedtls` to 0.9.0 - Extend dcap-ql,ias,sgx-isa's dependency of `mbedtls` to ">=0.8.0, <0.10.0" since there is no api changes from `mbedtls` --- Cargo.lock | 2 +- em-app/Cargo.toml | 2 +- em-app/examples/get-certificate/Cargo.toml | 2 +- em-app/examples/harmonize/Cargo.toml | 2 +- .../aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a4dc4db..811400b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2226,7 +2226,7 @@ dependencies = [ [[package]] name = "nitro-attestation-verify" -version = "0.1.1" +version = "0.2.0" dependencies = [ "aws-nitro-enclaves-cose 0.5.0", "chrono", diff --git a/em-app/Cargo.toml b/em-app/Cargo.toml index acf9778a..1d01b20d 100644 --- a/em-app/Cargo.toml +++ b/em-app/Cargo.toml @@ -14,7 +14,7 @@ b64-ct = "0.1.0" em-client = { version = "3.0.0", default-features = false, features = ["client"] } em-node-agent-client = "1.0.0" hyper = { version = "0.10", default-features = false } -mbedtls = { version = "0.9", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } pkix = "0.1.2" rustc-serialize = "0.3.24" diff --git a/em-app/examples/get-certificate/Cargo.toml b/em-app/examples/get-certificate/Cargo.toml index 31634c96..628e4b32 100644 --- a/em-app/examples/get-certificate/Cargo.toml +++ b/em-app/examples/get-certificate/Cargo.toml @@ -7,5 +7,5 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.9", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0" diff --git a/em-app/examples/harmonize/Cargo.toml b/em-app/examples/harmonize/Cargo.toml index a899aa7e..a7d2d4ec 100644 --- a/em-app/examples/harmonize/Cargo.toml +++ b/em-app/examples/harmonize/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.9", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0.62" serde = "1.0.123" serde_derive = "1.0.123" diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml index 9b152671..6b0d7d8b 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nitro-attestation-verify" -version = "0.1.1" +version = "0.2.0" authors = ["Adrian Cruceru "] edition = "2018" From 8d1c2d327da721158119d5515236e3d5aba394a2 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 25 Apr 2023 17:40:07 -0700 Subject: [PATCH 12/19] extend compatible pkix version --- em-app/Cargo.toml | 2 +- em-app/examples/harmonize/Cargo.toml | 2 +- .../aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml | 4 ++-- fortanix-vme/aws-nitro-enclaves/tests/nsm-test/Cargo.toml | 2 +- fortanix-vme/vme-pkix/Cargo.toml | 2 +- intel-sgx/ias/Cargo.toml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/em-app/Cargo.toml b/em-app/Cargo.toml index 1d01b20d..2a09dee1 100644 --- a/em-app/Cargo.toml +++ b/em-app/Cargo.toml @@ -15,7 +15,7 @@ em-client = { version = "3.0.0", default-features = false, features = ["client"] em-node-agent-client = "1.0.0" hyper = { version = "0.10", default-features = false } mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } -pkix = "0.1.2" +pkix = ">=0.1.2, <0.3.0" rustc-serialize = "0.3.24" sdkms = { version = "0.2.1", default-features = false } diff --git a/em-app/examples/harmonize/Cargo.toml b/em-app/examples/harmonize/Cargo.toml index a7d2d4ec..2eae53bb 100644 --- a/em-app/examples/harmonize/Cargo.toml +++ b/em-app/examples/harmonize/Cargo.toml @@ -18,5 +18,5 @@ hyper = "0.10" sdkms = { version = "0.2.1", default-features = false } rustc-serialize = "0.3.24" csv = "1.1" -pkix = "0.1.2" +pkix = ">=0.1.2, <0.3.0" url = "1" diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml index 6b0d7d8b..2761d054 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml @@ -14,12 +14,12 @@ mbedtls = { version = ">=0.8.0, <0.10.0", features = ["rdrand", "std", "time"], num-bigint = "0.4" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" -pkix = "0.1" +pkix = ">=0.1.1, <0.3.0" yasna = { version = "0.4", features = ["num-bigint"] } [dev-dependencies] chrono = "0.4.0" -pkix = "0.1" +pkix = ">=0.1.1, <0.3.0" lazy_static = "1.3.0" [features] diff --git a/fortanix-vme/aws-nitro-enclaves/tests/nsm-test/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/tests/nsm-test/Cargo.toml index b60bb782..93df3f34 100644 --- a/fortanix-vme/aws-nitro-enclaves/tests/nsm-test/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/tests/nsm-test/Cargo.toml @@ -7,4 +7,4 @@ edition = "2021" [dependencies] nsm = { path = "../../nsm" } -pkix = { version = "0.1" } +pkix = ">=0.1.1, <0.3.0" diff --git a/fortanix-vme/vme-pkix/Cargo.toml b/fortanix-vme/vme-pkix/Cargo.toml index 61e7c500..09d10fda 100644 --- a/fortanix-vme/vme-pkix/Cargo.toml +++ b/fortanix-vme/vme-pkix/Cargo.toml @@ -13,4 +13,4 @@ categories = ["cryptography"] [dependencies] lazy_static = "1" -pkix = "0.1.1" +pkix = ">=0.1.1, <0.3.0" diff --git a/intel-sgx/ias/Cargo.toml b/intel-sgx/ias/Cargo.toml index 978cb447..238c0279 100644 --- a/intel-sgx/ias/Cargo.toml +++ b/intel-sgx/ias/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0.7", features = ["derive"] } url = "2.2" mbedtls = { version = ">=0.8.0, <0.10.0", features = ["std"], default-features = false, optional = true } -pkix = "0.1" +pkix = ">=0.1.1, <0.3.0" sgx-isa = { version = "0.4", path = "../sgx-isa" } sgx_pkix = { version = "0.2", path = "../sgx_pkix" } From 3a871a23938762e6579dfb0ee605e6eeb25d3f21 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 25 Apr 2023 13:41:08 -0700 Subject: [PATCH 13/19] update code for api changes in `mbedtls` --- em-app/src/mbedtls_hyper.rs | 64 ++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/em-app/src/mbedtls_hyper.rs b/em-app/src/mbedtls_hyper.rs index a7b2d404..c7cf83eb 100644 --- a/em-app/src/mbedtls_hyper.rs +++ b/em-app/src/mbedtls_hyper.rs @@ -6,10 +6,10 @@ use hyper::net::{NetworkStream, SslClient, SslServer}; use std::fmt; use std::io; +use std::io::{Error as IoError, ErrorKind as IoErrorKind}; use std::net::SocketAddr; use std::sync::{Arc, Mutex}; use std::time::Duration; -use std::io::{Error as IoError, ErrorKind as IoErrorKind}; use mbedtls::ssl::{Config, Context}; @@ -19,26 +19,34 @@ pub struct TlsStream { context: Arc>>, } -impl TlsStream { +impl TlsStream +where + T: 'static, +{ pub fn new(context: Arc>>) -> io::Result { if context.lock().unwrap().io_mut().is_none() { - return Err(IoError::new(IoErrorKind::InvalidInput, "Peer set in context is not of expected type")); + return Err(IoError::new( + IoErrorKind::InvalidInput, + "Peer set in context is not of expected type", + )); } - Ok(TlsStream { - context, - }) + Ok(TlsStream { context: context }) } } -impl io::Read for TlsStream +impl io::Read for TlsStream +where + T: 'static + io::Read + io::Write, { fn read(&mut self, buf: &mut [u8]) -> io::Result { self.context.lock().unwrap().read(buf) } } -impl io::Write for TlsStream +impl io::Write for TlsStream +where + T: 'static + io::Read + io::Write, { fn write(&mut self, buf: &[u8]) -> io::Result { self.context.lock().unwrap().write(buf) @@ -49,28 +57,37 @@ impl io::Write for TlsStream } } -impl NetworkStream for TlsStream - where T: NetworkStream +impl NetworkStream for TlsStream +where + T: NetworkStream + 'static, { fn peer_addr(&mut self) -> io::Result { - self.context.lock().unwrap().io_mut() + self.context + .lock() + .unwrap() + .io_mut() .ok_or(IoError::new(IoErrorKind::NotFound, "No peer available"))? .peer_addr() } fn set_read_timeout(&self, dur: Option) -> io::Result<()> { - self.context.lock().unwrap().io_mut() + self.context + .lock() + .unwrap() + .io_mut() .ok_or(IoError::new(IoErrorKind::NotFound, "No peer available"))? .set_read_timeout(dur) } fn set_write_timeout(&self, dur: Option) -> io::Result<()> { - self.context.lock().unwrap().io_mut() + self.context + .lock() + .unwrap() + .io_mut() .ok_or(IoError::new(IoErrorKind::NotFound, "No peer available"))? .set_write_timeout(dur) } - } - +} #[derive(Clone)] pub struct MbedSSLServer { @@ -79,15 +96,15 @@ pub struct MbedSSLServer { impl MbedSSLServer { pub fn new(rc_config: Arc) -> Self { - MbedSSLServer { - rc_config, - } + MbedSSLServer { rc_config } } } -/// An abstraction to allow any SSL implementation to be used with server-side HttpsStreams. +/// An abstraction to allow any SSL implementation to be used with server-side +/// HttpsStreams. impl SslServer for MbedSSLServer - where T: NetworkStream + Send + Clone + fmt::Debug + Sync +where + T: NetworkStream + Send + Clone + fmt::Debug + Sync, { /// The protected stream. type Stream = TlsStream; @@ -107,7 +124,8 @@ pub struct MbedSSLClient { verify_hostname: bool, // This can be used when verify_hostname is set to true. - // It will force ssl client to send this specific SNI on all established connections disregarding any host provided by hyper. + // It will force ssl client to send this specific SNI on all established connections disregarding any host provided by + // hyper. override_sni: Option, } @@ -132,7 +150,8 @@ impl MbedSSLClient { } impl SslClient for MbedSSLClient - where T: NetworkStream + Send + Clone + fmt::Debug + Sync +where + T: NetworkStream + Send + Clone + fmt::Debug + Sync, { type Stream = TlsStream; @@ -150,4 +169,3 @@ impl SslClient for MbedSSLClient } } } - From b3b9f74a363e06cdd2efd88e2771b73257704be9 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 23 May 2023 16:46:24 -0700 Subject: [PATCH 14/19] upgrade dep mbedtls to 0.10.0 Signed-off-by: Yuxiang Cao --- Cargo.toml | 2 +- em-app/Cargo.toml | 2 +- em-app/examples/get-certificate/Cargo.toml | 2 +- em-app/examples/harmonize/Cargo.toml | 2 +- em-app/src/utils.rs | 10 +++++----- .../nitro-attestation-verify/Cargo.toml | 2 +- intel-sgx/dcap-ql/Cargo.toml | 4 ++-- intel-sgx/ias/Cargo.toml | 2 +- intel-sgx/sgx-isa/Cargo.toml | 2 +- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 90acffde..dbc8a4e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ exclude = ["examples"] [patch.crates-io] libc = { git = "https://github.com/fortanix/libc.git", branch = "fortanixvme" } -mbedtls = { git = "https://github.com/fortanix/rust-mbedtls", branch = "master" } +mbedtls = { git = "https://github.com/fortanix/rust-mbedtls", branch = "yx/upgrade-mbedtls" } nix = { git = "https://github.com/fortanix/nix.git", branch = "raoul/fortanixvme_r0.20.2" } serde = { git = "https://github.com/fortanix/serde.git", branch = "master" } vsock = { git = "https://github.com/fortanix/vsock-rs.git", branch = "fortanixvme" } diff --git a/em-app/Cargo.toml b/em-app/Cargo.toml index 2a09dee1..edbcb998 100644 --- a/em-app/Cargo.toml +++ b/em-app/Cargo.toml @@ -14,7 +14,7 @@ b64-ct = "0.1.0" em-client = { version = "3.0.0", default-features = false, features = ["client"] } em-node-agent-client = "1.0.0" hyper = { version = "0.10", default-features = false } -mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } pkix = ">=0.1.2, <0.3.0" rustc-serialize = "0.3.24" diff --git a/em-app/examples/get-certificate/Cargo.toml b/em-app/examples/get-certificate/Cargo.toml index 628e4b32..47a7dbbd 100644 --- a/em-app/examples/get-certificate/Cargo.toml +++ b/em-app/examples/get-certificate/Cargo.toml @@ -7,5 +7,5 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0" diff --git a/em-app/examples/harmonize/Cargo.toml b/em-app/examples/harmonize/Cargo.toml index 2eae53bb..156a54c9 100644 --- a/em-app/examples/harmonize/Cargo.toml +++ b/em-app/examples/harmonize/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.9.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0.62" serde = "1.0.123" serde_derive = "1.0.123" diff --git a/em-app/src/utils.rs b/em-app/src/utils.rs index 0a50e731..cf54af7c 100644 --- a/em-app/src/utils.rs +++ b/em-app/src/utils.rs @@ -45,7 +45,7 @@ pub fn get_runtime_configuration( let mut config = Config::new(Endpoint::Client, Transport::Stream, Preset::Default); config.set_rng(Arc::new(mbedtls::rng::Rdrand)); - config.set_min_version(Version::Tls1_2).map_err(|e| format!("TLS configuration failed: {:?}", e))?; + config.set_min_version(Version::Tls12).map_err(|e| format!("TLS configuration failed: {:?}", e))?; if let Some(ca_cert_list) = ca_cert_list { config.set_ca_list(ca_cert_list, ca_crl); @@ -77,7 +77,7 @@ pub fn get_sdkms_dataset( let mut config = Config::new(Endpoint::Client, Transport::Stream, Preset::Default); config.set_rng(Arc::new(mbedtls::rng::Rdrand)); - config.set_min_version(Version::Tls1_2).map_err(|e| format!("TLS configuration failed: {:?}", e))?; + config.set_min_version(Version::Tls12).map_err(|e| format!("TLS configuration failed: {:?}", e))?; if let Some(ca_cert_list) = ca_cert_list { config.set_ca_list(ca_cert_list, ca_crl); @@ -112,7 +112,7 @@ pub fn https_get(url: Url, let mut config = Config::new(Endpoint::Client, Transport::Stream, Preset::Default); config.set_rng(Arc::new(mbedtls::rng::Rdrand)); - config.set_min_version(Version::Tls1_2).map_err(|e| format!("TLS configuration failed: {:?}", e))?; + config.set_min_version(Version::Tls12).map_err(|e| format!("TLS configuration failed: {:?}", e))?; if let Some(ca_cert_list) = ca_cert_list { config.set_ca_list(ca_cert_list, ca_crl); @@ -144,7 +144,7 @@ pub fn https_put(url: Url, let mut config = Config::new(Endpoint::Client, Transport::Stream, Preset::Default); config.set_rng(Arc::new(mbedtls::rng::Rdrand)); - config.set_min_version(Version::Tls1_2).map_err(|e| format!("TLS configuration failed: {:?}", e))?; + config.set_min_version(Version::Tls12).map_err(|e| format!("TLS configuration failed: {:?}", e))?; if let Some(ca_cert_list) = ca_cert_list { config.set_ca_list(ca_cert_list, ca_crl); @@ -222,7 +222,7 @@ pub fn get_mbedtls_hyper_connector_pool(ca_chain: Vec>, client_pki: Opti let mut config = Config::new(Endpoint::Client, Transport::Stream, Preset::Default); config.set_rng(Arc::new(mbedtls::rng::Rdrand)); - config.set_min_version(Version::Tls1_2).map_err(|e| format!("TLS configuration failed: {:?}", e))?; + config.set_min_version(Version::Tls12).map_err(|e| format!("TLS configuration failed: {:?}", e))?; if !ca_chain.is_empty() { let mut list = MbedtlsList::::new(); diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml index 2761d054..6c13f8e8 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml @@ -10,7 +10,7 @@ serde_cbor = "0.11" # Required until PR36 is accepted # https://github.com/awslabs/aws-nitro-enclaves-cose/pull/36 aws-nitro-enclaves-cose = { version = "0.5.0", git = "https://github.com/fortanix/aws-nitro-enclaves-cose.git", branch = "raoul/crypto_abstraction_pinned", default-features = false } -mbedtls = { version = ">=0.8.0, <0.10.0", features = ["rdrand", "std", "time"], default-features = false, optional = true } +mbedtls = { version = "0.10.0", features = ["rdrand", "std", "time"], default-features = false, optional = true } num-bigint = "0.4" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" diff --git a/intel-sgx/dcap-ql/Cargo.toml b/intel-sgx/dcap-ql/Cargo.toml index 08be7a2e..ac759c1a 100644 --- a/intel-sgx/dcap-ql/Cargo.toml +++ b/intel-sgx/dcap-ql/Cargo.toml @@ -45,7 +45,7 @@ byteorder = "1.1.0" # Unlicense/MIT failure = "0.1.1" # MIT/Apache-2.0 lazy_static = "1" # MIT/Apache-2.0 libc = { version = "0.2", optional = true } # MIT/Apache-2.0 -mbedtls = { version = ">=0.8.0, <0.10.0", default-features = false, feature = ["std"], optional = true } +mbedtls = { version = "0.10.0", default-features = false, feature = ["std"], optional = true } num = { version = "0.2", optional = true } num-derive = "0.2" # MIT/Apache-2.0 num-traits = "0.2" # MIT/Apache-2.0 @@ -53,7 +53,7 @@ serde = { version = "1.0.104", features = ["derive"], optional = true } # MIT/Ap yasna = { version = "0.3", features = ["num-bigint", "bit-vec"], optional = true } [dev-dependencies] -mbedtls = { version = ">=0.8.0, <0.10.0" } +mbedtls = { version = "0.10.0" } report-test = { version = "0.3.1", path = "../report-test" } sgxs = { version = "0.7.0", path = "../sgxs" } serde = { version = "1.0.104", features = ["derive"] } diff --git a/intel-sgx/ias/Cargo.toml b/intel-sgx/ias/Cargo.toml index 238c0279..125975a1 100644 --- a/intel-sgx/ias/Cargo.toml +++ b/intel-sgx/ias/Cargo.toml @@ -20,7 +20,7 @@ serde_json = { version = "1", optional = true } serde = { version = "1.0.7", features = ["derive"] } url = "2.2" -mbedtls = { version = ">=0.8.0, <0.10.0", features = ["std"], default-features = false, optional = true } +mbedtls = { version = "0.10.0", features = ["std"], default-features = false, optional = true } pkix = ">=0.1.1, <0.3.0" sgx-isa = { version = "0.4", path = "../sgx-isa" } diff --git a/intel-sgx/sgx-isa/Cargo.toml b/intel-sgx/sgx-isa/Cargo.toml index 9292f697..f4eee0ab 100644 --- a/intel-sgx/sgx-isa/Cargo.toml +++ b/intel-sgx/sgx-isa/Cargo.toml @@ -17,7 +17,7 @@ categories = ["hardware-support"] [dev-dependencies] # External dependencies -mbedtls = { version = ">=0.8.0, <0.10.0", default-features = false, features = ["std"] } +mbedtls = { version = "0.10.0", default-features = false, features = ["std"] } [dependencies] # External dependencies From 768b627a41344b4b7ae6aa9d082475c9837d2561 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Tue, 25 Apr 2023 13:44:16 -0700 Subject: [PATCH 15/19] update Cargo.lock --- Cargo.lock | 280 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 183 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 811400b9..491a2e3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,13 +38,23 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.14" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] +[[package]] +name = "annotate-snippets" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b9d411ecbaf79885c6df4d75fff75858d5995ff25385657a28af47e82f9c36" +dependencies = [ + "unicode-width", + "yansi-term", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -192,7 +202,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" dependencies = [ "bitflags", - "cexpr", + "cexpr 0.4.0", "clang-sys", "clap", "env_logger 0.8.4", @@ -200,14 +210,38 @@ dependencies = [ "lazycell", "log 0.4.14", "peeking_take_while", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "regex", "rustc-hash", "shlex", "which 3.1.1", ] +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "annotate-snippets", + "bitflags", + "cexpr 0.6.0", + "clang-sys", + "lazy_static", + "lazycell", + "log 0.4.14", + "peeking_take_while", + "prettyplease", + "proc-macro2 1.0.60", + "quote 1.0.28", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.18", + "which 4.4.0", +] + [[package]] name = "bit-vec" version = "0.5.1" @@ -344,7 +378,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" dependencies = [ - "nom", + "nom 5.1.2", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", ] [[package]] @@ -684,8 +727,8 @@ checksum = "8e91455b86830a1c21799d94524df0845183fa55bafd9aa137b01c7d1065fa36" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "strsim 0.10.0", "syn 1.0.81", ] @@ -697,7 +740,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" dependencies = [ "darling_core", - "quote 1.0.10", + "quote 1.0.28", "syn 1.0.81", ] @@ -1022,8 +1065,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "rustversion", "syn 1.0.81", "synstructure", @@ -1054,8 +1097,8 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", "synstructure", ] @@ -1291,8 +1334,8 @@ checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" dependencies = [ "autocfg 1.0.1", "proc-macro-hack", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -1992,9 +2035,8 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "mbedtls" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f2c88dbe2fcc6fddc0dc33eb2694471fef46f48b2081996335adb2f8085c53" +version = "0.10.0" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" dependencies = [ "bitflags", "byteorder 1.3.4", @@ -2010,37 +2052,37 @@ dependencies = [ [[package]] name = "mbedtls-platform-support" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85be113c8c2dc54cc6a5fba20130c7128f78dcb3ed0bb6797427333fa8cdb54d" +version = "0.2.0" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" dependencies = [ "cc", "cfg-if 1.0.0", "chrono", "mbedtls-sys-auto", + "once_cell", + "zeroize", ] [[package]] name = "mbedtls-sys-auto" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2231108271d9a10052178d940926baf24b57a2eb1703732faae387592dd6ac3" +version = "3.5.0-alpha.1+0b3de6f" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" dependencies = [ - "bindgen", + "bindgen 0.65.1", "cc", "cfg-if 1.0.0", "cmake", "lazy_static", "libc", - "quote 1.0.10", + "quote 1.0.28", "syn 1.0.81", ] [[package]] name = "memchr" -version = "2.3.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -2097,6 +2139,12 @@ dependencies = [ "unicase 2.6.0", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -2246,7 +2294,7 @@ version = "1.1.0" source = "git+https://github.com/fortanix/aws-nitro-enclaves-cli.git?branch=main#93193b1317c544ff07dd3ddcc6e126f847449cbb" dependencies = [ "aws-nitro-enclaves-cose 0.1.0", - "bindgen", + "bindgen 0.58.1", "chrono", "clap", "eif_defs", @@ -2334,6 +2382,16 @@ dependencies = [ "version_check 0.9.2", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nsm" version = "0.1.0" @@ -2447,8 +2505,8 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -2512,9 +2570,9 @@ checksum = "37fd5004feb2ce328a52b0b3d01dbf4ffff72583493900ed15f22d4111c51693" [[package]] name = "once_cell" -version = "1.9.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -2732,8 +2790,8 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -2743,8 +2801,8 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -2802,6 +2860,16 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" +[[package]] +name = "prettyplease" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" +dependencies = [ + "proc-macro2 1.0.60", + "syn 2.0.18", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2809,8 +2877,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", "version_check 0.9.2", ] @@ -2821,8 +2889,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "version_check 0.9.2", ] @@ -2849,11 +2917,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.32" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ - "unicode-xid 0.2.1", + "unicode-ident", ] [[package]] @@ -2887,8 +2955,8 @@ version = "2.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6653d384a260fedff0a466e894e05c5b8d75e261a14e9f93e81e43ef86cad23" dependencies = [ - "log 0.3.9", - "which 4.0.2", + "log 0.4.14", + "which 4.4.0", ] [[package]] @@ -2933,11 +3001,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.10" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ - "proc-macro2 1.0.32", + "proc-macro2 1.0.60", ] [[package]] @@ -3126,14 +3194,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.1" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -3144,9 +3211,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "remove_dir_all" @@ -3297,8 +3364,8 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9bdc5e856e51e685846fb6c13a1f5e5432946c2c90501bdc76a1319f19e29da" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -3432,8 +3499,8 @@ name = "serde_derive" version = "1.0.130" source = "git+https://github.com/fortanix/serde.git?branch=master#80449547025fc4a016a333e96c0cdaf7e4a96f67" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -3443,8 +3510,8 @@ version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -3474,8 +3541,8 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "98d0516900518c29efa217c298fa1f4e6c6ffc85ae29fd7f4ee48f176e1a9ed5" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -3521,8 +3588,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48b35457e9d855d3dc05ef32a73e0df1e2c0fd72c38796a4ee909160c8eeec2" dependencies = [ "darling", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -3786,19 +3853,30 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "unicode-xid 0.2.1", ] +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2 1.0.60", + "quote 1.0.28", + "unicode-ident", +] + [[package]] name = "synstructure" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", "unicode-xid 0.2.1", ] @@ -3871,20 +3949,11 @@ version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - [[package]] name = "time" version = "0.1.44" @@ -4018,8 +4087,8 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -4029,8 +4098,8 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -4190,8 +4259,8 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", ] @@ -4273,6 +4342,12 @@ dependencies = [ "matches", ] +[[package]] +name = "unicode-ident" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" + [[package]] name = "unicode-normalization" version = "0.1.13" @@ -4482,8 +4557,8 @@ dependencies = [ "bumpalo", "lazy_static", "log 0.4.14", - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", "wasm-bindgen-shared", ] @@ -4506,7 +4581,7 @@ version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" dependencies = [ - "quote 1.0.10", + "quote 1.0.28", "wasm-bindgen-macro-support", ] @@ -4516,8 +4591,8 @@ version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" dependencies = [ - "proc-macro2 1.0.32", - "quote 1.0.10", + "proc-macro2 1.0.60", + "quote 1.0.28", "syn 1.0.81", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -4550,12 +4625,13 @@ dependencies = [ [[package]] name = "which" -version = "4.0.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87c14ef7e1b8b8ecfc75d5eca37949410046e66f15d185c01d70824f1f8111ef" +checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" dependencies = [ + "either", "libc", - "thiserror", + "once_cell", ] [[package]] @@ -4662,6 +4738,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71" +[[package]] +name = "yansi-term" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "yasna" version = "0.2.2" @@ -4697,7 +4782,8 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f1bc8a6b2005884962297587045002d8cfb8dcec9db332f4ca216ddc5de82c5" -[[patch.unused]] -name = "mbedtls" -version = "0.10.0" -source = "git+https://github.com/fortanix/rust-mbedtls?branch=master#ce358f430258a5514c8b699875c90705d7622d96" +[[package]] +name = "zeroize" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" From 5cc82bc50f164133467e7734fde61bd517d7a6a9 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Thu, 8 Jun 2023 22:28:33 -0700 Subject: [PATCH 16/19] fix(nitro-attestation-verify): update error & fix tests --- .../aws-nitro-enclaves/nitro-attestation-verify/src/lib.rs | 4 ++-- .../nitro-attestation-verify/src/mbedtls.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/lib.rs b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/lib.rs index b94f09f0..bd57275c 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/lib.rs +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/lib.rs @@ -328,8 +328,8 @@ mod tests { // Not After : Sep 9 13:19:20 2021 GMT static ref PROPER_TOKEN : Vec = include_bytes!("../data/request_proper.bin").to_vec(); static ref PROPER_VALIDITY: (DateTime, DateTime) = (Utc.ymd(2021, 9, 9).and_hms(10, 19, 19), Utc.ymd(2021, 9, 9).and_hms(13, 19, 21)); - static ref NOT_VALID_YET_ERR: NitroError = NitroError::CertificateVerifyFailure("Certificate verify failure: X509CertVerifyFailed, The certificate validity starts in the future\n".to_string()); - static ref EXPIRED_ERR: NitroError = NitroError::CertificateVerifyFailure("Certificate verify failure: X509CertVerifyFailed, The certificate validity has expired\n".to_string()); + static ref NOT_VALID_YET_ERR: NitroError = NitroError::CertificateVerifyFailure("Certificate verify failure: HighLevel(X509CertVerifyFailed), The certificate validity starts in the future\n".to_string()); + static ref EXPIRED_ERR: NitroError = NitroError::CertificateVerifyFailure("Certificate verify failure: HighLevel(X509CertVerifyFailed), The certificate validity has expired\n".to_string()); static ref TAMPERED_SIGNATURE : Vec = include_bytes!("../data/tampered_signature.bin").to_vec(); static ref TAMPERED_CERTIFICATE : Vec = include_bytes!("../data/tampered_certificate.bin").to_vec(); } diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/mbedtls.rs b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/mbedtls.rs index 5f2ac2a1..f84d3ef7 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/mbedtls.rs +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/src/mbedtls.rs @@ -97,7 +97,7 @@ impl SigningPublicKey for WrappedCert { // We'll throw error if signature verify does not work match pk.verify(*md, &digest, &sig) { Ok(_) => Ok(true), - Err(mbedtls::Error::EcpVerifyFailed) => Ok(false), + Err(mbedtls::Error::HighLevel(mbedtls::error::HiError::EcpVerifyFailed)) => Ok(false), Err(e) => Err(CoseError::SignatureError(Box::new(e))), } } From dfd59eec02933a679b04dabe04af577ce488b863 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Thu, 8 Jun 2023 23:06:37 -0700 Subject: [PATCH 17/19] ci: use nightly-2023-05-07 rust --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 52693a75..a3bda442 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ matrix: - clang-11 - musl-tools rust: - - nightly + - nightly-2023-05-07 env: - RUST_BACKTRACE=1 - CFLAGS_x86_64_fortanix_unknown_sgx="-isystem/usr/include/x86_64-linux-gnu -mlvi-hardening -mllvm -x86-experimental-lvi-inline-asm-hardening" From 9a13c59ef5d19806f4254f9c03619105f18f2e0a Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Mon, 26 Jun 2023 21:41:38 -0700 Subject: [PATCH 18/19] build: upgrade mbedtls to 0.11 --- em-app/Cargo.toml | 2 +- em-app/examples/get-certificate/Cargo.toml | 2 +- em-app/examples/harmonize/Cargo.toml | 2 +- .../aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml | 2 +- intel-sgx/dcap-ql/Cargo.toml | 4 ++-- intel-sgx/ias/Cargo.toml | 2 +- intel-sgx/sgx-isa/Cargo.toml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/em-app/Cargo.toml b/em-app/Cargo.toml index edbcb998..790ec4a8 100644 --- a/em-app/Cargo.toml +++ b/em-app/Cargo.toml @@ -14,7 +14,7 @@ b64-ct = "0.1.0" em-client = { version = "3.0.0", default-features = false, features = ["client"] } em-node-agent-client = "1.0.0" hyper = { version = "0.10", default-features = false } -mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.11.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } pkix = ">=0.1.2, <0.3.0" rustc-serialize = "0.3.24" diff --git a/em-app/examples/get-certificate/Cargo.toml b/em-app/examples/get-certificate/Cargo.toml index 47a7dbbd..99902194 100644 --- a/em-app/examples/get-certificate/Cargo.toml +++ b/em-app/examples/get-certificate/Cargo.toml @@ -7,5 +7,5 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.11.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0" diff --git a/em-app/examples/harmonize/Cargo.toml b/em-app/examples/harmonize/Cargo.toml index 156a54c9..2bcad53e 100644 --- a/em-app/examples/harmonize/Cargo.toml +++ b/em-app/examples/harmonize/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" [dependencies] em-app = { path = "../../" } -mbedtls = { version = "0.10.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } +mbedtls = { version = "0.11.0", features = [ "rdrand", "std", "force_aesni_support", "mpi_force_c_code" ], default-features = false } serde_json = "1.0.62" serde = "1.0.123" serde_derive = "1.0.123" diff --git a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml index 6c13f8e8..b26b4300 100644 --- a/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml +++ b/fortanix-vme/aws-nitro-enclaves/nitro-attestation-verify/Cargo.toml @@ -10,7 +10,7 @@ serde_cbor = "0.11" # Required until PR36 is accepted # https://github.com/awslabs/aws-nitro-enclaves-cose/pull/36 aws-nitro-enclaves-cose = { version = "0.5.0", git = "https://github.com/fortanix/aws-nitro-enclaves-cose.git", branch = "raoul/crypto_abstraction_pinned", default-features = false } -mbedtls = { version = "0.10.0", features = ["rdrand", "std", "time"], default-features = false, optional = true } +mbedtls = { version = "0.11.0", features = ["rdrand", "std", "time"], default-features = false, optional = true } num-bigint = "0.4" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" diff --git a/intel-sgx/dcap-ql/Cargo.toml b/intel-sgx/dcap-ql/Cargo.toml index ac759c1a..cd793e05 100644 --- a/intel-sgx/dcap-ql/Cargo.toml +++ b/intel-sgx/dcap-ql/Cargo.toml @@ -45,7 +45,7 @@ byteorder = "1.1.0" # Unlicense/MIT failure = "0.1.1" # MIT/Apache-2.0 lazy_static = "1" # MIT/Apache-2.0 libc = { version = "0.2", optional = true } # MIT/Apache-2.0 -mbedtls = { version = "0.10.0", default-features = false, feature = ["std"], optional = true } +mbedtls = { version = "0.11.0", default-features = false, feature = ["std"], optional = true } num = { version = "0.2", optional = true } num-derive = "0.2" # MIT/Apache-2.0 num-traits = "0.2" # MIT/Apache-2.0 @@ -53,7 +53,7 @@ serde = { version = "1.0.104", features = ["derive"], optional = true } # MIT/Ap yasna = { version = "0.3", features = ["num-bigint", "bit-vec"], optional = true } [dev-dependencies] -mbedtls = { version = "0.10.0" } +mbedtls = { version = "0.11.0" } report-test = { version = "0.3.1", path = "../report-test" } sgxs = { version = "0.7.0", path = "../sgxs" } serde = { version = "1.0.104", features = ["derive"] } diff --git a/intel-sgx/ias/Cargo.toml b/intel-sgx/ias/Cargo.toml index 125975a1..8a828d62 100644 --- a/intel-sgx/ias/Cargo.toml +++ b/intel-sgx/ias/Cargo.toml @@ -20,7 +20,7 @@ serde_json = { version = "1", optional = true } serde = { version = "1.0.7", features = ["derive"] } url = "2.2" -mbedtls = { version = "0.10.0", features = ["std"], default-features = false, optional = true } +mbedtls = { version = "0.11.0", features = ["std"], default-features = false, optional = true } pkix = ">=0.1.1, <0.3.0" sgx-isa = { version = "0.4", path = "../sgx-isa" } diff --git a/intel-sgx/sgx-isa/Cargo.toml b/intel-sgx/sgx-isa/Cargo.toml index f4eee0ab..3996177a 100644 --- a/intel-sgx/sgx-isa/Cargo.toml +++ b/intel-sgx/sgx-isa/Cargo.toml @@ -17,7 +17,7 @@ categories = ["hardware-support"] [dev-dependencies] # External dependencies -mbedtls = { version = "0.10.0", default-features = false, features = ["std"] } +mbedtls = { version = "0.11.0", default-features = false, features = ["std"] } [dependencies] # External dependencies From 702c91c6687dbb06b55badf7e5058fdaa619fb36 Mon Sep 17 00:00:00 2001 From: Yuxiang Cao Date: Mon, 26 Jun 2023 21:41:50 -0700 Subject: [PATCH 19/19] build: update lock files --- Cargo.lock | 80 ++++--------------- .../tests/outgoing_connection/Cargo.lock | 7 -- 2 files changed, 15 insertions(+), 72 deletions(-) delete mode 100644 fortanix-vme/tests/outgoing_connection/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 491a2e3e..7e31ecdf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -195,29 +195,6 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" -[[package]] -name = "bindgen" -version = "0.58.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" -dependencies = [ - "bitflags", - "cexpr 0.4.0", - "clang-sys", - "clap", - "env_logger 0.8.4", - "lazy_static", - "lazycell", - "log 0.4.14", - "peeking_take_while", - "proc-macro2 1.0.60", - "quote 1.0.28", - "regex", - "rustc-hash", - "shlex", - "which 3.1.1", -] - [[package]] name = "bindgen" version = "0.65.1" @@ -226,7 +203,7 @@ checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" dependencies = [ "annotate-snippets", "bitflags", - "cexpr 0.6.0", + "cexpr", "clang-sys", "lazy_static", "lazycell", @@ -239,7 +216,7 @@ dependencies = [ "rustc-hash", "shlex", "syn 2.0.18", - "which 4.4.0", + "which", ] [[package]] @@ -372,22 +349,13 @@ version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" -[[package]] -name = "cexpr" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" -dependencies = [ - "nom 5.1.2", -] - [[package]] name = "cexpr" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -2035,8 +2003,8 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "mbedtls" -version = "0.10.0" -source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" +version = "0.11.0" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#43a96b0afe3c60fc77f34ac346c91e21fa01ee0f" dependencies = [ "bitflags", "byteorder 1.3.4", @@ -2044,7 +2012,7 @@ dependencies = [ "cfg-if 1.0.0", "mbedtls-platform-support", "mbedtls-sys-auto", - "rs-libc 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rs-libc 0.2.4", "serde", "serde_derive 1.0.132", "yasna 0.2.2", @@ -2052,8 +2020,8 @@ dependencies = [ [[package]] name = "mbedtls-platform-support" -version = "0.2.0" -source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" +version = "0.3.0" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#43a96b0afe3c60fc77f34ac346c91e21fa01ee0f" dependencies = [ "cc", "cfg-if 1.0.0", @@ -2066,9 +2034,9 @@ dependencies = [ [[package]] name = "mbedtls-sys-auto" version = "3.5.0-alpha.1+0b3de6f" -source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#d054c6c8530f5df40a89157f9161abaa5ce0cf20" +source = "git+https://github.com/fortanix/rust-mbedtls?branch=yx/upgrade-mbedtls#43a96b0afe3c60fc77f34ac346c91e21fa01ee0f" dependencies = [ - "bindgen 0.65.1", + "bindgen", "cc", "cfg-if 1.0.0", "cmake", @@ -2294,7 +2262,7 @@ version = "1.1.0" source = "git+https://github.com/fortanix/aws-nitro-enclaves-cli.git?branch=main#93193b1317c544ff07dd3ddcc6e126f847449cbb" dependencies = [ "aws-nitro-enclaves-cose 0.1.0", - "bindgen 0.58.1", + "bindgen", "chrono", "clap", "eif_defs", @@ -2372,16 +2340,6 @@ dependencies = [ "memoffset 0.6.4", ] -[[package]] -name = "nom" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" -dependencies = [ - "memchr", - "version_check 0.9.2", -] - [[package]] name = "nom" version = "7.1.3" @@ -2956,7 +2914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6653d384a260fedff0a466e894e05c5b8d75e261a14e9f93e81e43ef86cad23" dependencies = [ "log 0.4.14", - "which 4.4.0", + "which", ] [[package]] @@ -3313,11 +3271,12 @@ dependencies = [ [[package]] name = "rs-libc" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914c985b921cf571d950d17ca33221ed54fed3c2001a329ee6fd5b15dd433260" +checksum = "683e8c8e8aac6ffa4b2287bac3c69575d5346accac4f218ae1e084303bb174ca" dependencies = [ "cc", + "zeroize", ] [[package]] @@ -4614,15 +4573,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "which" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" -dependencies = [ - "libc", -] - [[package]] name = "which" version = "4.4.0" diff --git a/fortanix-vme/tests/outgoing_connection/Cargo.lock b/fortanix-vme/tests/outgoing_connection/Cargo.lock deleted file mode 100644 index aa13ffec..00000000 --- a/fortanix-vme/tests/outgoing_connection/Cargo.lock +++ /dev/null @@ -1,7 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "tcp_connection" -version = "0.1.0"