diff --git a/src/librustc_mir/const_eval/error.rs b/src/librustc_mir/const_eval/error.rs index f7e28cf8d8c2f..3c3618f390c61 100644 --- a/src/librustc_mir/const_eval/error.rs +++ b/src/librustc_mir/const_eval/error.rs @@ -50,7 +50,7 @@ impl Error for ConstEvalErrKind {} /// Turn an interpreter error into something to report to the user. /// As a side-effect, if RUSTC_CTFE_BACKTRACE is set, this prints the backtrace. /// Should be called only if the error is actually going to to be reported! -pub fn error_to_const_error<'mir, 'tcx, M: Machine<'mir, 'tcx>>( +pub fn error_to_const_error<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>( ecx: &InterpCx<'mir, 'tcx, M>, mut error: InterpErrorInfo<'tcx>, ) -> ConstEvalErr<'tcx> { diff --git a/src/librustc_mir/const_eval/machine.rs b/src/librustc_mir/const_eval/machine.rs index e53ca6b31bb67..30990853b516a 100644 --- a/src/librustc_mir/const_eval/machine.rs +++ b/src/librustc_mir/const_eval/machine.rs @@ -19,7 +19,7 @@ use crate::interpret::{ use super::error::*; -impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> { +impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> { /// Evaluate a const function where all arguments (if any) are zero-sized types. /// The evaluation is memoized thanks to the query system. /// @@ -86,12 +86,15 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> { } /// Extra machine state for CTFE, and the Machine instance -pub struct CompileTimeInterpreter { +pub struct CompileTimeInterpreter<'mir, 'tcx> { /// For now, the number of terminators that can be evaluated before we throw a resource /// exhuastion error. /// /// Setting this to `0` disables the limit and allows the interpreter to run forever. pub steps_remaining: usize, + + /// The virtual call stack. + pub(crate) stack: Vec>, } #[derive(Copy, Clone, Debug)] @@ -100,9 +103,9 @@ pub struct MemoryExtra { pub(super) can_access_statics: bool, } -impl CompileTimeInterpreter { +impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> { pub(super) fn new(const_eval_limit: usize) -> Self { - CompileTimeInterpreter { steps_remaining: const_eval_limit } + CompileTimeInterpreter { steps_remaining: const_eval_limit, stack: Vec::new() } } } @@ -156,7 +159,8 @@ impl interpret::AllocMap for FxHashMap { } } -crate type CompileTimeEvalContext<'mir, 'tcx> = InterpCx<'mir, 'tcx, CompileTimeInterpreter>; +crate type CompileTimeEvalContext<'mir, 'tcx> = + InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>; impl interpret::MayLeak for ! { #[inline(always)] @@ -166,7 +170,7 @@ impl interpret::MayLeak for ! { } } -impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter { +impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, 'tcx> { type MemoryKind = !; type PointerTag = (); type ExtraFnVal = !; @@ -349,6 +353,20 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter { Ok(frame) } + #[inline(always)] + fn stack( + ecx: &'a InterpCx<'mir, 'tcx, Self>, + ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { + &ecx.machine.stack + } + + #[inline(always)] + fn stack_mut( + ecx: &'a mut InterpCx<'mir, 'tcx, Self>, + ) -> &'a mut Vec> { + &mut ecx.machine.stack + } + fn before_access_global( memory_extra: &MemoryExtra, alloc_id: AllocId, diff --git a/src/librustc_mir/interpret/cast.rs b/src/librustc_mir/interpret/cast.rs index 2fcdc9ca5d6ce..0e01652bc9002 100644 --- a/src/librustc_mir/interpret/cast.rs +++ b/src/librustc_mir/interpret/cast.rs @@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Ty, TypeAndMut, TypeFoldable}; use rustc_span::symbol::sym; use rustc_target::abi::{LayoutOf, Size, Variants}; -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn cast( &mut self, src: OpTy<'tcx, M::PointerTag>, diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs index 06dee62b3e70b..b2a041874d09d 100644 --- a/src/librustc_mir/interpret/eval_context.rs +++ b/src/librustc_mir/interpret/eval_context.rs @@ -28,6 +28,8 @@ use crate::util::storage::AlwaysLiveLocals; pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// Stores the `Machine` instance. + /// + /// Note: the stack is provided by the machine. pub machine: M, /// The results of the type checker, from rustc. @@ -39,9 +41,6 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> { /// The virtual memory system. pub memory: Memory<'mir, 'tcx, M>, - /// The virtual call stack. - pub(crate) stack: Vec>, - /// A cache for deduplicating vtables pub(super) vtables: FxHashMap<(Ty<'tcx>, Option>), Pointer>, @@ -297,7 +296,7 @@ pub(super) fn from_known_layout<'tcx>( } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn new( tcx: TyCtxtAt<'tcx>, param_env: ty::ParamEnv<'tcx>, @@ -309,7 +308,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { tcx, param_env, memory: Memory::new(tcx, memory_extra), - stack: Vec::new(), vtables: FxHashMap::default(), } } @@ -349,24 +347,32 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } #[inline(always)] - pub fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] { - &self.stack + pub(crate) fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] { + M::stack(self) + } + + #[inline(always)] + pub(crate) fn stack_mut( + &mut self, + ) -> &mut Vec> { + M::stack_mut(self) } #[inline(always)] - pub fn cur_frame(&self) -> usize { - assert!(!self.stack.is_empty()); - self.stack.len() - 1 + pub fn frame_idx(&self) -> usize { + let stack = self.stack(); + assert!(!stack.is_empty()); + stack.len() - 1 } #[inline(always)] pub fn frame(&self) -> &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> { - self.stack.last().expect("no call frames exist") + self.stack().last().expect("no call frames exist") } #[inline(always)] pub fn frame_mut(&mut self) -> &mut Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> { - self.stack.last_mut().expect("no call frames exist") + self.stack_mut().last_mut().expect("no call frames exist") } #[inline(always)] @@ -596,8 +602,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { return_place: Option>, return_to_block: StackPopCleanup, ) -> InterpResult<'tcx> { - if !self.stack.is_empty() { - info!("PAUSING({}) {}", self.cur_frame(), self.frame().instance); + if !self.stack().is_empty() { + info!("PAUSING({}) {}", self.frame_idx(), self.frame().instance); } ::log_settings::settings().indentation += 1; @@ -615,7 +621,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { extra: (), }; let frame = M::init_frame_extra(self, pre_frame)?; - self.stack.push(frame); + self.stack_mut().push(frame); // don't allocate at all for trivial constants if body.local_decls.len() > 1 { @@ -648,9 +654,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } M::after_stack_push(self)?; - info!("ENTERING({}) {}", self.cur_frame(), self.frame().instance); + info!("ENTERING({}) {}", self.frame_idx(), self.frame().instance); - if self.stack.len() > *self.tcx.sess.recursion_limit.get() { + if self.stack().len() > *self.tcx.sess.recursion_limit.get() { throw_exhaust!(StackFrameLimitReached) } else { Ok(()) @@ -705,7 +711,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx> { info!( "LEAVING({}) {} (unwinding = {})", - self.cur_frame(), + self.frame_idx(), self.frame().instance, unwinding ); @@ -720,7 +726,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ); ::log_settings::settings().indentation -= 1; - let frame = self.stack.pop().expect("tried to pop a stack frame, but there were none"); + let frame = + self.stack_mut().pop().expect("tried to pop a stack frame, but there were none"); // Now where do we jump next? @@ -735,7 +742,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; if !cleanup { - assert!(self.stack.is_empty(), "only the topmost frame should ever be leaked"); + assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked"); assert!(next_block.is_none(), "tried to skip cleanup when we have a next block!"); assert!(!unwinding, "tried to skip cleanup during unwinding"); // Leak the locals, skip validation, skip machine hook. @@ -784,10 +791,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } - if !self.stack.is_empty() { + if !self.stack().is_empty() { info!( "CONTINUING({}) {} (unwinding = {})", - self.cur_frame(), + self.frame_idx(), self.frame().instance, unwinding ); @@ -895,12 +902,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { Place::Local { frame, local } => { let mut allocs = Vec::new(); let mut msg = format!("{:?}", local); - if frame != self.cur_frame() { - write!(msg, " ({} frames up)", self.cur_frame() - frame).unwrap(); + if frame != self.frame_idx() { + write!(msg, " ({} frames up)", self.frame_idx() - frame).unwrap(); } write!(msg, ":").unwrap(); - match self.stack[frame].locals[local].value { + match self.stack()[frame].locals[local].value { LocalValue::Dead => write!(msg, " is dead").unwrap(), LocalValue::Uninitialized => write!(msg, " is uninitialized").unwrap(), LocalValue::Live(Operand::Indirect(mplace)) => match mplace.ptr { diff --git a/src/librustc_mir/interpret/intern.rs b/src/librustc_mir/interpret/intern.rs index b28cd3f0a8dd6..8bc0e2ee6a4c6 100644 --- a/src/librustc_mir/interpret/intern.rs +++ b/src/librustc_mir/interpret/intern.rs @@ -149,7 +149,7 @@ impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> InternVisitor<'rt, 'mir } } -impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> +impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> for InternVisitor<'rt, 'mir, 'tcx, M> { type V = MPlaceTy<'tcx>; @@ -286,7 +286,10 @@ pub fn intern_const_alloc_recursive>( intern_kind: InternKind, ret: MPlaceTy<'tcx>, ignore_interior_mut_in_const_validation: bool, -) -> InterpResult<'tcx> { +) -> InterpResult<'tcx> +where + 'tcx: 'mir, +{ let tcx = ecx.tcx; let (base_mutability, base_intern_mode) = match intern_kind { // `static mut` doesn't care about interior mutability, it's mutable anyway diff --git a/src/librustc_mir/interpret/intrinsics.rs b/src/librustc_mir/interpret/intrinsics.rs index d976df82577ea..c8bf328ce8eee 100644 --- a/src/librustc_mir/interpret/intrinsics.rs +++ b/src/librustc_mir/interpret/intrinsics.rs @@ -73,7 +73,7 @@ crate fn eval_nullary_intrinsic<'tcx>( }) } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Returns `true` if emulation happened. pub fn emulate_intrinsic( &mut self, diff --git a/src/librustc_mir/interpret/intrinsics/caller_location.rs b/src/librustc_mir/interpret/intrinsics/caller_location.rs index 754f45d9ec05b..91b046d7bb264 100644 --- a/src/librustc_mir/interpret/intrinsics/caller_location.rs +++ b/src/librustc_mir/interpret/intrinsics/caller_location.rs @@ -10,11 +10,11 @@ use crate::interpret::{ MPlaceTy, MemoryKind, Scalar, }; -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a /// frame which is not `#[track_caller]`. crate fn find_closest_untracked_caller_location(&self) -> Span { - self.stack + self.stack() .iter() .rev() // Find first non-`#[track_caller]` frame. diff --git a/src/librustc_mir/interpret/machine.rs b/src/librustc_mir/interpret/machine.rs index 8bf8d904cb29e..dfdd95c95a364 100644 --- a/src/librustc_mir/interpret/machine.rs +++ b/src/librustc_mir/interpret/machine.rs @@ -230,6 +230,34 @@ pub trait Machine<'mir, 'tcx>: Sized { id } + /// Called when converting a `ty::Const` to an operand (in + /// `eval_const_to_op`). + /// + /// Miri uses this callback for creating per thread allocations for thread + /// locals. In Rust, one way of creating a thread local is by marking a + /// static with `#[thread_local]`. On supported platforms this gets + /// translated to a LLVM thread local for which LLVM automatically ensures + /// that each thread gets its own copy. Since LLVM automatically handles + /// thread locals, the Rust compiler just treats thread local statics as + /// regular statics even though accessing a thread local static should be an + /// effectful computation that depends on the current thread. The long term + /// plan is to change MIR to make accesses to thread locals explicit + /// (https://github.com/rust-lang/rust/issues/70685). While the issue 70685 + /// is not fixed, our current workaround in Miri is to use this function to + /// make per-thread copies of thread locals. Please note that we cannot make + /// these copies in `canonical_alloc_id` because that is too late: for + /// example, if one created a pointer in thread `t1` to a thread local and + /// sent it to another thread `t2`, resolving the access in + /// `canonical_alloc_id` would result in pointer pointing to `t2`'s thread + /// local and not `t1` as it should. + #[inline] + fn adjust_global_const( + _ecx: &InterpCx<'mir, 'tcx, Self>, + val: mir::interpret::ConstValue<'tcx>, + ) -> InterpResult<'tcx, mir::interpret::ConstValue<'tcx>> { + Ok(val) + } + /// Called to initialize the "extra" state of an allocation and make the pointers /// it contains (in relocations) tagged. The way we construct allocations is /// to always first construct it without extra and then add the extra. @@ -285,6 +313,16 @@ pub trait Machine<'mir, 'tcx>: Sized { frame: Frame<'mir, 'tcx, Self::PointerTag>, ) -> InterpResult<'tcx, Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>; + /// Borrow the current thread's stack. + fn stack( + ecx: &'a InterpCx<'mir, 'tcx, Self>, + ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>]; + + /// Mutably borrow the current thread's stack. + fn stack_mut( + ecx: &'a mut InterpCx<'mir, 'tcx, Self>, + ) -> &'a mut Vec>; + /// Called immediately after a stack frame got pushed and its locals got initialized. fn after_stack_push(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> { Ok(()) diff --git a/src/librustc_mir/interpret/operand.rs b/src/librustc_mir/interpret/operand.rs index 2317a1fea0792..31e6fbdceee4a 100644 --- a/src/librustc_mir/interpret/operand.rs +++ b/src/librustc_mir/interpret/operand.rs @@ -209,7 +209,7 @@ impl<'tcx, Tag: Copy> ImmTy<'tcx, Tag> { } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Normalice `place.ptr` to a `Pointer` if this is a place and not a ZST. /// Can be helpful to avoid lots of `force_ptr` calls later, if this place is used a lot. #[inline] @@ -440,7 +440,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> { let op = match *place { Place::Ptr(mplace) => Operand::Indirect(mplace), - Place::Local { frame, local } => *self.access_local(&self.stack[frame], local, None)?, + Place::Local { frame, local } => { + *self.access_local(&self.stack()[frame], local, None)? + } }; Ok(OpTy { op, layout: place.layout }) } @@ -528,6 +530,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // potentially requiring the current static to be evaluated again. This is not a // problem here, because we are building an operand which means an actual read is // happening. + // + // The machine callback `adjust_global_const` below is guaranteed to + // be called for all constants because `const_eval` calls + // `eval_const_to_op` recursively. return Ok(self.const_eval(GlobalId { instance, promoted }, val.ty)?); } ty::ConstKind::Infer(..) @@ -537,6 +543,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } ty::ConstKind::Value(val_val) => val_val, }; + // This call allows the machine to create fresh allocation ids for + // thread-local statics (see the `adjust_global_const` function + // documentation). + let val_val = M::adjust_global_const(self, val_val)?; // Other cases need layout. let layout = from_known_layout(self.tcx, layout, || self.layout_of(val.ty))?; let op = match val_val { diff --git a/src/librustc_mir/interpret/operator.rs b/src/librustc_mir/interpret/operator.rs index 0aa7e98f3edfa..d651267f82b79 100644 --- a/src/librustc_mir/interpret/operator.rs +++ b/src/librustc_mir/interpret/operator.rs @@ -9,7 +9,7 @@ use rustc_target::abi::LayoutOf; use super::{ImmTy, Immediate, InterpCx, Machine, PlaceTy}; -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Applies the binary operation `op` to the two operands and writes a tuple of the result /// and a boolean signifying the potential overflow to the destination. pub fn binop_with_overflow( @@ -45,7 +45,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { fn binary_char_op( &self, bin_op: mir::BinOp, diff --git a/src/librustc_mir/interpret/place.rs b/src/librustc_mir/interpret/place.rs index 71275452fb688..af3a9da2f6ca6 100644 --- a/src/librustc_mir/interpret/place.rs +++ b/src/librustc_mir/interpret/place.rs @@ -283,7 +283,7 @@ impl<'tcx, Tag: ::std::fmt::Debug> PlaceTy<'tcx, Tag> { } // separating the pointer tag for `impl Trait`, see https://github.com/rust-lang/rust/issues/54385 -impl<'mir, 'tcx, Tag, M> InterpCx<'mir, 'tcx, M> +impl<'mir, 'tcx: 'mir, Tag, M> InterpCx<'mir, 'tcx, M> where // FIXME: Working around https://github.com/rust-lang/rust/issues/54385 Tag: ::std::fmt::Debug + Copy + Eq + Hash + 'static, @@ -662,7 +662,7 @@ where } local => PlaceTy { // This works even for dead/uninitialized locals; we check further when writing - place: Place::Local { frame: self.cur_frame(), local }, + place: Place::Local { frame: self.frame_idx(), local }, layout: self.layout_of_local(self.frame(), local, None)?, }, }; @@ -755,7 +755,7 @@ where // but not factored as a separate function. let mplace = match dest.place { Place::Local { frame, local } => { - match self.stack[frame].locals[local].access_mut()? { + match self.stack_mut()[frame].locals[local].access_mut()? { Ok(local) => { // Local can be updated in-place. *local = LocalValue::Live(Operand::Immediate(src)); @@ -985,14 +985,15 @@ where ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, Option)> { let (mplace, size) = match place.place { Place::Local { frame, local } => { - match self.stack[frame].locals[local].access_mut()? { + match self.stack_mut()[frame].locals[local].access_mut()? { Ok(&mut local_val) => { // We need to make an allocation. // We need the layout of the local. We can NOT use the layout we got, // that might e.g., be an inner field of a struct with `Scalar` layout, // that has different alignment than the outer field. - let local_layout = self.layout_of_local(&self.stack[frame], local, None)?; + let local_layout = + self.layout_of_local(&self.stack()[frame], local, None)?; // We also need to support unsized types, and hence cannot use `allocate`. let (size, align) = self .size_and_align_of(meta, local_layout)? @@ -1008,7 +1009,7 @@ where } // Now we can call `access_mut` again, asserting it goes well, // and actually overwrite things. - *self.stack[frame].locals[local].access_mut().unwrap().unwrap() = + *self.stack_mut()[frame].locals[local].access_mut().unwrap().unwrap() = LocalValue::Live(Operand::Indirect(mplace)); (mplace, Some(size)) } diff --git a/src/librustc_mir/interpret/step.rs b/src/librustc_mir/interpret/step.rs index 37740878f7043..aae708827b953 100644 --- a/src/librustc_mir/interpret/step.rs +++ b/src/librustc_mir/interpret/step.rs @@ -29,7 +29,7 @@ fn binop_right_homogeneous(op: mir::BinOp) -> bool { } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub fn run(&mut self) -> InterpResult<'tcx> { while self.step()? {} Ok(()) @@ -42,7 +42,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// This is marked `#inline(always)` to work around adverserial codegen when `opt-level = 3` #[inline(always)] pub fn step(&mut self) -> InterpResult<'tcx, bool> { - if self.stack.is_empty() { + if self.stack().is_empty() { return Ok(false); } @@ -60,10 +60,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let body = self.body(); let basic_block = &body.basic_blocks()[block]; - let old_frames = self.cur_frame(); + let old_frames = self.frame_idx(); if let Some(stmt) = basic_block.statements.get(stmt_id) { - assert_eq!(old_frames, self.cur_frame()); + assert_eq!(old_frames, self.frame_idx()); self.statement(stmt)?; return Ok(true); } @@ -71,7 +71,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { M::before_terminator(self)?; let terminator = basic_block.terminator(); - assert_eq!(old_frames, self.cur_frame()); + assert_eq!(old_frames, self.frame_idx()); self.terminator(terminator)?; Ok(true) } @@ -84,7 +84,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // Some statements (e.g., box) push new stack frames. // We have to record the stack frame number *before* executing the statement. - let frame_idx = self.cur_frame(); + let frame_idx = self.frame_idx(); match &stmt.kind { Assign(box (place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?, @@ -126,7 +126,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"), } - self.stack[frame_idx].stmt += 1; + self.stack_mut()[frame_idx].stmt += 1; Ok(()) } @@ -278,7 +278,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.set_span(terminator.source_info.span); self.eval_terminator(terminator)?; - if !self.stack.is_empty() { + if !self.stack().is_empty() { if let Some(block) = self.frame().block { info!("// executing {:?}", block); } diff --git a/src/librustc_mir/interpret/terminator.rs b/src/librustc_mir/interpret/terminator.rs index 49fee1bddcb6d..7157225e5c9bb 100644 --- a/src/librustc_mir/interpret/terminator.rs +++ b/src/librustc_mir/interpret/terminator.rs @@ -11,7 +11,7 @@ use super::{ FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, StackPopCleanup, }; -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { pub(super) fn eval_terminator( &mut self, terminator: &mir::Terminator<'tcx>, @@ -52,7 +52,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } Call { ref func, ref args, destination, ref cleanup, .. } => { - let old_stack = self.cur_frame(); + let old_stack = self.frame_idx(); let old_bb = self.frame().block; let func = self.eval_operand(func, None)?; let (fn_val, abi) = match func.layout.ty.kind { @@ -80,7 +80,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self.eval_fn_call(fn_val, abi, &args[..], ret, *cleanup)?; // Sanity-check that `eval_fn_call` either pushed a new frame or // did a jump to another block. - if self.cur_frame() == old_stack && self.frame().block == old_bb { + if self.frame_idx() == old_stack && self.frame().block == old_bb { span_bug!(terminator.source_info.span, "evaluating this call made no progress"); } } @@ -372,7 +372,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { }; match res { Err(err) => { - self.stack.pop(); + self.stack_mut().pop(); Err(err) } Ok(()) => Ok(()), diff --git a/src/librustc_mir/interpret/traits.rs b/src/librustc_mir/interpret/traits.rs index fb9401c7d8f28..7edd787c986bd 100644 --- a/src/librustc_mir/interpret/traits.rs +++ b/src/librustc_mir/interpret/traits.rs @@ -6,7 +6,7 @@ use rustc_target::abi::{Align, HasDataLayout, LayoutOf, Size}; use super::{FnVal, InterpCx, Machine, MemoryKind}; -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { /// Creates a dynamic vtable for the given type and vtable origin. This is used only for /// objects. /// diff --git a/src/librustc_mir/interpret/validity.rs b/src/librustc_mir/interpret/validity.rs index 2dffd78d776bd..e97a39f8c6fec 100644 --- a/src/librustc_mir/interpret/validity.rs +++ b/src/librustc_mir/interpret/validity.rs @@ -178,7 +178,7 @@ struct ValidityVisitor<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> { ecx: &'rt InterpCx<'mir, 'tcx, M>, } -impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> { +impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> { fn aggregate_field_path_elem(&mut self, layout: TyAndLayout<'tcx>, field: usize) -> PathElem { // First, check if we are projecting to a variant. match layout.variants { @@ -610,7 +610,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M } } -impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> +impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> for ValidityVisitor<'rt, 'mir, 'tcx, M> { type V = OpTy<'tcx, M::PointerTag>; @@ -813,7 +813,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { fn validate_operand_internal( &self, op: OpTy<'tcx, M::PointerTag>, diff --git a/src/librustc_mir/interpret/visitor.rs b/src/librustc_mir/interpret/visitor.rs index 00d39452c49d5..903aa377a3d7d 100644 --- a/src/librustc_mir/interpret/visitor.rs +++ b/src/librustc_mir/interpret/visitor.rs @@ -37,7 +37,7 @@ pub trait Value<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy { // Operands and memory-places are both values. // Places in general are not due to `place_field` having to do `force_allocation`. -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> { #[inline(always)] fn layout(&self) -> TyAndLayout<'tcx> { self.layout @@ -75,7 +75,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M:: } } -impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for MPlaceTy<'tcx, M::PointerTag> { +impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> + for MPlaceTy<'tcx, M::PointerTag> +{ #[inline(always)] fn layout(&self) -> TyAndLayout<'tcx> { self.layout diff --git a/src/librustc_mir/transform/const_prop.rs b/src/librustc_mir/transform/const_prop.rs index cf1c70241bc6e..79dba2c5db8fb 100644 --- a/src/librustc_mir/transform/const_prop.rs +++ b/src/librustc_mir/transform/const_prop.rs @@ -158,9 +158,18 @@ impl<'tcx> MirPass<'tcx> for ConstProp { } } -struct ConstPropMachine; +struct ConstPropMachine<'mir, 'tcx> { + /// The virtual call stack. + stack: Vec>, +} -impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine { +impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> { + fn new() -> Self { + Self { stack: Vec::new() } + } +} + +impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> { type MemoryKind = !; type PointerTag = (); type ExtraFnVal = !; @@ -296,11 +305,25 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine { ) -> InterpResult<'tcx, Frame<'mir, 'tcx>> { Ok(frame) } + + #[inline(always)] + fn stack( + ecx: &'a InterpCx<'mir, 'tcx, Self>, + ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { + &ecx.machine.stack + } + + #[inline(always)] + fn stack_mut( + ecx: &'a mut InterpCx<'mir, 'tcx, Self>, + ) -> &'a mut Vec> { + &mut ecx.machine.stack + } } /// Finds optimization opportunities on the MIR. struct ConstPropagator<'mir, 'tcx> { - ecx: InterpCx<'mir, 'tcx, ConstPropMachine>, + ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>, tcx: TyCtxt<'tcx>, can_const_prop: IndexVec, param_env: ParamEnv<'tcx>, @@ -349,7 +372,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let param_env = tcx.param_env(def_id).with_reveal_all(); let span = tcx.def_span(def_id); - let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine, ()); + let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine::new(), ()); let can_const_prop = CanConstProp::check(body); let ret = ecx