diff --git a/src/librustc_mir/const_eval/error.rs b/src/librustc_mir/const_eval/error.rs
index f7e28cf8d8c2f..3c3618f390c61 100644
--- a/src/librustc_mir/const_eval/error.rs
+++ b/src/librustc_mir/const_eval/error.rs
@@ -50,7 +50,7 @@ impl Error for ConstEvalErrKind {}
/// Turn an interpreter error into something to report to the user.
/// As a side-effect, if RUSTC_CTFE_BACKTRACE is set, this prints the backtrace.
/// Should be called only if the error is actually going to to be reported!
-pub fn error_to_const_error<'mir, 'tcx, M: Machine<'mir, 'tcx>>(
+pub fn error_to_const_error<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>(
ecx: &InterpCx<'mir, 'tcx, M>,
mut error: InterpErrorInfo<'tcx>,
) -> ConstEvalErr<'tcx> {
diff --git a/src/librustc_mir/const_eval/machine.rs b/src/librustc_mir/const_eval/machine.rs
index e53ca6b31bb67..30990853b516a 100644
--- a/src/librustc_mir/const_eval/machine.rs
+++ b/src/librustc_mir/const_eval/machine.rs
@@ -19,7 +19,7 @@ use crate::interpret::{
use super::error::*;
-impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> {
+impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
/// Evaluate a const function where all arguments (if any) are zero-sized types.
/// The evaluation is memoized thanks to the query system.
///
@@ -86,12 +86,15 @@ impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter> {
}
/// Extra machine state for CTFE, and the Machine instance
-pub struct CompileTimeInterpreter {
+pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// For now, the number of terminators that can be evaluated before we throw a resource
/// exhuastion error.
///
/// Setting this to `0` disables the limit and allows the interpreter to run forever.
pub steps_remaining: usize,
+
+ /// The virtual call stack.
+ pub(crate) stack: Vec>,
}
#[derive(Copy, Clone, Debug)]
@@ -100,9 +103,9 @@ pub struct MemoryExtra {
pub(super) can_access_statics: bool,
}
-impl CompileTimeInterpreter {
+impl<'mir, 'tcx> CompileTimeInterpreter<'mir, 'tcx> {
pub(super) fn new(const_eval_limit: usize) -> Self {
- CompileTimeInterpreter { steps_remaining: const_eval_limit }
+ CompileTimeInterpreter { steps_remaining: const_eval_limit, stack: Vec::new() }
}
}
@@ -156,7 +159,8 @@ impl interpret::AllocMap for FxHashMap {
}
}
-crate type CompileTimeEvalContext<'mir, 'tcx> = InterpCx<'mir, 'tcx, CompileTimeInterpreter>;
+crate type CompileTimeEvalContext<'mir, 'tcx> =
+ InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>>;
impl interpret::MayLeak for ! {
#[inline(always)]
@@ -166,7 +170,7 @@ impl interpret::MayLeak for ! {
}
}
-impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter {
+impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, 'tcx> {
type MemoryKind = !;
type PointerTag = ();
type ExtraFnVal = !;
@@ -349,6 +353,20 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter {
Ok(frame)
}
+ #[inline(always)]
+ fn stack(
+ ecx: &'a InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] {
+ &ecx.machine.stack
+ }
+
+ #[inline(always)]
+ fn stack_mut(
+ ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a mut Vec> {
+ &mut ecx.machine.stack
+ }
+
fn before_access_global(
memory_extra: &MemoryExtra,
alloc_id: AllocId,
diff --git a/src/librustc_mir/interpret/cast.rs b/src/librustc_mir/interpret/cast.rs
index 2fcdc9ca5d6ce..0e01652bc9002 100644
--- a/src/librustc_mir/interpret/cast.rs
+++ b/src/librustc_mir/interpret/cast.rs
@@ -12,7 +12,7 @@ use rustc_middle::ty::{self, Ty, TypeAndMut, TypeFoldable};
use rustc_span::symbol::sym;
use rustc_target::abi::{LayoutOf, Size, Variants};
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn cast(
&mut self,
src: OpTy<'tcx, M::PointerTag>,
diff --git a/src/librustc_mir/interpret/eval_context.rs b/src/librustc_mir/interpret/eval_context.rs
index 06dee62b3e70b..b2a041874d09d 100644
--- a/src/librustc_mir/interpret/eval_context.rs
+++ b/src/librustc_mir/interpret/eval_context.rs
@@ -28,6 +28,8 @@ use crate::util::storage::AlwaysLiveLocals;
pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
/// Stores the `Machine` instance.
+ ///
+ /// Note: the stack is provided by the machine.
pub machine: M,
/// The results of the type checker, from rustc.
@@ -39,9 +41,6 @@ pub struct InterpCx<'mir, 'tcx, M: Machine<'mir, 'tcx>> {
/// The virtual memory system.
pub memory: Memory<'mir, 'tcx, M>,
- /// The virtual call stack.
- pub(crate) stack: Vec>,
-
/// A cache for deduplicating vtables
pub(super) vtables:
FxHashMap<(Ty<'tcx>, Option>), Pointer>,
@@ -297,7 +296,7 @@ pub(super) fn from_known_layout<'tcx>(
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn new(
tcx: TyCtxtAt<'tcx>,
param_env: ty::ParamEnv<'tcx>,
@@ -309,7 +308,6 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
tcx,
param_env,
memory: Memory::new(tcx, memory_extra),
- stack: Vec::new(),
vtables: FxHashMap::default(),
}
}
@@ -349,24 +347,32 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
#[inline(always)]
- pub fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] {
- &self.stack
+ pub(crate) fn stack(&self) -> &[Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra>] {
+ M::stack(self)
+ }
+
+ #[inline(always)]
+ pub(crate) fn stack_mut(
+ &mut self,
+ ) -> &mut Vec> {
+ M::stack_mut(self)
}
#[inline(always)]
- pub fn cur_frame(&self) -> usize {
- assert!(!self.stack.is_empty());
- self.stack.len() - 1
+ pub fn frame_idx(&self) -> usize {
+ let stack = self.stack();
+ assert!(!stack.is_empty());
+ stack.len() - 1
}
#[inline(always)]
pub fn frame(&self) -> &Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> {
- self.stack.last().expect("no call frames exist")
+ self.stack().last().expect("no call frames exist")
}
#[inline(always)]
pub fn frame_mut(&mut self) -> &mut Frame<'mir, 'tcx, M::PointerTag, M::FrameExtra> {
- self.stack.last_mut().expect("no call frames exist")
+ self.stack_mut().last_mut().expect("no call frames exist")
}
#[inline(always)]
@@ -596,8 +602,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
return_place: Option>,
return_to_block: StackPopCleanup,
) -> InterpResult<'tcx> {
- if !self.stack.is_empty() {
- info!("PAUSING({}) {}", self.cur_frame(), self.frame().instance);
+ if !self.stack().is_empty() {
+ info!("PAUSING({}) {}", self.frame_idx(), self.frame().instance);
}
::log_settings::settings().indentation += 1;
@@ -615,7 +621,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
extra: (),
};
let frame = M::init_frame_extra(self, pre_frame)?;
- self.stack.push(frame);
+ self.stack_mut().push(frame);
// don't allocate at all for trivial constants
if body.local_decls.len() > 1 {
@@ -648,9 +654,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
M::after_stack_push(self)?;
- info!("ENTERING({}) {}", self.cur_frame(), self.frame().instance);
+ info!("ENTERING({}) {}", self.frame_idx(), self.frame().instance);
- if self.stack.len() > *self.tcx.sess.recursion_limit.get() {
+ if self.stack().len() > *self.tcx.sess.recursion_limit.get() {
throw_exhaust!(StackFrameLimitReached)
} else {
Ok(())
@@ -705,7 +711,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub(super) fn pop_stack_frame(&mut self, unwinding: bool) -> InterpResult<'tcx> {
info!(
"LEAVING({}) {} (unwinding = {})",
- self.cur_frame(),
+ self.frame_idx(),
self.frame().instance,
unwinding
);
@@ -720,7 +726,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
);
::log_settings::settings().indentation -= 1;
- let frame = self.stack.pop().expect("tried to pop a stack frame, but there were none");
+ let frame =
+ self.stack_mut().pop().expect("tried to pop a stack frame, but there were none");
// Now where do we jump next?
@@ -735,7 +742,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
};
if !cleanup {
- assert!(self.stack.is_empty(), "only the topmost frame should ever be leaked");
+ assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
assert!(next_block.is_none(), "tried to skip cleanup when we have a next block!");
assert!(!unwinding, "tried to skip cleanup during unwinding");
// Leak the locals, skip validation, skip machine hook.
@@ -784,10 +791,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}
- if !self.stack.is_empty() {
+ if !self.stack().is_empty() {
info!(
"CONTINUING({}) {} (unwinding = {})",
- self.cur_frame(),
+ self.frame_idx(),
self.frame().instance,
unwinding
);
@@ -895,12 +902,12 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
Place::Local { frame, local } => {
let mut allocs = Vec::new();
let mut msg = format!("{:?}", local);
- if frame != self.cur_frame() {
- write!(msg, " ({} frames up)", self.cur_frame() - frame).unwrap();
+ if frame != self.frame_idx() {
+ write!(msg, " ({} frames up)", self.frame_idx() - frame).unwrap();
}
write!(msg, ":").unwrap();
- match self.stack[frame].locals[local].value {
+ match self.stack()[frame].locals[local].value {
LocalValue::Dead => write!(msg, " is dead").unwrap(),
LocalValue::Uninitialized => write!(msg, " is uninitialized").unwrap(),
LocalValue::Live(Operand::Indirect(mplace)) => match mplace.ptr {
diff --git a/src/librustc_mir/interpret/intern.rs b/src/librustc_mir/interpret/intern.rs
index b28cd3f0a8dd6..8bc0e2ee6a4c6 100644
--- a/src/librustc_mir/interpret/intern.rs
+++ b/src/librustc_mir/interpret/intern.rs
@@ -149,7 +149,7 @@ impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> InternVisitor<'rt, 'mir
}
}
-impl<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
+impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
for InternVisitor<'rt, 'mir, 'tcx, M>
{
type V = MPlaceTy<'tcx>;
@@ -286,7 +286,10 @@ pub fn intern_const_alloc_recursive>(
intern_kind: InternKind,
ret: MPlaceTy<'tcx>,
ignore_interior_mut_in_const_validation: bool,
-) -> InterpResult<'tcx> {
+) -> InterpResult<'tcx>
+where
+ 'tcx: 'mir,
+{
let tcx = ecx.tcx;
let (base_mutability, base_intern_mode) = match intern_kind {
// `static mut` doesn't care about interior mutability, it's mutable anyway
diff --git a/src/librustc_mir/interpret/intrinsics.rs b/src/librustc_mir/interpret/intrinsics.rs
index d976df82577ea..c8bf328ce8eee 100644
--- a/src/librustc_mir/interpret/intrinsics.rs
+++ b/src/librustc_mir/interpret/intrinsics.rs
@@ -73,7 +73,7 @@ crate fn eval_nullary_intrinsic<'tcx>(
})
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Returns `true` if emulation happened.
pub fn emulate_intrinsic(
&mut self,
diff --git a/src/librustc_mir/interpret/intrinsics/caller_location.rs b/src/librustc_mir/interpret/intrinsics/caller_location.rs
index 754f45d9ec05b..91b046d7bb264 100644
--- a/src/librustc_mir/interpret/intrinsics/caller_location.rs
+++ b/src/librustc_mir/interpret/intrinsics/caller_location.rs
@@ -10,11 +10,11 @@ use crate::interpret::{
MPlaceTy, MemoryKind, Scalar,
};
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a
/// frame which is not `#[track_caller]`.
crate fn find_closest_untracked_caller_location(&self) -> Span {
- self.stack
+ self.stack()
.iter()
.rev()
// Find first non-`#[track_caller]` frame.
diff --git a/src/librustc_mir/interpret/machine.rs b/src/librustc_mir/interpret/machine.rs
index 8bf8d904cb29e..dfdd95c95a364 100644
--- a/src/librustc_mir/interpret/machine.rs
+++ b/src/librustc_mir/interpret/machine.rs
@@ -230,6 +230,34 @@ pub trait Machine<'mir, 'tcx>: Sized {
id
}
+ /// Called when converting a `ty::Const` to an operand (in
+ /// `eval_const_to_op`).
+ ///
+ /// Miri uses this callback for creating per thread allocations for thread
+ /// locals. In Rust, one way of creating a thread local is by marking a
+ /// static with `#[thread_local]`. On supported platforms this gets
+ /// translated to a LLVM thread local for which LLVM automatically ensures
+ /// that each thread gets its own copy. Since LLVM automatically handles
+ /// thread locals, the Rust compiler just treats thread local statics as
+ /// regular statics even though accessing a thread local static should be an
+ /// effectful computation that depends on the current thread. The long term
+ /// plan is to change MIR to make accesses to thread locals explicit
+ /// (https://github.com/rust-lang/rust/issues/70685). While the issue 70685
+ /// is not fixed, our current workaround in Miri is to use this function to
+ /// make per-thread copies of thread locals. Please note that we cannot make
+ /// these copies in `canonical_alloc_id` because that is too late: for
+ /// example, if one created a pointer in thread `t1` to a thread local and
+ /// sent it to another thread `t2`, resolving the access in
+ /// `canonical_alloc_id` would result in pointer pointing to `t2`'s thread
+ /// local and not `t1` as it should.
+ #[inline]
+ fn adjust_global_const(
+ _ecx: &InterpCx<'mir, 'tcx, Self>,
+ val: mir::interpret::ConstValue<'tcx>,
+ ) -> InterpResult<'tcx, mir::interpret::ConstValue<'tcx>> {
+ Ok(val)
+ }
+
/// Called to initialize the "extra" state of an allocation and make the pointers
/// it contains (in relocations) tagged. The way we construct allocations is
/// to always first construct it without extra and then add the extra.
@@ -285,6 +313,16 @@ pub trait Machine<'mir, 'tcx>: Sized {
frame: Frame<'mir, 'tcx, Self::PointerTag>,
) -> InterpResult<'tcx, Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>>;
+ /// Borrow the current thread's stack.
+ fn stack(
+ ecx: &'a InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>];
+
+ /// Mutably borrow the current thread's stack.
+ fn stack_mut(
+ ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a mut Vec>;
+
/// Called immediately after a stack frame got pushed and its locals got initialized.
fn after_stack_push(_ecx: &mut InterpCx<'mir, 'tcx, Self>) -> InterpResult<'tcx> {
Ok(())
diff --git a/src/librustc_mir/interpret/operand.rs b/src/librustc_mir/interpret/operand.rs
index 2317a1fea0792..31e6fbdceee4a 100644
--- a/src/librustc_mir/interpret/operand.rs
+++ b/src/librustc_mir/interpret/operand.rs
@@ -209,7 +209,7 @@ impl<'tcx, Tag: Copy> ImmTy<'tcx, Tag> {
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Normalice `place.ptr` to a `Pointer` if this is a place and not a ZST.
/// Can be helpful to avoid lots of `force_ptr` calls later, if this place is used a lot.
#[inline]
@@ -440,7 +440,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
) -> InterpResult<'tcx, OpTy<'tcx, M::PointerTag>> {
let op = match *place {
Place::Ptr(mplace) => Operand::Indirect(mplace),
- Place::Local { frame, local } => *self.access_local(&self.stack[frame], local, None)?,
+ Place::Local { frame, local } => {
+ *self.access_local(&self.stack()[frame], local, None)?
+ }
};
Ok(OpTy { op, layout: place.layout })
}
@@ -528,6 +530,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// potentially requiring the current static to be evaluated again. This is not a
// problem here, because we are building an operand which means an actual read is
// happening.
+ //
+ // The machine callback `adjust_global_const` below is guaranteed to
+ // be called for all constants because `const_eval` calls
+ // `eval_const_to_op` recursively.
return Ok(self.const_eval(GlobalId { instance, promoted }, val.ty)?);
}
ty::ConstKind::Infer(..)
@@ -537,6 +543,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
ty::ConstKind::Value(val_val) => val_val,
};
+ // This call allows the machine to create fresh allocation ids for
+ // thread-local statics (see the `adjust_global_const` function
+ // documentation).
+ let val_val = M::adjust_global_const(self, val_val)?;
// Other cases need layout.
let layout = from_known_layout(self.tcx, layout, || self.layout_of(val.ty))?;
let op = match val_val {
diff --git a/src/librustc_mir/interpret/operator.rs b/src/librustc_mir/interpret/operator.rs
index 0aa7e98f3edfa..d651267f82b79 100644
--- a/src/librustc_mir/interpret/operator.rs
+++ b/src/librustc_mir/interpret/operator.rs
@@ -9,7 +9,7 @@ use rustc_target::abi::LayoutOf;
use super::{ImmTy, Immediate, InterpCx, Machine, PlaceTy};
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Applies the binary operation `op` to the two operands and writes a tuple of the result
/// and a boolean signifying the potential overflow to the destination.
pub fn binop_with_overflow(
@@ -45,7 +45,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn binary_char_op(
&self,
bin_op: mir::BinOp,
diff --git a/src/librustc_mir/interpret/place.rs b/src/librustc_mir/interpret/place.rs
index 71275452fb688..af3a9da2f6ca6 100644
--- a/src/librustc_mir/interpret/place.rs
+++ b/src/librustc_mir/interpret/place.rs
@@ -283,7 +283,7 @@ impl<'tcx, Tag: ::std::fmt::Debug> PlaceTy<'tcx, Tag> {
}
// separating the pointer tag for `impl Trait`, see https://github.com/rust-lang/rust/issues/54385
-impl<'mir, 'tcx, Tag, M> InterpCx<'mir, 'tcx, M>
+impl<'mir, 'tcx: 'mir, Tag, M> InterpCx<'mir, 'tcx, M>
where
// FIXME: Working around https://github.com/rust-lang/rust/issues/54385
Tag: ::std::fmt::Debug + Copy + Eq + Hash + 'static,
@@ -662,7 +662,7 @@ where
}
local => PlaceTy {
// This works even for dead/uninitialized locals; we check further when writing
- place: Place::Local { frame: self.cur_frame(), local },
+ place: Place::Local { frame: self.frame_idx(), local },
layout: self.layout_of_local(self.frame(), local, None)?,
},
};
@@ -755,7 +755,7 @@ where
// but not factored as a separate function.
let mplace = match dest.place {
Place::Local { frame, local } => {
- match self.stack[frame].locals[local].access_mut()? {
+ match self.stack_mut()[frame].locals[local].access_mut()? {
Ok(local) => {
// Local can be updated in-place.
*local = LocalValue::Live(Operand::Immediate(src));
@@ -985,14 +985,15 @@ where
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::PointerTag>, Option)> {
let (mplace, size) = match place.place {
Place::Local { frame, local } => {
- match self.stack[frame].locals[local].access_mut()? {
+ match self.stack_mut()[frame].locals[local].access_mut()? {
Ok(&mut local_val) => {
// We need to make an allocation.
// We need the layout of the local. We can NOT use the layout we got,
// that might e.g., be an inner field of a struct with `Scalar` layout,
// that has different alignment than the outer field.
- let local_layout = self.layout_of_local(&self.stack[frame], local, None)?;
+ let local_layout =
+ self.layout_of_local(&self.stack()[frame], local, None)?;
// We also need to support unsized types, and hence cannot use `allocate`.
let (size, align) = self
.size_and_align_of(meta, local_layout)?
@@ -1008,7 +1009,7 @@ where
}
// Now we can call `access_mut` again, asserting it goes well,
// and actually overwrite things.
- *self.stack[frame].locals[local].access_mut().unwrap().unwrap() =
+ *self.stack_mut()[frame].locals[local].access_mut().unwrap().unwrap() =
LocalValue::Live(Operand::Indirect(mplace));
(mplace, Some(size))
}
diff --git a/src/librustc_mir/interpret/step.rs b/src/librustc_mir/interpret/step.rs
index 37740878f7043..aae708827b953 100644
--- a/src/librustc_mir/interpret/step.rs
+++ b/src/librustc_mir/interpret/step.rs
@@ -29,7 +29,7 @@ fn binop_right_homogeneous(op: mir::BinOp) -> bool {
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub fn run(&mut self) -> InterpResult<'tcx> {
while self.step()? {}
Ok(())
@@ -42,7 +42,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// This is marked `#inline(always)` to work around adverserial codegen when `opt-level = 3`
#[inline(always)]
pub fn step(&mut self) -> InterpResult<'tcx, bool> {
- if self.stack.is_empty() {
+ if self.stack().is_empty() {
return Ok(false);
}
@@ -60,10 +60,10 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let body = self.body();
let basic_block = &body.basic_blocks()[block];
- let old_frames = self.cur_frame();
+ let old_frames = self.frame_idx();
if let Some(stmt) = basic_block.statements.get(stmt_id) {
- assert_eq!(old_frames, self.cur_frame());
+ assert_eq!(old_frames, self.frame_idx());
self.statement(stmt)?;
return Ok(true);
}
@@ -71,7 +71,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
M::before_terminator(self)?;
let terminator = basic_block.terminator();
- assert_eq!(old_frames, self.cur_frame());
+ assert_eq!(old_frames, self.frame_idx());
self.terminator(terminator)?;
Ok(true)
}
@@ -84,7 +84,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Some statements (e.g., box) push new stack frames.
// We have to record the stack frame number *before* executing the statement.
- let frame_idx = self.cur_frame();
+ let frame_idx = self.frame_idx();
match &stmt.kind {
Assign(box (place, rvalue)) => self.eval_rvalue_into_place(rvalue, *place)?,
@@ -126,7 +126,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
LlvmInlineAsm { .. } => throw_unsup_format!("inline assembly is not supported"),
}
- self.stack[frame_idx].stmt += 1;
+ self.stack_mut()[frame_idx].stmt += 1;
Ok(())
}
@@ -278,7 +278,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.set_span(terminator.source_info.span);
self.eval_terminator(terminator)?;
- if !self.stack.is_empty() {
+ if !self.stack().is_empty() {
if let Some(block) = self.frame().block {
info!("// executing {:?}", block);
}
diff --git a/src/librustc_mir/interpret/terminator.rs b/src/librustc_mir/interpret/terminator.rs
index 49fee1bddcb6d..7157225e5c9bb 100644
--- a/src/librustc_mir/interpret/terminator.rs
+++ b/src/librustc_mir/interpret/terminator.rs
@@ -11,7 +11,7 @@ use super::{
FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, StackPopCleanup,
};
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
pub(super) fn eval_terminator(
&mut self,
terminator: &mir::Terminator<'tcx>,
@@ -52,7 +52,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
Call { ref func, ref args, destination, ref cleanup, .. } => {
- let old_stack = self.cur_frame();
+ let old_stack = self.frame_idx();
let old_bb = self.frame().block;
let func = self.eval_operand(func, None)?;
let (fn_val, abi) = match func.layout.ty.kind {
@@ -80,7 +80,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self.eval_fn_call(fn_val, abi, &args[..], ret, *cleanup)?;
// Sanity-check that `eval_fn_call` either pushed a new frame or
// did a jump to another block.
- if self.cur_frame() == old_stack && self.frame().block == old_bb {
+ if self.frame_idx() == old_stack && self.frame().block == old_bb {
span_bug!(terminator.source_info.span, "evaluating this call made no progress");
}
}
@@ -372,7 +372,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
};
match res {
Err(err) => {
- self.stack.pop();
+ self.stack_mut().pop();
Err(err)
}
Ok(()) => Ok(()),
diff --git a/src/librustc_mir/interpret/traits.rs b/src/librustc_mir/interpret/traits.rs
index fb9401c7d8f28..7edd787c986bd 100644
--- a/src/librustc_mir/interpret/traits.rs
+++ b/src/librustc_mir/interpret/traits.rs
@@ -6,7 +6,7 @@ use rustc_target::abi::{Align, HasDataLayout, LayoutOf, Size};
use super::{FnVal, InterpCx, Machine, MemoryKind};
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
/// Creates a dynamic vtable for the given type and vtable origin. This is used only for
/// objects.
///
diff --git a/src/librustc_mir/interpret/validity.rs b/src/librustc_mir/interpret/validity.rs
index 2dffd78d776bd..e97a39f8c6fec 100644
--- a/src/librustc_mir/interpret/validity.rs
+++ b/src/librustc_mir/interpret/validity.rs
@@ -178,7 +178,7 @@ struct ValidityVisitor<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> {
ecx: &'rt InterpCx<'mir, 'tcx, M>,
}
-impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> {
+impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M> {
fn aggregate_field_path_elem(&mut self, layout: TyAndLayout<'tcx>, field: usize) -> PathElem {
// First, check if we are projecting to a variant.
match layout.variants {
@@ -610,7 +610,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, 'tcx, M
}
}
-impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
+impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
for ValidityVisitor<'rt, 'mir, 'tcx, M>
{
type V = OpTy<'tcx, M::PointerTag>;
@@ -813,7 +813,7 @@ impl<'rt, 'mir, 'tcx, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
fn validate_operand_internal(
&self,
op: OpTy<'tcx, M::PointerTag>,
diff --git a/src/librustc_mir/interpret/visitor.rs b/src/librustc_mir/interpret/visitor.rs
index 00d39452c49d5..903aa377a3d7d 100644
--- a/src/librustc_mir/interpret/visitor.rs
+++ b/src/librustc_mir/interpret/visitor.rs
@@ -37,7 +37,7 @@ pub trait Value<'mir, 'tcx, M: Machine<'mir, 'tcx>>: Copy {
// Operands and memory-places are both values.
// Places in general are not due to `place_field` having to do `force_allocation`.
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::PointerTag> {
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
@@ -75,7 +75,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for OpTy<'tcx, M::
}
}
-impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M> for MPlaceTy<'tcx, M::PointerTag> {
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> Value<'mir, 'tcx, M>
+ for MPlaceTy<'tcx, M::PointerTag>
+{
#[inline(always)]
fn layout(&self) -> TyAndLayout<'tcx> {
self.layout
diff --git a/src/librustc_mir/transform/const_prop.rs b/src/librustc_mir/transform/const_prop.rs
index cf1c70241bc6e..79dba2c5db8fb 100644
--- a/src/librustc_mir/transform/const_prop.rs
+++ b/src/librustc_mir/transform/const_prop.rs
@@ -158,9 +158,18 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
}
}
-struct ConstPropMachine;
+struct ConstPropMachine<'mir, 'tcx> {
+ /// The virtual call stack.
+ stack: Vec>,
+}
-impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine {
+impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> {
+ fn new() -> Self {
+ Self { stack: Vec::new() }
+ }
+}
+
+impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> {
type MemoryKind = !;
type PointerTag = ();
type ExtraFnVal = !;
@@ -296,11 +305,25 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine {
) -> InterpResult<'tcx, Frame<'mir, 'tcx>> {
Ok(frame)
}
+
+ #[inline(always)]
+ fn stack(
+ ecx: &'a InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] {
+ &ecx.machine.stack
+ }
+
+ #[inline(always)]
+ fn stack_mut(
+ ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
+ ) -> &'a mut Vec> {
+ &mut ecx.machine.stack
+ }
}
/// Finds optimization opportunities on the MIR.
struct ConstPropagator<'mir, 'tcx> {
- ecx: InterpCx<'mir, 'tcx, ConstPropMachine>,
+ ecx: InterpCx<'mir, 'tcx, ConstPropMachine<'mir, 'tcx>>,
tcx: TyCtxt<'tcx>,
can_const_prop: IndexVec,
param_env: ParamEnv<'tcx>,
@@ -349,7 +372,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
let param_env = tcx.param_env(def_id).with_reveal_all();
let span = tcx.def_span(def_id);
- let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine, ());
+ let mut ecx = InterpCx::new(tcx.at(span), param_env, ConstPropMachine::new(), ());
let can_const_prop = CanConstProp::check(body);
let ret = ecx