From e2bc16c101e76532420e3af7e7805071f445b2f7 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 11 Sep 2024 09:12:02 +0200 Subject: [PATCH] interpret: simplify SIMD type handling --- .../src/interpret/intrinsics.rs | 6 +- .../rustc_const_eval/src/interpret/operand.rs | 24 ------ .../rustc_const_eval/src/interpret/place.rs | 45 +++++----- .../src/interpret/projection.rs | 13 +++ .../rustc_const_eval/src/interpret/util.rs | 4 +- src/tools/miri/src/intrinsics/simd.rs | 82 +++++++++--------- src/tools/miri/src/shims/foreign_items.rs | 4 +- src/tools/miri/src/shims/x86/avx.rs | 12 +-- src/tools/miri/src/shims/x86/avx2.rs | 38 ++++----- src/tools/miri/src/shims/x86/mod.rs | 83 ++++++++++--------- src/tools/miri/src/shims/x86/sha.rs | 22 ++--- src/tools/miri/src/shims/x86/sse.rs | 10 +-- src/tools/miri/src/shims/x86/sse2.rs | 24 +++--- src/tools/miri/src/shims/x86/sse41.rs | 10 +-- src/tools/miri/src/shims/x86/ssse3.rs | 12 +-- 15 files changed, 190 insertions(+), 199 deletions(-) diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index bedc56de0da25..8a07f90c95104 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -384,8 +384,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { sym::simd_insert => { let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); let elem = &args[2]; - let (input, input_len) = self.operand_to_simd(&args[0])?; - let (dest, dest_len) = self.mplace_to_simd(dest)?; + let (input, input_len) = self.project_to_simd(&args[0])?; + let (dest, dest_len) = self.project_to_simd(dest)?; assert_eq!(input_len, dest_len, "Return vector length must match input length"); // Bounds are not checked by typeck so we have to do it ourselves. if index >= input_len { @@ -406,7 +406,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } sym::simd_extract => { let index = u64::from(self.read_scalar(&args[1])?.to_u32()?); - let (input, input_len) = self.operand_to_simd(&args[0])?; + let (input, input_len) = self.project_to_simd(&args[0])?; // Bounds are not checked by typeck so we have to do it ourselves. if index >= input_len { throw_ub_format!( diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index bb7e58b83ac7a..2e02d1001c859 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -681,30 +681,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { Ok(str) } - /// Converts a repr(simd) operand into an operand where `place_index` accesses the SIMD elements. - /// Also returns the number of elements. - /// - /// Can (but does not always) trigger UB if `op` is uninitialized. - pub fn operand_to_simd( - &self, - op: &OpTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> { - // Basically we just transmute this place into an array following simd_size_and_type. - // This only works in memory, but repr(simd) types should never be immediates anyway. - assert!(op.layout.ty.is_simd()); - match op.as_mplace_or_imm() { - Left(mplace) => self.mplace_to_simd(&mplace), - Right(imm) => match *imm { - Immediate::Uninit => { - throw_ub!(InvalidUninitBytes(None)) - } - Immediate::Scalar(..) | Immediate::ScalarPair(..) => { - bug!("arrays/slices can never have Scalar/ScalarPair layout") - } - }, - } - } - /// Read from a local of the current frame. /// Will not access memory, instead an indirect `Operand` is returned. /// diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs index 1eca92812dfb4..9dd9ca803852c 100644 --- a/compiler/rustc_const_eval/src/interpret/place.rs +++ b/compiler/rustc_const_eval/src/interpret/place.rs @@ -377,13 +377,15 @@ where Prov: Provenance, M: Machine<'tcx, Provenance = Prov>, { - pub fn ptr_with_meta_to_mplace( + fn ptr_with_meta_to_mplace( &self, ptr: Pointer>, meta: MemPlaceMeta, layout: TyAndLayout<'tcx>, + unaligned: bool, ) -> MPlaceTy<'tcx, M::Provenance> { - let misaligned = self.is_ptr_misaligned(ptr, layout.align.abi); + let misaligned = + if unaligned { None } else { self.is_ptr_misaligned(ptr, layout.align.abi) }; MPlaceTy { mplace: MemPlace { ptr, meta, misaligned }, layout } } @@ -393,7 +395,16 @@ where layout: TyAndLayout<'tcx>, ) -> MPlaceTy<'tcx, M::Provenance> { assert!(layout.is_sized()); - self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout) + self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ false) + } + + pub fn ptr_to_mplace_unaligned( + &self, + ptr: Pointer>, + layout: TyAndLayout<'tcx>, + ) -> MPlaceTy<'tcx, M::Provenance> { + assert!(layout.is_sized()); + self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ true) } /// Take a value, which represents a (thin or wide) reference, and make it a place. @@ -414,7 +425,7 @@ where // `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced; // we hence can't call `size_and_align_of` since that asserts more validity than we want. let ptr = ptr.to_pointer(self)?; - Ok(self.ptr_with_meta_to_mplace(ptr, meta, layout)) + Ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, /*unaligned*/ false)) } /// Turn a mplace into a (thin or wide) mutable raw pointer, pointing to the same space. @@ -484,23 +495,6 @@ where Ok(a) } - /// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements. - /// Also returns the number of elements. - pub fn mplace_to_simd( - &self, - mplace: &MPlaceTy<'tcx, M::Provenance>, - ) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> { - // Basically we want to transmute this place into an array following simd_size_and_type. - let (len, e_ty) = mplace.layout.ty.simd_size_and_type(*self.tcx); - // Some SIMD types have padding, so `len` many `e_ty` does not cover the entire place. - // Therefore we cannot transmute, and instead we project at offset 0, which side-steps - // the size check. - let array_layout = self.layout_of(Ty::new_array(self.tcx.tcx, e_ty, len))?; - assert!(array_layout.size <= mplace.layout.size); - let mplace = mplace.offset(Size::ZERO, array_layout, self)?; - Ok((mplace, len)) - } - /// Turn a local in the current frame into a place. pub fn local_to_place( &self, @@ -986,7 +980,7 @@ where span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known") }; let ptr = self.allocate_ptr(size, align, kind)?; - Ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout)) + Ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false)) } pub fn allocate( @@ -1021,7 +1015,12 @@ where }; let meta = Scalar::from_target_usize(u64::try_from(str.len()).unwrap(), self); let layout = self.layout_of(self.tcx.types.str_).unwrap(); - Ok(self.ptr_with_meta_to_mplace(ptr.into(), MemPlaceMeta::Meta(meta), layout)) + Ok(self.ptr_with_meta_to_mplace( + ptr.into(), + MemPlaceMeta::Meta(meta), + layout, + /*unaligned*/ false, + )) } pub fn raw_const_to_mplace( diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs index dd8dd21e0e8b9..641ed5bb7c0f1 100644 --- a/compiler/rustc_const_eval/src/interpret/projection.rs +++ b/compiler/rustc_const_eval/src/interpret/projection.rs @@ -244,6 +244,19 @@ where base.offset(offset, field_layout, self) } + /// Converts a repr(simd) value into an array of the right size, such that `project_index` + /// accesses the SIMD elements. Also returns the number of elements. + pub fn project_to_simd>( + &self, + base: &P, + ) -> InterpResult<'tcx, (P, u64)> { + assert!(base.layout().ty.ty_adt_def().unwrap().repr().simd()); + // SIMD types must be newtypes around arrays, so all we have to do is project to their only field. + let array = self.project_field(base, 0)?; + let len = array.len(self)?; + Ok((array, len)) + } + fn project_constant_index>( &self, base: &P, diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs index 6f5bcebbbb66f..490355f416cf4 100644 --- a/compiler/rustc_const_eval/src/interpret/util.rs +++ b/compiler/rustc_const_eval/src/interpret/util.rs @@ -9,7 +9,7 @@ use rustc_middle::ty::{ }; use tracing::debug; -use super::{throw_inval, InterpCx, MPlaceTy, MemPlaceMeta, MemoryKind}; +use super::{throw_inval, InterpCx, MPlaceTy, MemoryKind}; use crate::const_eval::{CompileTimeInterpCx, CompileTimeMachine, InterpretationResult}; /// Checks whether a type contains generic parameters which must be instantiated. @@ -103,5 +103,5 @@ pub(crate) fn create_static_alloc<'tcx>( assert_eq!(ecx.machine.static_root_ids, None); ecx.machine.static_root_ids = Some((alloc_id, static_def_id)); assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none()); - Ok(ecx.ptr_with_meta_to_mplace(Pointer::from(alloc_id).into(), MemPlaceMeta::None, layout)) + Ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout)) } diff --git a/src/tools/miri/src/intrinsics/simd.rs b/src/tools/miri/src/intrinsics/simd.rs index 85d7459bb4595..aa91b89d0ec6b 100644 --- a/src/tools/miri/src/intrinsics/simd.rs +++ b/src/tools/miri/src/intrinsics/simd.rs @@ -50,8 +50,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { | "bitreverse" => { let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); @@ -200,9 +200,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { use mir::BinOp; let [left, right] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -291,10 +291,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "fma" => { let [a, b, c] = check_arg_count(args)?; - let (a, a_len) = this.operand_to_simd(a)?; - let (b, b_len) = this.operand_to_simd(b)?; - let (c, c_len) = this.operand_to_simd(c)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (a, a_len) = this.project_to_simd(a)?; + let (b, b_len) = this.project_to_simd(b)?; + let (c, c_len) = this.project_to_simd(c)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, a_len); assert_eq!(dest_len, b_len); @@ -345,7 +345,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { use mir::BinOp; let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; + let (op, op_len) = this.project_to_simd(op)?; let imm_from_bool = |b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool); @@ -408,7 +408,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { use mir::BinOp; let [op, init] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; + let (op, op_len) = this.project_to_simd(op)?; let init = this.read_immediate(init)?; let mir_op = match intrinsic_name { @@ -426,10 +426,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "select" => { let [mask, yes, no] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (yes, yes_len) = this.operand_to_simd(yes)?; - let (no, no_len) = this.operand_to_simd(no)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (mask, mask_len) = this.project_to_simd(mask)?; + let (yes, yes_len) = this.project_to_simd(yes)?; + let (no, no_len) = this.project_to_simd(no)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, mask_len); assert_eq!(dest_len, yes_len); @@ -448,9 +448,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Variant of `select` that takes a bitmask rather than a "vector of bool". "select_bitmask" => { let [mask, yes, no] = check_arg_count(args)?; - let (yes, yes_len) = this.operand_to_simd(yes)?; - let (no, no_len) = this.operand_to_simd(no)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (yes, yes_len) = this.project_to_simd(yes)?; + let (no, no_len) = this.project_to_simd(no)?; + let (dest, dest_len) = this.project_to_simd(dest)?; let bitmask_len = dest_len.next_multiple_of(8); if bitmask_len > 64 { throw_unsup_format!( @@ -522,7 +522,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Converts a "vector of bool" into a bitmask. "bitmask" => { let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; + let (op, op_len) = this.project_to_simd(op)?; let bitmask_len = op_len.next_multiple_of(8); if bitmask_len > 64 { throw_unsup_format!( @@ -570,8 +570,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => { let [op] = check_arg_count(args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); @@ -627,9 +627,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "shuffle_generic" => { let [left, right] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; let index = generic_args[2] .expect_const() @@ -662,15 +662,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "shuffle" => { let [left, right, index] = check_arg_count(args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; // `index` is an array or a SIMD type let (index, index_len) = match index.layout.ty.kind() { // FIXME: remove this once `index` must always be a SIMD vector. - ty::Array(..) => (index.assert_mem_place(), index.len(this)?), - _ => this.operand_to_simd(index)?, + ty::Array(..) => (index.clone(), index.len(this)?), + _ => this.project_to_simd(index)?, }; assert_eq!(left_len, right_len); @@ -699,10 +699,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "gather" => { let [passthru, ptrs, mask] = check_arg_count(args)?; - let (passthru, passthru_len) = this.operand_to_simd(passthru)?; - let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (passthru, passthru_len) = this.project_to_simd(passthru)?; + let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?; + let (mask, mask_len) = this.project_to_simd(mask)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, passthru_len); assert_eq!(dest_len, ptrs_len); @@ -725,9 +725,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "scatter" => { let [value, ptrs, mask] = check_arg_count(args)?; - let (value, value_len) = this.operand_to_simd(value)?; - let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; + let (value, value_len) = this.project_to_simd(value)?; + let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?; + let (mask, mask_len) = this.project_to_simd(mask)?; assert_eq!(ptrs_len, value_len); assert_eq!(ptrs_len, mask_len); @@ -745,10 +745,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "masked_load" => { let [mask, ptr, default] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; + let (mask, mask_len) = this.project_to_simd(mask)?; let ptr = this.read_pointer(ptr)?; - let (default, default_len) = this.operand_to_simd(default)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (default, default_len) = this.project_to_simd(default)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, mask_len); assert_eq!(dest_len, default_len); @@ -772,9 +772,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } "masked_store" => { let [mask, ptr, vals] = check_arg_count(args)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; + let (mask, mask_len) = this.project_to_simd(mask)?; let ptr = this.read_pointer(ptr)?; - let (vals, vals_len) = this.operand_to_simd(vals)?; + let (vals, vals_len) = this.project_to_simd(vals)?; assert_eq!(mask_len, vals_len); diff --git a/src/tools/miri/src/shims/foreign_items.rs b/src/tools/miri/src/shims/foreign_items.rs index 1b45bc22038b1..88ab012f97814 100644 --- a/src/tools/miri/src/shims/foreign_items.rs +++ b/src/tools/miri/src/shims/foreign_items.rs @@ -903,8 +903,8 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> { name if name.starts_with("llvm.ctpop.v") => { let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); diff --git a/src/tools/miri/src/shims/x86/avx.rs b/src/tools/miri/src/shims/x86/avx.rs index 2f6569e1823de..1ddc9efcf6bae 100644 --- a/src/tools/miri/src/shims/x86/avx.rs +++ b/src/tools/miri/src/shims/x86/avx.rs @@ -159,9 +159,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [data, control] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (data, data_len) = this.operand_to_simd(data)?; - let (control, control_len) = this.operand_to_simd(control)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (data, data_len) = this.project_to_simd(data)?; + let (control, control_len) = this.project_to_simd(control)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, data_len); assert_eq!(dest_len, control_len); @@ -193,9 +193,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [data, control] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (data, data_len) = this.operand_to_simd(data)?; - let (control, control_len) = this.operand_to_simd(control)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (data, data_len) = this.project_to_simd(data)?; + let (control, control_len) = this.project_to_simd(control)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, data_len); assert_eq!(dest_len, control_len); diff --git a/src/tools/miri/src/shims/x86/avx2.rs b/src/tools/miri/src/shims/x86/avx2.rs index 7f6c9336a9749..74d4673bd825b 100644 --- a/src/tools/miri/src/shims/x86/avx2.rs +++ b/src/tools/miri/src/shims/x86/avx2.rs @@ -62,10 +62,10 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { assert_eq!(dest.layout, src.layout); - let (src, _) = this.operand_to_simd(src)?; - let (offsets, offsets_len) = this.operand_to_simd(offsets)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (src, _) = this.project_to_simd(src)?; + let (offsets, offsets_len) = this.project_to_simd(offsets)?; + let (mask, mask_len) = this.project_to_simd(mask)?; + let (dest, dest_len) = this.project_to_simd(dest)?; // There are cases like dest: i32x4, offsets: i64x2 // If dest has more elements than offset, extra dest elements are filled with zero. @@ -118,9 +118,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(left_len, right_len); assert_eq!(dest_len.strict_mul(2), left_len); @@ -155,9 +155,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(left_len, right_len); assert_eq!(dest_len.strict_mul(2), left_len); @@ -271,9 +271,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -330,9 +330,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(left_len, right_len); assert_eq!(left_len, dest_len.strict_mul(8)); @@ -363,9 +363,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); diff --git a/src/tools/miri/src/shims/x86/mod.rs b/src/tools/miri/src/shims/x86/mod.rs index 305e59fa0cd60..d7241f87d0e8a 100644 --- a/src/tools/miri/src/shims/x86/mod.rs +++ b/src/tools/miri/src/shims/x86/mod.rs @@ -314,9 +314,9 @@ fn bin_op_simd_float_first<'tcx, F: rustc_apfloat::Float>( right: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -344,9 +344,9 @@ fn bin_op_simd_float_all<'tcx, F: rustc_apfloat::Float>( right: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -430,8 +430,8 @@ fn unary_op_ss<'tcx>( op: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); @@ -453,8 +453,8 @@ fn unary_op_ps<'tcx>( op: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); @@ -491,8 +491,8 @@ fn shift_simd_by_scalar<'tcx>( which: ShiftOp, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); // `right` may have a different length, and we only care about its @@ -547,9 +547,9 @@ fn shift_simd_by_simd<'tcx>( which: ShiftOp, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -613,9 +613,9 @@ fn round_first<'tcx, F: rustc_apfloat::Float>( rounding: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -643,8 +643,8 @@ fn round_all<'tcx, F: rustc_apfloat::Float>( rounding: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, op_len); @@ -695,8 +695,8 @@ fn convert_float_to_int<'tcx>( rnd: rustc_apfloat::Round, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; // Output must be *signed* integers. assert!(matches!(dest.layout.field(this, 0).ty.kind(), ty::Int(_))); @@ -729,8 +729,8 @@ fn int_abs<'tcx>( op: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(op_len, dest_len); @@ -906,8 +906,8 @@ fn test_bits_masked<'tcx>( ) -> InterpResult<'tcx, (bool, bool)> { assert_eq!(op.layout, mask.layout); - let (op, op_len) = this.operand_to_simd(op)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; + let (op, op_len) = this.project_to_simd(op)?; + let (mask, mask_len) = this.project_to_simd(mask)?; assert_eq!(op_len, mask_len); @@ -937,8 +937,8 @@ fn test_high_bits_masked<'tcx>( ) -> InterpResult<'tcx, (bool, bool)> { assert_eq!(op.layout, mask.layout); - let (op, op_len) = this.operand_to_simd(op)?; - let (mask, mask_len) = this.operand_to_simd(mask)?; + let (op, op_len) = this.project_to_simd(op)?; + let (mask, mask_len) = this.project_to_simd(mask)?; assert_eq!(op_len, mask_len); @@ -967,8 +967,8 @@ fn mask_load<'tcx>( mask: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (mask, mask_len) = this.project_to_simd(mask)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, mask_len); @@ -1000,8 +1000,8 @@ fn mask_store<'tcx>( mask: &OpTy<'tcx>, value: &OpTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (mask, mask_len) = this.operand_to_simd(mask)?; - let (value, value_len) = this.operand_to_simd(value)?; + let (mask, mask_len) = this.project_to_simd(mask)?; + let (value, value_len) = this.project_to_simd(value)?; assert_eq!(value_len, mask_len); @@ -1014,9 +1014,12 @@ fn mask_store<'tcx>( let value = this.project_index(&value, i)?; if this.read_scalar(&mask)?.to_uint(mask_item_size)? >> high_bit_offset != 0 { + // *Non-inbounds* pointer arithmetic to compute the destination. + // (That's why we can't use a place projection.) let ptr = ptr.wrapping_offset(value.layout.size * i, &this.tcx); - // Unaligned copy, which is what we want. - this.mem_copy(value.ptr(), ptr, value.layout.size, /*nonoverlapping*/ true)?; + // Deref the pointer *unaligned*, and do the copy. + let dest = this.ptr_to_mplace_unaligned(ptr, value.layout); + this.copy_op(&value, &dest)?; } } @@ -1095,9 +1098,9 @@ fn pmulhrsw<'tcx>( right: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -1313,9 +1316,9 @@ fn psign<'tcx>( right: &OpTy<'tcx>, dest: &MPlaceTy<'tcx>, ) -> InterpResult<'tcx, ()> { - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); diff --git a/src/tools/miri/src/shims/x86/sha.rs b/src/tools/miri/src/shims/x86/sha.rs index e9cc28be34cff..d9cd34377e683 100644 --- a/src/tools/miri/src/shims/x86/sha.rs +++ b/src/tools/miri/src/shims/x86/sha.rs @@ -23,7 +23,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Prefix should have already been checked. let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.sha").unwrap(); - fn read<'c>(this: &mut MiriInterpCx<'c>, reg: &MPlaceTy<'c>) -> InterpResult<'c, [u32; 4]> { + fn read<'c>(this: &mut MiriInterpCx<'c>, reg: &OpTy<'c>) -> InterpResult<'c, [u32; 4]> { let mut res = [0; 4]; // We reverse the order because x86 is little endian but the copied implementation uses // big endian. @@ -53,10 +53,10 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { "256rnds2" => { let [a, b, k] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (a_reg, a_len) = this.operand_to_simd(a)?; - let (b_reg, b_len) = this.operand_to_simd(b)?; - let (k_reg, k_len) = this.operand_to_simd(k)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (a_reg, a_len) = this.project_to_simd(a)?; + let (b_reg, b_len) = this.project_to_simd(b)?; + let (k_reg, k_len) = this.project_to_simd(k)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(a_len, 4); assert_eq!(b_len, 4); @@ -74,9 +74,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { "256msg1" => { let [a, b] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (a_reg, a_len) = this.operand_to_simd(a)?; - let (b_reg, b_len) = this.operand_to_simd(b)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (a_reg, a_len) = this.project_to_simd(a)?; + let (b_reg, b_len) = this.project_to_simd(b)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(a_len, 4); assert_eq!(b_len, 4); @@ -92,9 +92,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { "256msg2" => { let [a, b] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (a_reg, a_len) = this.operand_to_simd(a)?; - let (b_reg, b_len) = this.operand_to_simd(b)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (a_reg, a_len) = this.project_to_simd(a)?; + let (b_reg, b_len) = this.project_to_simd(b)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(a_len, 4); assert_eq!(b_len, 4); diff --git a/src/tools/miri/src/shims/x86/sse.rs b/src/tools/miri/src/shims/x86/sse.rs index 07a4eaa85f3e3..8de7ddc79313b 100644 --- a/src/tools/miri/src/shims/x86/sse.rs +++ b/src/tools/miri/src/shims/x86/sse.rs @@ -130,8 +130,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; assert_eq!(left_len, right_len); @@ -157,7 +157,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Converts the first component of `op` from f32 to i32/i64. "cvtss2si" | "cvttss2si" | "cvtss2si64" | "cvttss2si64" => { let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (op, _) = this.operand_to_simd(op)?; + let (op, _) = this.project_to_simd(op)?; let op = this.read_immediate(&this.project_index(&op, 0)?)?; @@ -187,8 +187,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); diff --git a/src/tools/miri/src/shims/x86/sse2.rs b/src/tools/miri/src/shims/x86/sse2.rs index 163d74a6de4e9..bdb52a04a91d1 100644 --- a/src/tools/miri/src/shims/x86/sse2.rs +++ b/src/tools/miri/src/shims/x86/sse2.rs @@ -42,9 +42,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(left_len, right_len); assert_eq!(dest_len.strict_mul(2), left_len); @@ -81,9 +81,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; // left and right are u8x16, dest is u64x2 assert_eq!(left_len, right_len); @@ -270,8 +270,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; assert_eq!(left_len, right_len); @@ -297,7 +297,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { // Converts the first component of `op` from f64 to i32/i64. "cvtsd2si" | "cvttsd2si" | "cvtsd2si64" | "cvttsd2si64" => { let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (op, _) = this.operand_to_simd(op)?; + let (op, _) = this.project_to_simd(op)?; let op = this.read_immediate(&this.project_index(&op, 0)?)?; @@ -325,9 +325,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, _) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, _) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); diff --git a/src/tools/miri/src/shims/x86/sse41.rs b/src/tools/miri/src/shims/x86/sse41.rs index 9e048fb9eb841..8fcdc491c0ca3 100644 --- a/src/tools/miri/src/shims/x86/sse41.rs +++ b/src/tools/miri/src/shims/x86/sse41.rs @@ -29,9 +29,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right, imm] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -118,8 +118,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { "phminposuw" => { let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (op, op_len) = this.operand_to_simd(op)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (op, op_len) = this.project_to_simd(op)?; + let (dest, dest_len) = this.project_to_simd(dest)?; // Find minimum let mut min_value = u16::MAX; diff --git a/src/tools/miri/src/shims/x86/ssse3.rs b/src/tools/miri/src/shims/x86/ssse3.rs index ecacaeb9af524..096b0fb9e4b17 100644 --- a/src/tools/miri/src/shims/x86/ssse3.rs +++ b/src/tools/miri/src/shims/x86/ssse3.rs @@ -34,9 +34,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(dest_len, left_len); assert_eq!(dest_len, right_len); @@ -84,9 +84,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { let [left, right] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?; - let (left, left_len) = this.operand_to_simd(left)?; - let (right, right_len) = this.operand_to_simd(right)?; - let (dest, dest_len) = this.mplace_to_simd(dest)?; + let (left, left_len) = this.project_to_simd(left)?; + let (right, right_len) = this.project_to_simd(right)?; + let (dest, dest_len) = this.project_to_simd(dest)?; assert_eq!(left_len, right_len); assert_eq!(dest_len.strict_mul(2), left_len);