safe transmute: gracefully bubble-up layout errors

Changes `.unwrap()`s to `?` to avoid ICEs. Adds ui tests.

Fixes #129327

compiler/rustc_transmute/src/layout/tree.rs

@@ -179,7 +179,7 @@ pub(crate) mod rustc {
     };
 
     use super::Tree;
-    use crate::layout::rustc::{Def, Ref};
+    use crate::layout::rustc::{layout_of, Def, Ref};
 
     #[derive(Debug, Copy, Clone)]
     pub(crate) enum Err {
@@ -206,7 +206,7 @@ pub(crate) mod rustc {
     impl<'tcx> Tree<Def<'tcx>, Ref<'tcx>> {
         pub fn from_ty(ty: Ty<'tcx>, cx: LayoutCx<'tcx, TyCtxt<'tcx>>) -> Result<Self, Err> {
             use rustc_target::abi::HasDataLayout;
-            let layout = ty_layout(cx, ty);
+            let layout = layout_of(cx, ty)?;
 
             if let Err(e) = ty.error_reported() {
                 return Err(Err::TypeError(e));
@@ -239,7 +239,7 @@ pub(crate) mod rustc {
                     let FieldsShape::Array { stride, count } = &layout.fields else {
                         return Err(Err::NotYetSupported);
                     };
-                    let inner_layout = ty_layout(cx, *inner_ty);
+                    let inner_layout = layout_of(cx, *inner_ty)?;
                     assert_eq!(*stride, inner_layout.size);
                     let elt = Tree::from_ty(*inner_ty, cx)?;
                     Ok(std::iter::repeat(elt)
@@ -254,7 +254,7 @@ pub(crate) mod rustc {
                 },
 
                 ty::Ref(lifetime, ty, mutability) => {
-                    let layout = ty_layout(cx, *ty);
+                    let layout = layout_of(cx, *ty)?;
                     let align = layout.align.abi.bytes_usize();
                     let size = layout.size.bytes_usize();
                     Ok(Tree::Ref(Ref {
@@ -280,7 +280,7 @@ pub(crate) mod rustc {
                 FieldsShape::Primitive => {
                     assert_eq!(members.len(), 1);
                     let inner_ty = members[0];
-                    let inner_layout = ty_layout(cx, inner_ty);
+                    let inner_layout = layout_of(cx, inner_ty)?;
                     Self::from_ty(inner_ty, cx)
                 }
                 FieldsShape::Arbitrary { offsets, .. } => {
@@ -413,7 +413,7 @@ pub(crate) mod rustc {
                 let padding = Self::padding(padding_needed.bytes_usize());
 
                 let field_ty = ty_field(cx, (ty, layout), field_idx);
-                let field_layout = ty_layout(cx, field_ty);
+                let field_layout = layout_of(cx, field_ty)?;
                 let field_tree = Self::from_ty(field_ty, cx)?;
 
                 struct_tree = struct_tree.then(padding).then(field_tree);
@@ -471,7 +471,7 @@ pub(crate) mod rustc {
                 |fields, (idx, field_def)| {
                     let field_def = Def::Field(field_def);
                     let field_ty = ty_field(cx, (ty, layout), idx);
-                    let field_layout = ty_layout(cx, field_ty);
+                    let field_layout = layout_of(cx, field_ty)?;
                     let field = Self::from_ty(field_ty, cx)?;
                     let trailing_padding_needed = layout.size - field_layout.size;
                     let trailing_padding = Self::padding(trailing_padding_needed.bytes_usize());
@@ -484,10 +484,6 @@ pub(crate) mod rustc {
         }
     }
 
-    pub(crate) fn ty_layout<'tcx>(cx: LayoutCx<'tcx, TyCtxt<'tcx>>, ty: Ty<'tcx>) -> Layout<'tcx> {
-        crate::layout::rustc::layout_of(cx, ty).unwrap()
-    }
-
     fn ty_field<'tcx>(
         cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
         (ty, layout): (Ty<'tcx>, Layout<'tcx>),

compiler/rustc_transmute/src/maybe_transmutable/mod.rs

@@ -44,18 +44,11 @@ mod rustc {
             let Self { src, dst, assume, context } = self;
 
             let layout_cx = LayoutCx { tcx: context, param_env: ParamEnv::reveal_all() };
-            let layout_of = |ty| {
-                crate::layout::rustc::layout_of(layout_cx, ty)
-                    .map_err(|_| Err::NotYetSupported)
-                    .and_then(|_| Tree::from_ty(ty, layout_cx))
-            };
 
             // Convert `src` and `dst` from their rustc representations, to `Tree`-based
-            // representations. If these conversions fail, conclude that the transmutation is
-            // unacceptable; the layouts of both the source and destination types must be
-            // well-defined.
-            let src = layout_of(src);
-            let dst = layout_of(dst);
+            // representations.
+            let src = Tree::from_ty(src, layout_cx);
+            let dst = Tree::from_ty(dst, layout_cx);
 
             match (src, dst) {
                 (Err(Err::TypeError(_)), _) | (_, Err(Err::TypeError(_))) => {

tests/ui/transmutability/arrays/huge-len.stderr

@@ -2,7 +2,7 @@ error[E0277]: `()` cannot be safely transmuted into `ExplicitlyPadded`
   --> $DIR/huge-len.rs:21:41
    |
 LL |     assert::is_maybe_transmutable::<(), ExplicitlyPadded>();
-   |                                         ^^^^^^^^^^^^^^^^ analyzing the transmutability of `ExplicitlyPadded` is not yet supported
+   |                                         ^^^^^^^^^^^^^^^^ values of the type `ExplicitlyPadded` are too big for the current architecture
    |
 note: required by a bound in `is_maybe_transmutable`
   --> $DIR/huge-len.rs:8:14
@@ -17,7 +17,7 @@ error[E0277]: `ExplicitlyPadded` cannot be safely transmuted into `()`
   --> $DIR/huge-len.rs:24:55
    |
 LL |     assert::is_maybe_transmutable::<ExplicitlyPadded, ()>();
-   |                                                       ^^ analyzing the transmutability of `ExplicitlyPadded` is not yet supported
+   |                                                       ^^ values of the type `ExplicitlyPadded` are too big for the current architecture
    |
 note: required by a bound in `is_maybe_transmutable`
   --> $DIR/huge-len.rs:8:14

tests/ui/transmutability/malformed-program-gracefulness/unknown_dst_field.rs

@@ -0,0 +1,26 @@
+// An unknown destination type should be gracefully handled.
+
+#![crate_type = "lib"]
+#![feature(transmutability)]
+#![allow(dead_code, incomplete_features, non_camel_case_types)]
+
+mod assert {
+    use std::mem::BikeshedIntrinsicFrom;
+
+    pub fn is_transmutable<Src, Dst>()
+    where
+        Dst: BikeshedIntrinsicFrom<Src>
+    {}
+}
+
+fn should_gracefully_handle_unknown_dst_field() {
+    #[repr(C)] struct Src;
+    #[repr(C)] struct Dst(Missing); //~ cannot find type
+    assert::is_transmutable::<Src, Dst>(); //~ ERROR cannot be safely transmuted
+}
+
+fn should_gracefully_handle_unknown_dst_ref_field() {
+    #[repr(C)] struct Src(&'static Src);
+    #[repr(C)] struct Dst(&'static Missing); //~ cannot find type
+    assert::is_transmutable::<Src, Dst>(); //~ ERROR cannot be safely transmuted
+}

tests/ui/transmutability/malformed-program-gracefulness/unknown_dst_field.stderr

@@ -0,0 +1,46 @@
+error[E0412]: cannot find type `Missing` in this scope
+  --> $DIR/unknown_dst_field.rs:18:27
+   |
+LL |     #[repr(C)] struct Dst(Missing);
+   |                           ^^^^^^^ not found in this scope
+
+error[E0412]: cannot find type `Missing` in this scope
+  --> $DIR/unknown_dst_field.rs:24:36
+   |
+LL |     #[repr(C)] struct Dst(&'static Missing);
+   |                                    ^^^^^^^ not found in this scope
+
+error[E0277]: `should_gracefully_handle_unknown_dst_field::Src` cannot be safely transmuted into `should_gracefully_handle_unknown_dst_field::Dst`
+  --> $DIR/unknown_dst_field.rs:19:36
+   |
+LL |     assert::is_transmutable::<Src, Dst>();
+   |                                    ^^^ `should_gracefully_handle_unknown_dst_field::Dst` has an unknown layout
+   |
+note: required by a bound in `is_transmutable`
+  --> $DIR/unknown_dst_field.rs:12:14
+   |
+LL |     pub fn is_transmutable<Src, Dst>()
+   |            --------------- required by a bound in this function
+LL |     where
+LL |         Dst: BikeshedIntrinsicFrom<Src>
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_transmutable`
+
+error[E0277]: `should_gracefully_handle_unknown_dst_ref_field::Src` cannot be safely transmuted into `should_gracefully_handle_unknown_dst_ref_field::Dst`
+  --> $DIR/unknown_dst_field.rs:25:36
+   |
+LL |     assert::is_transmutable::<Src, Dst>();
+   |                                    ^^^ `should_gracefully_handle_unknown_dst_ref_field::Dst` has an unknown layout
+   |
+note: required by a bound in `is_transmutable`
+  --> $DIR/unknown_dst_field.rs:12:14
+   |
+LL |     pub fn is_transmutable<Src, Dst>()
+   |            --------------- required by a bound in this function
+LL |     where
+LL |         Dst: BikeshedIntrinsicFrom<Src>
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_transmutable`
+
+error: aborting due to 4 previous errors
+
+Some errors have detailed explanations: E0277, E0412.
+For more information about an error, try `rustc --explain E0277`.

tests/ui/transmutability/malformed-program-gracefulness/unknown_src_field.rs

@@ -13,8 +13,14 @@ mod assert {
     {}
 }
 
-fn should_gracefully_handle_unknown_dst_field() {
-    #[repr(C)] struct Src;
-    #[repr(C)] struct Dst(Missing); //~ cannot find type
+fn should_gracefully_handle_unknown_src_field() {
+    #[repr(C)] struct Src(Missing); //~ cannot find type
+    #[repr(C)] struct Dst();
+    assert::is_transmutable::<Src, Dst>(); //~ ERROR cannot be safely transmuted
+}
+
+fn should_gracefully_handle_unknown_src_ref_field() {
+    #[repr(C)] struct Src(&'static Missing); //~ cannot find type
+    #[repr(C)] struct Dst(&'static Dst);
     assert::is_transmutable::<Src, Dst>(); //~ ERROR cannot be safely transmuted
 }

tests/ui/transmutability/malformed-program-gracefulness/unknown_src_field.stderr

@@ -1,14 +1,35 @@
 error[E0412]: cannot find type `Missing` in this scope
-  --> $DIR/unknown_src_field.rs:18:27
+  --> $DIR/unknown_src_field.rs:17:27
    |
-LL |     #[repr(C)] struct Dst(Missing);
+LL |     #[repr(C)] struct Src(Missing);
    |                           ^^^^^^^ not found in this scope
 
-error[E0277]: `Src` cannot be safely transmuted into `Dst`
+error[E0412]: cannot find type `Missing` in this scope
+  --> $DIR/unknown_src_field.rs:23:36
+   |
+LL |     #[repr(C)] struct Src(&'static Missing);
+   |                                    ^^^^^^^ not found in this scope
+
+error[E0277]: `should_gracefully_handle_unknown_src_field::Src` cannot be safely transmuted into `should_gracefully_handle_unknown_src_field::Dst`
   --> $DIR/unknown_src_field.rs:19:36
    |
 LL |     assert::is_transmutable::<Src, Dst>();
-   |                                    ^^^ analyzing the transmutability of `Dst` is not yet supported
+   |                                    ^^^ `should_gracefully_handle_unknown_src_field::Src` has an unknown layout
+   |
+note: required by a bound in `is_transmutable`
+  --> $DIR/unknown_src_field.rs:12:14
+   |
+LL |     pub fn is_transmutable<Src, Dst>()
+   |            --------------- required by a bound in this function
+LL |     where
+LL |         Dst: BikeshedIntrinsicFrom<Src>
+   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_transmutable`
+
+error[E0277]: `should_gracefully_handle_unknown_src_ref_field::Src` cannot be safely transmuted into `should_gracefully_handle_unknown_src_ref_field::Dst`
+  --> $DIR/unknown_src_field.rs:25:36
+   |
+LL |     assert::is_transmutable::<Src, Dst>();
+   |                                    ^^^ `should_gracefully_handle_unknown_src_ref_field::Src` has an unknown layout
    |
 note: required by a bound in `is_transmutable`
   --> $DIR/unknown_src_field.rs:12:14
@@ -19,7 +40,7 @@ LL |     where
 LL |         Dst: BikeshedIntrinsicFrom<Src>
    |              ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_transmutable`
 
-error: aborting due to 2 previous errors
+error: aborting due to 4 previous errors
 
 Some errors have detailed explanations: E0277, E0412.
 For more information about an error, try `rustc --explain E0277`.

tests/ui/transmutability/structs/repr/transmute_infinitely_recursive_type.rs

@@ -22,5 +22,4 @@ fn should_pad_explicitly_packed_field() {
     //~^ ERROR: recursive type
 
     assert::is_maybe_transmutable::<ExplicitlyPadded, ()>();
-    //~^ ERROR: cannot be safely transmuted
 }

tests/ui/transmutability/structs/repr/transmute_infinitely_recursive_type.stderr

@@ -15,22 +15,7 @@ error[E0391]: cycle detected when computing layout of `should_pad_explicitly_pac
    = note: cycle used when evaluating trait selection obligation `(): core::mem::transmutability::BikeshedIntrinsicFrom<should_pad_explicitly_packed_field::ExplicitlyPadded, core::mem::transmutability::Assume { alignment: false, lifetimes: false, safety: false, validity: false }>`
    = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
 
-error[E0277]: `ExplicitlyPadded` cannot be safely transmuted into `()`
-  --> $DIR/transmute_infinitely_recursive_type.rs:24:55
-   |
-LL |     assert::is_maybe_transmutable::<ExplicitlyPadded, ()>();
-   |                                                       ^^ analyzing the transmutability of `ExplicitlyPadded` is not yet supported
-   |
-note: required by a bound in `is_maybe_transmutable`
-  --> $DIR/transmute_infinitely_recursive_type.rs:14:14
-   |
-LL |     pub fn is_maybe_transmutable<Src, Dst>()
-   |            --------------------- required by a bound in this function
-LL |     where
-LL |         Dst: BikeshedIntrinsicFrom<Src>,
-   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `is_maybe_transmutable`
-
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors
 
-Some errors have detailed explanations: E0072, E0277, E0391.
+Some errors have detailed explanations: E0072, E0391.
 For more information about an error, try `rustc --explain E0072`.