Auto merge of rust-lang#11507 - J-ZhengLi:issue11485, r=dswij

trigger [`transmute_null_to_fn`] on chain of casts

fixes: rust-lang#11485

changelog: trigger [`transmute_null_to_fn`] on chain of casts

clippy_lints/src/transmute/transmute_null_to_fn.rs

@@ -28,35 +28,43 @@ pub(super) fn check<'tcx>(cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>, arg: &'t
         return false;
     }
 
-    match arg.kind {
+    let casts_peeled = peel_casts(arg);
+    match casts_peeled.kind {
         // Catching:
         // transmute over constants that resolve to `null`.
-        ExprKind::Path(ref _qpath) if matches!(constant(cx, cx.typeck_results(), arg), Some(Constant::RawPtr(0))) => {
+        ExprKind::Path(ref _qpath)
+            if matches!(
+                constant(cx, cx.typeck_results(), casts_peeled),
+                Some(Constant::RawPtr(0))
+            ) =>
+        {
             lint_expr(cx, expr);
             true
         },
-
-        // Catching:
-        // `std::mem::transmute(0 as *const i32)`
-        ExprKind::Cast(inner_expr, _cast_ty) if is_integer_literal(inner_expr, 0) => {
-            lint_expr(cx, expr);
-            true
-        },
-
         // Catching:
         // `std::mem::transmute(std::ptr::null::<i32>())`
         ExprKind::Call(func1, []) if is_path_diagnostic_item(cx, func1, sym::ptr_null) => {
             lint_expr(cx, expr);
             true
         },
-
         _ => {
             // FIXME:
             // Also catch transmutations of variables which are known nulls.
             // To do this, MIR const propagation seems to be the better tool.
             // Whenever MIR const prop routines are more developed, this will
             // become available. As of this writing (25/03/19) it is not yet.
+            if is_integer_literal(casts_peeled, 0) {
+                lint_expr(cx, expr);
+                return true;
+            }
             false
         },
     }
 }
+
+fn peel_casts<'tcx>(expr: &'tcx Expr<'tcx>) -> &'tcx Expr<'tcx> {
+    match &expr.kind {
+        ExprKind::Cast(inner_expr, _) => peel_casts(inner_expr),
+        _ => expr,
+    }
+}

tests/ui/transmute_null_to_fn.rs

@@ -25,6 +25,17 @@ fn transmute_const() {
     }
 }
 
+fn issue_11485() {
+    unsafe {
+        let _: fn() = std::mem::transmute(0 as *const u8 as *const ());
+        //~^ ERROR: transmuting a known null pointer into a function pointer
+        let _: fn() = std::mem::transmute(std::ptr::null::<()>() as *const u8);
+        //~^ ERROR: transmuting a known null pointer into a function pointer
+        let _: fn() = std::mem::transmute(ZPTR as *const u8);
+        //~^ ERROR: transmuting a known null pointer into a function pointer
+    }
+}
+
 fn main() {
     one_liners();
     transmute_const();

tests/ui/transmute_null_to_fn.stderr

@@ -24,5 +24,29 @@ LL |         let _: fn() = std::mem::transmute(ZPTR);
    |
    = help: try wrapping your function pointer type in `Option<T>` instead, and using `None` as a null pointer value
 
-error: aborting due to 3 previous errors
+error: transmuting a known null pointer into a function pointer
+  --> $DIR/transmute_null_to_fn.rs:30:23
+   |
+LL |         let _: fn() = std::mem::transmute(0 as *const u8 as *const ());
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this transmute results in undefined behavior
+   |
+   = help: try wrapping your function pointer type in `Option<T>` instead, and using `None` as a null pointer value
+
+error: transmuting a known null pointer into a function pointer
+  --> $DIR/transmute_null_to_fn.rs:32:23
+   |
+LL |         let _: fn() = std::mem::transmute(std::ptr::null::<()>() as *const u8);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this transmute results in undefined behavior
+   |
+   = help: try wrapping your function pointer type in `Option<T>` instead, and using `None` as a null pointer value
+
+error: transmuting a known null pointer into a function pointer
+  --> $DIR/transmute_null_to_fn.rs:34:23
+   |
+LL |         let _: fn() = std::mem::transmute(ZPTR as *const u8);
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ this transmute results in undefined behavior
+   |
+   = help: try wrapping your function pointer type in `Option<T>` instead, and using `None` as a null pointer value
+
+error: aborting due to 6 previous errors