From 597ed3e19b65828102a8710d9c378536f9b4f71a Mon Sep 17 00:00:00 2001 From: Erik Desjardins Date: Thu, 10 Mar 2022 17:10:36 -0500 Subject: [PATCH 1/2] Reapply: Mark drop calls in landing pads cold instead of noinline This reverts commit 4a56cbec59903a830a5fc06c5c81956de4199584, reversing changes made to 6e5a6ffb14fc47051b0a23410c681ad6e4af045f. --- compiler/rustc_codegen_gcc/src/builder.rs | 2 +- compiler/rustc_codegen_llvm/src/builder.rs | 17 +++++++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 + compiler/rustc_codegen_llvm/src/llvm_util.rs | 6 +++ compiler/rustc_codegen_ssa/src/mir/block.rs | 10 ++--- .../rustc_codegen_ssa/src/traits/builder.rs | 2 +- .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 8 ++++ src/test/codegen/unwind-landingpad-cold.rs | 15 ++++++++ src/test/codegen/unwind-landingpad-inline.rs | 37 +++++++++++++++++++ src/test/codegen/vec-shrink-panik.rs | 19 ---------- 10 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 src/test/codegen/unwind-landingpad-cold.rs create mode 100644 src/test/codegen/unwind-landingpad-inline.rs diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index be3f6a12706f8..aa7ffb97444ca 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -1403,7 +1403,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { self.cx } - fn do_not_inline(&mut self, _llret: RValue<'gcc>) { + fn apply_attrs_to_cleanup_callsite(&mut self, _llret: RValue<'gcc>) { unimplemented!(); } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index c4eb593d2976a..9a417c71779d3 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -23,6 +23,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::Span; use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; use rustc_target::spec::{HasTargetSpec, Target}; +use smallvec::SmallVec; use std::borrow::Cow; use std::ffi::CStr; use std::iter; @@ -1188,9 +1189,19 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } } - fn do_not_inline(&mut self, llret: &'ll Value) { - let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); - attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + let mut attrs = SmallVec::<[_; 2]>::new(); + + // Cleanup is always the cold path. + attrs.push(llvm::AttributeKind::Cold.create_attr(self.llcx)); + + // In LLVM versions with deferred inlining (currently, system LLVM < 14), + // inlining drop glue can lead to exponential size blowup, see #41696 and #92110. + if !llvm_util::is_rust_llvm() && llvm_util::get_version() < (14, 0, 0) { + attrs.push(llvm::AttributeKind::NoInline.create_attr(self.llcx)); + } + + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &attrs); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 375b9927c8672..a3e55e5b6b665 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1929,6 +1929,8 @@ extern "C" { pub fn LLVMRustVersionMinor() -> u32; pub fn LLVMRustVersionPatch() -> u32; + pub fn LLVMRustIsRustLLVM() -> bool; + /// Add LLVM module flags. /// /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 4b324740a1f83..8b27166c07aa4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -261,6 +261,12 @@ pub fn get_version() -> (u32, u32, u32) { } } +/// Returns `true` if this LLVM is Rust's bundled LLVM (and not system LLVM). +pub fn is_rust_llvm() -> bool { + // Can be called without initializing LLVM + unsafe { llvm::LLVMRustIsRustLLVM() } +} + pub fn print_passes() { // Can be called without initializing LLVM unsafe { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index a87daa8d6b80b..f6c41af41b4d3 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -166,7 +166,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { bx.invoke(fn_ty, fn_ptr, &llargs, ret_llbb, unwind_block, self.funclet(fx)); bx.apply_attrs_callsite(&fn_abi, invokeret); if fx.mir[self.bb].is_cleanup { - bx.do_not_inline(invokeret); + bx.apply_attrs_to_cleanup_callsite(invokeret); } if let Some((ret_dest, target)) = destination { @@ -178,11 +178,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx)); bx.apply_attrs_callsite(&fn_abi, llret); if fx.mir[self.bb].is_cleanup { - // Cleanup is always the cold path. Don't inline - // drop glue. Also, when there is a deeply-nested - // struct, there are "symmetry" issues that cause - // exponential inlining - see issue #41696. - bx.do_not_inline(llret); + bx.apply_attrs_to_cleanup_callsite(llret); } if let Some((ret_dest, target)) = destination { @@ -1448,7 +1444,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let llret = bx.call(fn_ty, fn_ptr, &[], None); bx.apply_attrs_callsite(&fn_abi, llret); - bx.do_not_inline(llret); + bx.apply_attrs_to_cleanup_callsite(llret); bx.unreachable(); diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 37f2bfd3c4fbd..1c7fe060ea4fb 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -479,5 +479,5 @@ pub trait BuilderMethods<'a, 'tcx>: ) -> Self::Value; fn zext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value; - fn do_not_inline(&mut self, llret: Self::Value); + fn apply_attrs_to_cleanup_callsite(&mut self, llret: Self::Value); } diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 51739df067f9d..467558ee78a5b 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -663,6 +663,14 @@ extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } +extern "C" bool LLVMRustIsRustLLVM() { +#ifdef LLVM_RUSTLLVM + return true; +#else + return false; +#endif +} + extern "C" void LLVMRustAddModuleFlag( LLVMModuleRef M, Module::ModFlagBehavior MergeBehavior, diff --git a/src/test/codegen/unwind-landingpad-cold.rs b/src/test/codegen/unwind-landingpad-cold.rs new file mode 100644 index 0000000000000..aa00b7936541e --- /dev/null +++ b/src/test/codegen/unwind-landingpad-cold.rs @@ -0,0 +1,15 @@ +// no-system-llvm: needs #92110 +// compile-flags: -Cno-prepopulate-passes +#![crate_type = "lib"] + +// This test checks that drop calls in unwind landing pads +// get the `cold` attribute. + +// CHECK-LABEL: @check_cold +// CHECK: {{(call|invoke) void .+}}drop_in_place{{.+}} [[ATTRIBUTES:#[0-9]+]] +// CHECK: attributes [[ATTRIBUTES]] = { cold } +#[no_mangle] +pub fn check_cold(f: fn(), x: Box) { + // this may unwind + f(); +} diff --git a/src/test/codegen/unwind-landingpad-inline.rs b/src/test/codegen/unwind-landingpad-inline.rs new file mode 100644 index 0000000000000..ce78d075dd0f4 --- /dev/null +++ b/src/test/codegen/unwind-landingpad-inline.rs @@ -0,0 +1,37 @@ +// no-system-llvm: needs #92110 + patch for Rust alloc/dealloc functions +// compile-flags: -Copt-level=3 +#![crate_type = "lib"] + +// This test checks that we can inline drop_in_place in +// unwind landing pads. + +// Without inlining, the box pointers escape via the call to drop_in_place, +// and LLVM will not optimize out the pointer comparison. +// With inlining, everything should be optimized out. +// See https://github.com/rust-lang/rust/issues/46515 +// CHECK-LABEL: @check_no_escape_in_landingpad +// CHECK: start: +// CHECK-NEXT: ret void +#[no_mangle] +pub fn check_no_escape_in_landingpad(f: fn()) { + let x = &*Box::new(0); + let y = &*Box::new(0); + + if x as *const _ == y as *const _ { + f(); + } +} + +// Without inlining, the compiler can't tell that +// dropping an empty string (in a landing pad) does nothing. +// With inlining, the landing pad should be optimized out. +// See https://github.com/rust-lang/rust/issues/87055 +// CHECK-LABEL: @check_eliminate_noop_drop +// CHECK: start: +// CHECK-NEXT: call void %g() +// CHECK-NEXT: ret void +#[no_mangle] +pub fn check_eliminate_noop_drop(g: fn()) { + let _var = String::new(); + g(); +} diff --git a/src/test/codegen/vec-shrink-panik.rs b/src/test/codegen/vec-shrink-panik.rs index 18409014bdede..6b0ac78857a0e 100644 --- a/src/test/codegen/vec-shrink-panik.rs +++ b/src/test/codegen/vec-shrink-panik.rs @@ -16,14 +16,6 @@ pub fn shrink_to_fit(vec: &mut Vec) { // CHECK-LABEL: @issue71861 #[no_mangle] pub fn issue71861(vec: Vec) -> Box<[u32]> { - // CHECK-NOT: panic - - // Call to panic_no_unwind in case of double-panic is expected, - // but other panics are not. - // CHECK: cleanup - // CHECK-NEXT: ; call core::panicking::panic_no_unwind - // CHECK-NEXT: panic_no_unwind - // CHECK-NOT: panic vec.into_boxed_slice() } @@ -31,17 +23,6 @@ pub fn issue71861(vec: Vec) -> Box<[u32]> { // CHECK-LABEL: @issue75636 #[no_mangle] pub fn issue75636<'a>(iter: &[&'a str]) -> Box<[&'a str]> { - // CHECK-NOT: panic - - // Call to panic_no_unwind in case of double-panic is expected, - // but other panics are not. - // CHECK: cleanup - // CHECK-NEXT: ; call core::panicking::panic_no_unwind - // CHECK-NEXT: panic_no_unwind - // CHECK-NOT: panic iter.iter().copied().collect() } - -// CHECK: ; core::panicking::panic_no_unwind -// CHECK: declare void @{{.*}}panic_no_unwind From 5266aa79cfd9cd6e46552490dc3f5bd8b8057bc1 Mon Sep 17 00:00:00 2001 From: Erik Desjardins Date: Fri, 11 Mar 2022 00:12:27 -0500 Subject: [PATCH 2/2] remove is_rust_llvm check, since fixes are in final llvm 14 release --- compiler/rustc_codegen_llvm/src/builder.rs | 4 ++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2 -- compiler/rustc_codegen_llvm/src/llvm_util.rs | 6 ------ compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp | 8 -------- 4 files changed, 2 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9a417c71779d3..097180b3bd599 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1195,9 +1195,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { // Cleanup is always the cold path. attrs.push(llvm::AttributeKind::Cold.create_attr(self.llcx)); - // In LLVM versions with deferred inlining (currently, system LLVM < 14), + // LLVM 14 contains fixes for catastrophic inlining behavior, without which // inlining drop glue can lead to exponential size blowup, see #41696 and #92110. - if !llvm_util::is_rust_llvm() && llvm_util::get_version() < (14, 0, 0) { + if llvm_util::get_version() < (14, 0, 0) { attrs.push(llvm::AttributeKind::NoInline.create_attr(self.llcx)); } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index a3e55e5b6b665..375b9927c8672 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1929,8 +1929,6 @@ extern "C" { pub fn LLVMRustVersionMinor() -> u32; pub fn LLVMRustVersionPatch() -> u32; - pub fn LLVMRustIsRustLLVM() -> bool; - /// Add LLVM module flags. /// /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 8b27166c07aa4..4b324740a1f83 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -261,12 +261,6 @@ pub fn get_version() -> (u32, u32, u32) { } } -/// Returns `true` if this LLVM is Rust's bundled LLVM (and not system LLVM). -pub fn is_rust_llvm() -> bool { - // Can be called without initializing LLVM - unsafe { llvm::LLVMRustIsRustLLVM() } -} - pub fn print_passes() { // Can be called without initializing LLVM unsafe { diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 467558ee78a5b..51739df067f9d 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -663,14 +663,6 @@ extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } -extern "C" bool LLVMRustIsRustLLVM() { -#ifdef LLVM_RUSTLLVM - return true; -#else - return false; -#endif -} - extern "C" void LLVMRustAddModuleFlag( LLVMModuleRef M, Module::ModFlagBehavior MergeBehavior,