Skip to content

Commit

Permalink
Merge pull request #8130 from aschwaighofer/coro_noinline
Browse files Browse the repository at this point in the history
[Coro] [async] Disable inlining in async coroutine splitting
  • Loading branch information
aschwaighofer committed Feb 8, 2024
2 parents 50bdb94 + 41c7cea commit 99aec59
Show file tree
Hide file tree
Showing 8 changed files with 225 additions and 77 deletions.
12 changes: 1 addition & 11 deletions llvm/lib/Transforms/Coroutines/CoroSplit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -208,17 +208,12 @@ static bool replaceCoroEndAsync(AnyCoroEndInst *End) {
// Insert the return instruction.
Builder.SetInsertPoint(End);
Builder.CreateRetVoid();
InlineFunctionInfo FnInfo;

// Remove the rest of the block, by splitting it into an unreachable block.
auto *BB = End->getParent();
BB->splitBasicBlock(End);
BB->getTerminator()->eraseFromParent();

auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
assert(InlineRes.isSuccess() && "Expected inlining to succeed");
(void)InlineRes;

// We have cleaned up the coro.end block above.
return false;
}
Expand Down Expand Up @@ -1794,13 +1789,8 @@ static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
SmallVector<Value *, 8> Args(Suspend->args());
auto FnArgs = ArrayRef<Value *>(Args).drop_front(
CoroSuspendAsyncInst::MustTailCallFuncArg + 1);
auto *TailCall =
coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
Builder.CreateRetVoid();
InlineFunctionInfo FnInfo;
auto InlineRes = InlineFunction(*TailCall, FnInfo);
assert(InlineRes.isSuccess() && "Expected inlining to succeed");
(void)InlineRes;

// Replace the lvm.coro.async.resume intrisic call.
replaceAsyncResumeFunction(Suspend, Continuation);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ declare void @my_other_async_function(ptr %async.ctxt)
i32 128 ; Initial async context size without space for frame
}>

define swiftcc void @my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) {
tail call swiftcc void %fnPtr(ptr %async.ctxt)
define swifttailcc void @my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) alwaysinline {
tail call swifttailcc void %fnPtr(ptr %async.ctxt)
ret void
}

Expand All @@ -37,12 +37,12 @@ entry:

; The address of alloca escapes but the analysis based on lifetimes fails to see
; that it can't localize this alloca.
; CHECK: define swiftcc void @my_async_function(ptr swiftasync %async.ctxt) {
; CHECK: define swifttailcc void @my_async_function(ptr swiftasync %async.ctxt) {
; CHECK: entry:
; CHECK-NOT: ret
; CHECK-NOT: [[ESCAPED_ADDR:%.*]] = alloca i64, align 8
; CHECK: ret
define swiftcc void @my_async_function(ptr swiftasync %async.ctxt) {
define swifttailcc void @my_async_function(ptr swiftasync %async.ctxt) {
entry:
%escaped_addr = alloca i64

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ declare void @my_other_async_function(ptr %async.ctxt)
i32 128 ; Initial async context size without space for frame
}>

define swiftcc void @my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) {
tail call swiftcc void %fnPtr(ptr %async.ctxt)
define swifttailcc void @my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) alwaysinline {
tail call swifttailcc void %fnPtr(ptr %async.ctxt)
ret void
}

Expand All @@ -36,7 +36,7 @@ entry:
ret ptr %resume_ctxt
}

define swiftcc void @my_async_function(ptr swiftasync %async.ctxt) {
define swifttailcc void @my_async_function(ptr swiftasync %async.ctxt) {
entry:
%escaped_addr = alloca i64

Expand Down
10 changes: 5 additions & 5 deletions llvm/test/Transforms/Coroutines/coro-async-dyn-align.ll
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,20 @@ declare swiftcc void @asyncReturn(ptr)
declare swiftcc void @asyncSuspend(ptr)
declare {ptr} @llvm.coro.suspend.async(i32, ptr, ptr, ...)

define swiftcc void @my_async_function.my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) {
tail call swiftcc void %fnPtr(ptr %async.ctxt)
define swifttailcc void @my_async_function.my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt) alwaysinline {
musttail call swifttailcc void %fnPtr(ptr %async.ctxt)
ret void
}

define ptr @__swift_async_resume_project_context(ptr %ctxt) {
define ptr @__swift_async_resume_project_context(ptr %ctxt) alwaysinline {
entry:
%resume_ctxt = load ptr, ptr %ctxt, align 8
ret ptr %resume_ctxt
}


; CHECK: %my_async_function.Frame = type { i64, [48 x i8], i64, i64, [16 x i8], ptr, i64, ptr }
; CHECK: define swiftcc void @my_async_function
; CHECK: define swifttailcc void @my_async_function
; CHECK: [[T0:%.*]] = getelementptr inbounds %my_async_function.Frame, ptr %async.ctx.frameptr, i32 0, i32 3
; CHECK: [[T1:%.*]] = ptrtoint ptr [[T0]] to i64
; CHECK: [[T2:%.*]] = add i64 [[T1]], 31
Expand All @@ -60,7 +60,7 @@ entry:
; CHECK: store i64 2, ptr [[T4]]
; CHECK: store i64 3, ptr [[T9]]

define swiftcc void @my_async_function(ptr swiftasync %async.ctxt) presplitcoroutine {
define swifttailcc void @my_async_function(ptr swiftasync %async.ctxt) presplitcoroutine {
entry:
%tmp = alloca i64, align 8
%tmp2 = alloca i64, align 16
Expand Down
158 changes: 158 additions & 0 deletions llvm/test/Transforms/Coroutines/coro-async-mutal-recursive.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
; RUN: opt < %s -passes='default<O2>' -S | FileCheck --check-prefixes=CHECK %s
; RUN: opt < %s -O0 -S | FileCheck --check-prefixes=CHECK-O0 %s


; CHECK-NOT: llvm.coro.suspend.async
; CHECK-O0-NOT: llvm.coro.suspend.async

; This test used to crash during updating the call graph in coro splitting.

target datalayout = "p:64:64:64"

%swift.async_func_pointer = type <{ i32, i32 }>

@"$s1d3fooyySbYaFTu" = hidden global %swift.async_func_pointer <{ i32 trunc (i64 sub (i64 ptrtoint (ptr @"$s1d3fooyySbYaF" to i64), i64 ptrtoint (ptr @"$s1d3fooyySbYaFTu" to i64)) to i32), i32 16 }>
@"$s1d3baryySbYaFTu" = hidden global %swift.async_func_pointer <{ i32 trunc (i64 sub (i64 ptrtoint (ptr @"$s1d3baryySbYaF" to i64), i64 ptrtoint (ptr @"$s1d3baryySbYaFTu" to i64)) to i32), i32 16 }>

define swifttailcc void @"$s1d3fooyySbYaF"(ptr swiftasync %0, i1 %1) {
entry:
%2 = alloca ptr, align 8
%c.debug = alloca i1, align 8
%3 = call token @llvm.coro.id.async(i32 16, i32 16, i32 0, ptr @"$s1d3fooyySbYaFTu")
%4 = call ptr @llvm.coro.begin(token %3, ptr null)
store ptr %0, ptr %2, align 8
call void @llvm.memset.p0.i64(ptr align 8 %c.debug, i8 0, i64 1, i1 false)
store i1 %1, ptr %c.debug, align 8
call void asm sideeffect "", "r"(ptr %c.debug)
%5 = load i32, ptr getelementptr inbounds (%swift.async_func_pointer, ptr @"$s1d3baryySbYaFTu", i32 0, i32 1), align 8
%6 = zext i32 %5 to i64
%7 = call swiftcc ptr @swift_task_alloc(i64 %6) #4
call void @llvm.lifetime.start.p0(i64 -1, ptr %7)
%8 = load ptr, ptr %2, align 8
%9 = getelementptr inbounds <{ ptr, ptr }>, ptr %7, i32 0, i32 0
store ptr %8, ptr %9, align 8
%10 = call ptr @llvm.coro.async.resume()
%11 = getelementptr inbounds <{ ptr, ptr }>, ptr %7, i32 0, i32 1
store ptr %10, ptr %11, align 8
%12 = call { ptr } (i32, ptr, ptr, ...) @llvm.coro.suspend.async.sl_p0s(i32 0, ptr %10, ptr @__swift_async_resume_project_context, ptr @"$s1d3fooyySbYaF.0", ptr @"$s1d3baryySbYaF", ptr %7, i1 %1)
%13 = extractvalue { ptr } %12, 0
%14 = call ptr @__swift_async_resume_project_context(ptr %13)
store ptr %14, ptr %2, align 8
call swiftcc void @swift_task_dealloc(ptr %7) #4
call void @llvm.lifetime.end.p0(i64 -1, ptr %7)
%15 = load ptr, ptr %2, align 8
%16 = getelementptr inbounds <{ ptr, ptr }>, ptr %15, i32 0, i32 1
%17 = load ptr, ptr %16, align 8
%18 = load ptr, ptr %2, align 8
%19 = call i1 (ptr, i1, ...) @llvm.coro.end.async(ptr %4, i1 false, ptr @"$s1d3fooyySbYaF.0.1", ptr %17, ptr %18)
unreachable
}

declare token @llvm.coro.id.async(i32, i32, i32, ptr) #1

declare void @llvm.trap() #2

declare ptr @llvm.coro.begin(token, ptr) #1

declare void @llvm.memset.p0.i64(ptr nocapture, i8, i64, i1 immarg) #3

define hidden swifttailcc void @"$s1d3baryySbYaF"(ptr swiftasync %0, i1 %1) {
entry:
%2 = alloca ptr, align 8
%c.debug = alloca i1, align 8
%3 = call token @llvm.coro.id.async(i32 16, i32 16, i32 0, ptr @"$s1d3baryySbYaFTu")
%4 = call ptr @llvm.coro.begin(token %3, ptr null)
store ptr %0, ptr %2, align 8
call void @llvm.memset.p0.i64(ptr align 8 %c.debug, i8 0, i64 1, i1 false)
store i1 %1, ptr %c.debug, align 8
call void asm sideeffect "", "r"(ptr %c.debug)
br i1 %1, label %5, label %17

5: ; preds = %entry
%6 = xor i1 %1, true
%7 = load i32, ptr getelementptr inbounds (%swift.async_func_pointer, ptr @"$s1d3fooyySbYaFTu", i32 0, i32 1), align 8
%8 = zext i32 %7 to i64
%9 = call swiftcc ptr @swift_task_alloc(i64 %8) #4
call void @llvm.lifetime.start.p0(i64 -1, ptr %9)
%10 = load ptr, ptr %2, align 8
%11 = getelementptr inbounds <{ ptr, ptr }>, ptr %9, i32 0, i32 0
store ptr %10, ptr %11, align 8
%12 = call ptr @llvm.coro.async.resume()
%13 = getelementptr inbounds <{ ptr, ptr }>, ptr %9, i32 0, i32 1
store ptr %12, ptr %13, align 8
%14 = call { ptr } (i32, ptr, ptr, ...) @llvm.coro.suspend.async.sl_p0s(i32 0, ptr %12, ptr @__swift_async_resume_project_context, ptr @"$s1d3baryySbYaF.0.2", ptr @"$s1d3fooyySbYaF", ptr %9, i1 %6)
%15 = extractvalue { ptr } %14, 0
%16 = call ptr @__swift_async_resume_project_context(ptr %15)
store ptr %16, ptr %2, align 8
call swiftcc void @swift_task_dealloc(ptr %9) #4
call void @llvm.lifetime.end.p0(i64 -1, ptr %9)
br label %18

17: ; preds = %entry
br label %18

18: ; preds = %5, %17
%19 = load ptr, ptr %2, align 8
%20 = getelementptr inbounds <{ ptr, ptr }>, ptr %19, i32 0, i32 1
%21 = load ptr, ptr %20, align 8
%22 = load ptr, ptr %2, align 8
%23 = call i1 (ptr, i1, ...) @llvm.coro.end.async(ptr %4, i1 false, ptr @"$s1d3baryySbYaF.0", ptr %21, ptr %22)
unreachable
}

declare swiftcc ptr @swift_task_alloc(i64) #4

declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #5

declare ptr @llvm.coro.async.resume() #6

define linkonce_odr hidden ptr @__swift_async_resume_project_context(ptr %0) #7 {
entry:
%1 = load ptr, ptr %0, align 8
%2 = call ptr @llvm.swift.async.context.addr()
store ptr %1, ptr %2, align 8
ret ptr %1
}

declare ptr @llvm.swift.async.context.addr() #1

define internal swifttailcc void @"$s1d3fooyySbYaF.0"(ptr %0, ptr %1, i1 %2) #8 {
entry:
musttail call swifttailcc void %0(ptr swiftasync %1, i1 %2)
ret void
}

declare { ptr } @llvm.coro.suspend.async.sl_p0s(i32, ptr, ptr, ...) #6

declare swiftcc void @swift_task_dealloc(ptr) #4

declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #5

define internal swifttailcc void @"$s1d3fooyySbYaF.0.1"(ptr %0, ptr %1) #8 {
entry:
musttail call swifttailcc void %0(ptr swiftasync %1)
ret void
}

declare i1 @llvm.coro.end.async(ptr, i1, ...) #1

define internal swifttailcc void @"$s1d3baryySbYaF.0"(ptr %0, ptr %1) #8 {
entry:
musttail call swifttailcc void %0(ptr swiftasync %1)
ret void
}

define internal swifttailcc void @"$s1d3baryySbYaF.0.2"(ptr %0, ptr %1, i1 %2) #8 {
entry:
musttail call swifttailcc void %0(ptr swiftasync %1, i1 %2)
ret void
}

attributes #1 = { nounwind }
attributes #2 = { cold noreturn nounwind }
attributes #3 = { nocallback nofree nounwind willreturn}
attributes #4 = { nounwind }
attributes #5 = { nocallback nofree nosync nounwind willreturn }
attributes #6 = { nomerge nounwind }
attributes #7 = { alwaysinline nounwind }
attributes #8 = { alwaysinline nounwind }
10 changes: 5 additions & 5 deletions llvm/test/Transforms/Coroutines/coro-async-unreachable.ll
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ target datalayout = "p:64:64:64"
declare void @my_other_async_function(ptr %async.ctxt)

; Function that implements the dispatch to the callee function.
define swiftcc void @my_async_function.my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt, ptr %task, ptr %actor) {
tail call swiftcc void %fnPtr(ptr %async.ctxt, ptr %task, ptr %actor)
define swifttailcc void @my_async_function.my_other_async_function_fp.apply(ptr %fnPtr, ptr %async.ctxt, ptr %task, ptr %actor) alwaysinline {
musttail call swifttailcc void %fnPtr(ptr %async.ctxt, ptr %task, ptr %actor)
ret void
}

Expand All @@ -38,7 +38,7 @@ entry:
i32 128 ; Initial async context size without space for frame
}>

define swiftcc void @unreachable(ptr %async.ctxt, ptr %task, ptr %actor) {
define swifttailcc void @unreachable(ptr %async.ctxt, ptr %task, ptr %actor) {
entry:
%tmp = alloca { i64, i64 }, align 8
%proj.1 = getelementptr inbounds { i64, i64 }, ptr %tmp, i64 0, i32 0
Expand Down Expand Up @@ -77,11 +77,11 @@ entry:
unreachable
}

; CHECK: define swiftcc void @unreachable
; CHECK: define swifttailcc void @unreachable
; CHECK-NOT: @llvm.coro.suspend.async
; CHECK: return

; CHECK: define internal swiftcc void @unreachable.resume.0
; CHECK: define internal swifttailcc void @unreachable.resume.0
; CHECK: unreachable

declare ptr @llvm.coro.prepare.async(ptr)
Expand Down
Loading

0 comments on commit 99aec59

Please sign in to comment.