-
-
Notifications
You must be signed in to change notification settings - Fork 5.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Revert "Remove llvm-muladd pass and move it's functionality to to llv…
- Loading branch information
KristofferC
committed
Oct 24, 2024
1 parent
69ed5fd
commit 71838b9
Showing
10 changed files
with
201 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| // This file is a part of Julia. License is MIT: https://julialang.org/license | ||
|
|
||
| #include "llvm-version.h" | ||
| #include "passes.h" | ||
|
|
||
| #include <llvm-c/Core.h> | ||
| #include <llvm-c/Types.h> | ||
|
|
||
| #include <llvm/ADT/Statistic.h> | ||
| #include <llvm/Analysis/OptimizationRemarkEmitter.h> | ||
| #include <llvm/IR/Value.h> | ||
| #include <llvm/IR/PassManager.h> | ||
| #include <llvm/IR/Function.h> | ||
| #include <llvm/IR/Instructions.h> | ||
| #include <llvm/IR/IntrinsicInst.h> | ||
| #include <llvm/IR/Module.h> | ||
| #include <llvm/IR/Operator.h> | ||
| #include <llvm/IR/IRBuilder.h> | ||
| #include <llvm/IR/Verifier.h> | ||
| #include <llvm/Pass.h> | ||
| #include <llvm/Support/Debug.h> | ||
|
|
||
| #include "julia.h" | ||
| #include "julia_assert.h" | ||
|
|
||
| #define DEBUG_TYPE "combine-muladd" | ||
| #undef DEBUG | ||
|
|
||
| using namespace llvm; | ||
| STATISTIC(TotalContracted, "Total number of multiplies marked for FMA"); | ||
|
|
||
| #ifndef __clang_gcanalyzer__ | ||
| #define REMARK(remark) ORE.emit(remark) | ||
| #else | ||
| #define REMARK(remark) (void) 0; | ||
| #endif | ||
|
|
||
| /** | ||
| * Combine | ||
| * ``` | ||
| * %v0 = fmul ... %a, %b | ||
| * %v = fadd contract ... %v0, %c | ||
| * ``` | ||
| * to | ||
| * `%v = call contract @llvm.fmuladd.<...>(... %a, ... %b, ... %c)` | ||
| * when `%v0` has no other use | ||
| */ | ||
|
|
||
| // Return true if we changed the mulOp | ||
| static bool checkCombine(Value *maybeMul, OptimizationRemarkEmitter &ORE) JL_NOTSAFEPOINT | ||
| { | ||
| auto mulOp = dyn_cast<Instruction>(maybeMul); | ||
| if (!mulOp || mulOp->getOpcode() != Instruction::FMul) | ||
| return false; | ||
| if (!mulOp->hasOneUse()) { | ||
| LLVM_DEBUG(dbgs() << "mulOp has multiple uses: " << *maybeMul << "\n"); | ||
| REMARK([&](){ | ||
| return OptimizationRemarkMissed(DEBUG_TYPE, "Multiuse FMul", mulOp) | ||
| << "fmul had multiple uses " << ore::NV("fmul", mulOp); | ||
| }); | ||
| return false; | ||
| } | ||
| // On 5.0+ we only need to mark the mulOp as contract and the backend will do the work for us. | ||
| auto fmf = mulOp->getFastMathFlags(); | ||
| if (!fmf.allowContract()) { | ||
| LLVM_DEBUG(dbgs() << "Marking mulOp for FMA: " << *maybeMul << "\n"); | ||
| REMARK([&](){ | ||
| return OptimizationRemark(DEBUG_TYPE, "Marked for FMA", mulOp) | ||
| << "marked for fma " << ore::NV("fmul", mulOp); | ||
| }); | ||
| ++TotalContracted; | ||
| fmf.setAllowContract(true); | ||
| mulOp->copyFastMathFlags(fmf); | ||
| return true; | ||
| } | ||
| return false; | ||
| } | ||
|
|
||
| static bool combineMulAdd(Function &F) JL_NOTSAFEPOINT | ||
| { | ||
| OptimizationRemarkEmitter ORE(&F); | ||
| bool modified = false; | ||
| for (auto &BB: F) { | ||
| for (auto it = BB.begin(); it != BB.end();) { | ||
| auto &I = *it; | ||
| it++; | ||
| switch (I.getOpcode()) { | ||
| case Instruction::FAdd: { | ||
| if (!I.hasAllowContract()) | ||
| continue; | ||
| modified |= checkCombine(I.getOperand(0), ORE) || checkCombine(I.getOperand(1), ORE); | ||
| break; | ||
| } | ||
| case Instruction::FSub: { | ||
| if (!I.hasAllowContract()) | ||
| continue; | ||
| modified |= checkCombine(I.getOperand(0), ORE) || checkCombine(I.getOperand(1), ORE); | ||
| break; | ||
| } | ||
| default: | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| #ifdef JL_VERIFY_PASSES | ||
| assert(!verifyLLVMIR(F)); | ||
| #endif | ||
| return modified; | ||
| } | ||
|
|
||
| PreservedAnalyses CombineMulAddPass::run(Function &F, FunctionAnalysisManager &AM) JL_NOTSAFEPOINT | ||
| { | ||
| if (combineMulAdd(F)) { | ||
| return PreservedAnalyses::allInSet<CFGAnalyses>(); | ||
| } | ||
| return PreservedAnalyses::all(); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| ; This file is a part of Julia. License is MIT: https://julialang.org/license | ||
|
|
||
| ; RUN: opt -enable-new-pm=1 --opaque-pointers=0 --load-pass-plugin=libjulia-codegen%shlibext -passes='CombineMulAdd' -S %s | FileCheck %s | ||
|
|
||
| ; RUN: opt -enable-new-pm=1 --opaque-pointers=1 --load-pass-plugin=libjulia-codegen%shlibext -passes='CombineMulAdd' -S %s | FileCheck %s | ||
|
|
||
|
|
||
| ; CHECK-LABEL: @fast_muladd1 | ||
| define double @fast_muladd1(double %a, double %b, double %c) { | ||
| top: | ||
| ; CHECK: {{contract|fmuladd}} | ||
| %v1 = fmul double %a, %b | ||
| %v2 = fadd fast double %v1, %c | ||
| ; CHECK: ret double | ||
| ret double %v2 | ||
| } | ||
|
|
||
| ; CHECK-LABEL: @fast_mulsub1 | ||
| define double @fast_mulsub1(double %a, double %b, double %c) { | ||
| top: | ||
| ; CHECK: {{contract|fmuladd}} | ||
| %v1 = fmul double %a, %b | ||
| %v2 = fsub fast double %v1, %c | ||
| ; CHECK: ret double | ||
| ret double %v2 | ||
| } | ||
|
|
||
| ; CHECK-LABEL: @fast_mulsub_vec1 | ||
| define <2 x double> @fast_mulsub_vec1(<2 x double> %a, <2 x double> %b, <2 x double> %c) { | ||
| top: | ||
| ; CHECK: {{contract|fmuladd}} | ||
| %v1 = fmul <2 x double> %a, %b | ||
| %v2 = fsub fast <2 x double> %c, %v1 | ||
| ; CHECK: ret <2 x double> | ||
| ret <2 x double> %v2 | ||
| } | ||
|
|
||
| ; COM: Should not mark fmul as contract when multiple uses of fmul exist | ||
| ; CHECK-LABEL: @slow_muladd1 | ||
| define double @slow_muladd1(double %a, double %b, double %c) { | ||
| top: | ||
| ; CHECK: %v1 = fmul double %a, %b | ||
| %v1 = fmul double %a, %b | ||
| ; CHECK: %v2 = fadd fast double %v1, %c | ||
| %v2 = fadd fast double %v1, %c | ||
| ; CHECK: %v3 = fadd fast double %v1, %b | ||
| %v3 = fadd fast double %v1, %b | ||
| ; CHECK: %v4 = fadd fast double %v3, %v2 | ||
| %v4 = fadd fast double %v3, %v2 | ||
| ; CHECK: ret double %v4 | ||
| ret double %v4 | ||
| } | ||
|
|
||
| ; COM: Should not mark fadd->fadd fast as contract | ||
| ; CHECK-LABEL: @slow_addadd1 | ||
| define double @slow_addadd1(double %a, double %b, double %c) { | ||
| top: | ||
| ; CHECK: %v1 = fadd double %a, %b | ||
| %v1 = fadd double %a, %b | ||
| ; CHECK: %v2 = fadd fast double %v1, %c | ||
| %v2 = fadd fast double %v1, %c | ||
| ; CHECK: ret double %v2 | ||
| ret double %v2 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters