1//===-- AMDGPUUniformIntrinsicCombine.cpp ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass simplifies certain intrinsic calls when the arguments are uniform.
11/// It's true that this pass has transforms that can lead to a situation where
12/// some instruction whose operand was previously recognized as statically
13/// uniform is later on no longer recognized as statically uniform. However, the
14/// semantics of how programs execute don't (and must not, for this precise
15/// reason) care about static uniformity, they only ever care about dynamic
16/// uniformity. And every instruction that's downstream and cares about dynamic
17/// uniformity must be convergent (and isel will introduce v_readfirstlane for
18/// them if their operands can't be proven statically uniform).
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "llvm/Analysis/LoopInfo.h"
23#include "llvm/Analysis/ScalarEvolution.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/Analysis/UniformityAnalysis.h"
26#include "llvm/CodeGen/TargetPassConfig.h"
27#include "llvm/IR/IRBuilder.h"
28#include "llvm/IR/InstIterator.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Target/TargetMachine.h"
33#include "llvm/Transforms/Utils/BasicBlockUtils.h"
34
35#define DEBUG_TYPE "amdgpu-uniform-intrinsic-combine"
36
37using namespace llvm;
38using namespace llvm::AMDGPU;
39using namespace llvm::PatternMatch;
40
41/// Optimizes uniform intrinsics calls if their operand can be proven uniform.
42static bool optimizeUniformIntrinsic(IntrinsicInst &II, UniformityInfo &UI) {
43 llvm::Intrinsic::ID IID = II.getIntrinsicID();
44 /// We deliberately do not simplify readfirstlane with a uniform argument, so
45 /// that frontends can use it to force a copy to SGPR and thereby prevent the
46 /// backend from generating unwanted waterfall loops.
47 switch (IID) {
48 case Intrinsic::amdgcn_permlane64:
49 case Intrinsic::amdgcn_readlane: {
50 Value *Src = II.getArgOperand(i: 0);
51 if (UI.isDivergentAtUse(U: II.getOperandUse(i: 0)))
52 return false;
53 LLVM_DEBUG(dbgs() << "Replacing " << II << " with " << *Src << '\n');
54 II.replaceAllUsesWith(V: Src);
55 UI.forgetValue(V: &II);
56 II.eraseFromParent();
57 return true;
58 }
59 case Intrinsic::amdgcn_ballot: {
60 Value *Src = II.getArgOperand(i: 0);
61 if (UI.isDivergentAtUse(U: II.getOperandUse(i: 0)))
62 return false;
63 LLVM_DEBUG(dbgs() << "Found uniform ballot intrinsic: " << II << '\n');
64
65 bool Changed = false;
66 for (User *U : make_early_inc_range(Range: II.users())) {
67 if (auto *ICmp = dyn_cast<ICmpInst>(Val: U)) {
68 Value *Op0 = ICmp->getOperand(i_nocapture: 0);
69 Value *Op1 = ICmp->getOperand(i_nocapture: 1);
70 ICmpInst::Predicate Pred = ICmp->getPredicate();
71 Value *OtherOp = Op0 == &II ? Op1 : Op0;
72
73 if (Pred == ICmpInst::ICMP_EQ && match(V: OtherOp, P: m_Zero())) {
74 // Case: (icmp eq %ballot, 0) -> xor %ballot_arg, 1
75 Instruction *NotOp =
76 BinaryOperator::CreateNot(Op: Src, Name: "", InsertBefore: ICmp->getIterator());
77 LLVM_DEBUG(dbgs() << "Replacing ICMP_EQ: " << *NotOp << '\n');
78 ICmp->replaceAllUsesWith(V: NotOp);
79 Changed = true;
80 } else if (Pred == ICmpInst::ICMP_NE && match(V: OtherOp, P: m_Zero())) {
81 // Case: (icmp ne %ballot, 0) -> %ballot_arg
82 LLVM_DEBUG(dbgs() << "Replacing ICMP_NE with ballot argument: "
83 << *Src << '\n');
84 ICmp->replaceAllUsesWith(V: Src);
85 Changed = true;
86 }
87 }
88 }
89 // Erase the intrinsic if it has no remaining uses.
90 if (II.use_empty()) {
91 UI.forgetValue(V: &II);
92 II.eraseFromParent();
93 Changed = true;
94 }
95 return Changed;
96 }
97 case Intrinsic::amdgcn_wave_shuffle: {
98 Use &Val = II.getOperandUse(i: 0);
99 Use &Idx = II.getOperandUse(i: 1);
100
101 // Like with readlane, if Value is uniform then just propagate it
102 if (!UI.isDivergentAtUse(U: Val)) {
103 II.replaceAllUsesWith(V: Val);
104 UI.forgetValue(V: &II);
105 II.eraseFromParent();
106 return true;
107 }
108
109 // Otherwise, when Index is uniform, this is just a readlane operation
110 if (UI.isDivergentAtUse(U: Idx))
111 return false;
112
113 // The readlane intrinsic we want to call has the exact same function
114 // signature, so we can quickly modify the instruction in-place
115 Module *Mod = II.getModule();
116 II.setCalledFunction(Intrinsic::getOrInsertDeclaration(
117 M: Mod, id: Intrinsic::amdgcn_readlane, OverloadTys: II.getType()));
118 return true;
119 }
120 default:
121 return false;
122 }
123 return false;
124}
125
126/// Fold `icmp pred (readfirstlane X), X` when X is uniform at that use.
127static bool optimizeUniformCmpReadFirstLane(ICmpInst &Cmp, UniformityInfo &UI) {
128 Value *X;
129 IntrinsicInst *RFL;
130 if (!match(V: &Cmp,
131 P: m_c_ICmp(L: m_CombineAnd(Ps: m_Intrinsic<Intrinsic::amdgcn_readfirstlane>(
132 Ops: m_Value(V&: X)),
133 Ps: m_AnyIntrinsic(I&: RFL)),
134 R: m_Deferred(V: X))))
135 return false;
136
137 if (UI.isDivergentAtUse(U: RFL->getArgOperandUse(i: 0)))
138 return false;
139
140 // If readfirstlane's operand is uniform then we know we are comparing two
141 // identical values.
142 bool Holds = ICmpInst::isTrueWhenEqual(predicate: Cmp.getPredicate());
143 LLVM_DEBUG(dbgs() << "Uniform readfirstlane operand, replacing " << Cmp
144 << " with " << (Holds ? "true" : "false") << '\n');
145 Cmp.replaceAllUsesWith(V: ConstantInt::get(Ty: Cmp.getType(), V: Holds));
146 UI.forgetValue(V: &Cmp);
147 Cmp.eraseFromParent();
148 return true;
149}
150
151/// Iterates over the Function, optimizing uniform intrinsic calls and
152/// comparisons against a readfirstlane of a uniform value.
153static bool runUniformIntrinsicCombine(Function &F, UniformityInfo &UI) {
154 bool IsChanged = false;
155 for (Instruction &I : make_early_inc_range(Range: instructions(F))) {
156 if (auto *Cmp = dyn_cast<ICmpInst>(Val: &I)) {
157 IsChanged |= optimizeUniformCmpReadFirstLane(Cmp&: *Cmp, UI);
158 continue;
159 }
160 auto *II = dyn_cast<IntrinsicInst>(Val: &I);
161 if (!II)
162 continue;
163 IsChanged |= optimizeUniformIntrinsic(II&: *II, UI);
164 }
165 return IsChanged;
166}
167
168PreservedAnalyses
169AMDGPUUniformIntrinsicCombinePass::run(Function &F,
170 FunctionAnalysisManager &AM) {
171 UniformityInfo &UI = AM.getResult<UniformityInfoAnalysis>(IR&: F);
172 if (!runUniformIntrinsicCombine(F, UI))
173 return PreservedAnalyses::all();
174
175 PreservedAnalyses PA;
176 PA.preserveSet<CFGAnalyses>();
177 return PA;
178}
179
180namespace {
181class AMDGPUUniformIntrinsicCombineLegacy : public FunctionPass {
182public:
183 static char ID;
184 AMDGPUUniformIntrinsicCombineLegacy() : FunctionPass(ID) {}
185
186private:
187 bool runOnFunction(Function &F) override;
188 void getAnalysisUsage(AnalysisUsage &AU) const override {
189 AU.setPreservesCFG();
190 AU.addRequired<UniformityInfoWrapperPass>();
191 AU.addRequired<TargetPassConfig>();
192 }
193};
194} // namespace
195
196char AMDGPUUniformIntrinsicCombineLegacy::ID = 0;
197char &llvm::AMDGPUUniformIntrinsicCombineLegacyPassID =
198 AMDGPUUniformIntrinsicCombineLegacy::ID;
199
200bool AMDGPUUniformIntrinsicCombineLegacy::runOnFunction(Function &F) {
201 if (skipFunction(F))
202 return false;
203 UniformityInfo &UI =
204 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
205 return runUniformIntrinsicCombine(F, UI);
206}
207
208INITIALIZE_PASS_BEGIN(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,
209 "AMDGPU Uniform Intrinsic Combine", false, false)
210INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
211INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
212INITIALIZE_PASS_END(AMDGPUUniformIntrinsicCombineLegacy, DEBUG_TYPE,
213 "AMDGPU Uniform Intrinsic Combine", false, false)
214
215FunctionPass *llvm::createAMDGPUUniformIntrinsicCombineLegacyPass() {
216 return new AMDGPUUniformIntrinsicCombineLegacy();
217}
218