1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/IntrinsicInst.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/InitializePasses.h"
21#include "llvm/Transforms/Utils/BasicBlockUtils.h"
22
23#define DEBUG_TYPE "amdgpu-lower-intrinsics"
24
25using namespace llvm;
26
27namespace {
28
29class AMDGPULowerIntrinsicsImpl {
30public:
31 Module &M;
32 const AMDGPUTargetMachine &TM;
33
34 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
35 : M(M), TM(TM) {}
36
37 bool run();
38
39private:
40 bool visitBarrier(IntrinsicInst &I);
41 bool visitPtrSBufferLoad(IntrinsicInst &I);
42};
43
44class AMDGPULowerIntrinsicsLegacy : public ModulePass {
45public:
46 static char ID;
47
48 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
49
50 bool runOnModule(Module &M) override;
51
52 void getAnalysisUsage(AnalysisUsage &AU) const override {
53 AU.addRequired<TargetPassConfig>();
54 }
55};
56
57template <class T> static void forEachCall(Function &Intrin, T Callback) {
58 for (User *U : make_early_inc_range(Range: Intrin.users())) {
59 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
60 Callback(CI);
61 }
62}
63
64} // anonymous namespace
65
66bool AMDGPULowerIntrinsicsImpl::run() {
67 bool Changed = false;
68
69 for (Function &F : M) {
70 switch (F.getIntrinsicID()) {
71 default:
72 continue;
73 case Intrinsic::amdgcn_s_barrier:
74 case Intrinsic::amdgcn_s_barrier_signal:
75 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
76 case Intrinsic::amdgcn_s_barrier_wait:
77 case Intrinsic::amdgcn_s_cluster_barrier:
78 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
79 break;
80 case Intrinsic::amdgcn_ptr_s_buffer_load:
81 forEachCall(
82 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitPtrSBufferLoad(I&: *II); });
83 break;
84 }
85 }
86
87 return Changed;
88}
89
90// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
91// barrier intrinsics.
92bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
93 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
94 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
95 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
96 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
97 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
98
99 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
100 bool IsSingleWaveWG = false;
101
102 if (TM.getOptLevel() > CodeGenOptLevel::None) {
103 unsigned WGMaxSize = ST.getFlatWorkGroupSizes(F: *I.getFunction()).second;
104 IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
105 }
106
107 IRBuilder<> B(&I);
108
109 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
110 // hardware instruction in any subtarget.
111 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
112 // The default cluster barrier expects one signal per workgroup. So we need
113 // a workgroup barrier first.
114 if (IsSingleWaveWG) {
115 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
116 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
117 ->copyMetadata(SrcInst: I);
118 } else {
119 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
120 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
121 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
122 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
123 Args: {BarrierID_32});
124 IsFirst->copyMetadata(SrcInst: I);
125 B.CreateIntrinsicWithoutFolding(
126 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
127 ->copyMetadata(SrcInst: I);
128
129 Instruction *ThenTerm =
130 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
131 B.SetInsertPoint(ThenTerm);
132 }
133
134 // Now we can signal the cluster barrier from a single wave and wait for the
135 // barrier in all waves.
136 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
137 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
138 B.CreateIntrinsicWithoutFolding(
139 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
140 ->copyMetadata(SrcInst: I);
141
142 B.SetInsertPoint(&I);
143 B.CreateIntrinsicWithoutFolding(
144 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
145 ->copyMetadata(SrcInst: I);
146
147 I.eraseFromParent();
148 return true;
149 }
150
151 bool IsWorkgroupScope = false;
152
153 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
154 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
155 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
156 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
157 if (BarrierID == AMDGPU::Barrier::TRAP ||
158 BarrierID == AMDGPU::Barrier::WORKGROUP ||
159 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
160 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
161 IsWorkgroupScope = true;
162 } else {
163 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
164 IsWorkgroupScope = true;
165 }
166
167 if (IsWorkgroupScope && IsSingleWaveWG) {
168 // Down-grade waits, remove split signals.
169 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
170 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
171 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
172 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
173 ->copyMetadata(SrcInst: I);
174 } else if (I.getIntrinsicID() ==
175 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
176 // If we're the only wave of the workgroup, we're always first.
177 I.replaceAllUsesWith(V: B.getInt1(V: true));
178 }
179 I.eraseFromParent();
180 return true;
181 }
182
183 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
184 ST.hasSplitBarriers()) {
185 // Lower to split barriers.
186 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
187 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
188 B.CreateIntrinsicWithoutFolding(
189 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
190 ->copyMetadata(SrcInst: I);
191 B.CreateIntrinsicWithoutFolding(
192 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
193 ->copyMetadata(SrcInst: I);
194 I.eraseFromParent();
195 return true;
196 }
197
198 return false;
199}
200
201bool AMDGPULowerIntrinsicsImpl::visitPtrSBufferLoad(IntrinsicInst &I) {
202 assert(I.getIntrinsicID() == Intrinsic::amdgcn_ptr_s_buffer_load);
203
204 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
205 return false;
206
207 I.setMetadata(KindID: LLVMContext::MD_invariant_load,
208 Node: MDNode::get(Context&: I.getContext(), MDs: {}));
209 return true;
210}
211
212PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
213 ModuleAnalysisManager &MAM) {
214 AMDGPULowerIntrinsicsImpl Impl(M, TM);
215 if (!Impl.run())
216 return PreservedAnalyses::all();
217 return PreservedAnalyses::none();
218}
219
220bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
221 auto &TPC = getAnalysis<TargetPassConfig>();
222 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
223
224 AMDGPULowerIntrinsicsImpl Impl(M, TM);
225 return Impl.run();
226}
227
228#define PASS_DESC "AMDGPU lower intrinsics"
229INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
230 false)
231INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
232INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
233 false)
234
235char AMDGPULowerIntrinsicsLegacy::ID = 0;
236
237ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
238 return new AMDGPULowerIntrinsicsLegacy;
239}
240