1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/IR/DiagnosticInfo.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/IntrinsicsAMDGPU.h"
21#include "llvm/InitializePasses.h"
22#include "llvm/Transforms/Utils/BasicBlockUtils.h"
23
24#define DEBUG_TYPE "amdgpu-lower-intrinsics"
25
26using namespace llvm;
27
28namespace {
29
30class AMDGPULowerIntrinsicsImpl {
31public:
32 Module &M;
33 const AMDGPUTargetMachine &TM;
34
35 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
36 : M(M), TM(TM) {}
37
38 bool run();
39
40private:
41 bool visitBarrier(IntrinsicInst &I);
42 bool visitPtrSBufferLoad(IntrinsicInst &I);
43 bool visitMonitorSleep(IntrinsicInst &I);
44};
45
46class AMDGPULowerIntrinsicsLegacy : public ModulePass {
47public:
48 static char ID;
49
50 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
51
52 bool runOnModule(Module &M) override;
53
54 void getAnalysisUsage(AnalysisUsage &AU) const override {
55 AU.addRequired<TargetPassConfig>();
56 }
57};
58
59template <class T> static void forEachCall(Function &Intrin, T Callback) {
60 for (User *U : make_early_inc_range(Range: Intrin.users())) {
61 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
62 Callback(CI);
63 }
64}
65
66} // anonymous namespace
67
68bool AMDGPULowerIntrinsicsImpl::run() {
69 bool Changed = false;
70
71 for (Function &F : M) {
72 switch (F.getIntrinsicID()) {
73 default:
74 continue;
75 case Intrinsic::amdgcn_s_barrier:
76 case Intrinsic::amdgcn_s_barrier_signal:
77 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
78 case Intrinsic::amdgcn_s_barrier_wait:
79 case Intrinsic::amdgcn_s_cluster_barrier:
80 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
81 break;
82 case Intrinsic::amdgcn_ptr_s_buffer_load:
83 forEachCall(
84 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitPtrSBufferLoad(I&: *II); });
85 break;
86 case Intrinsic::amdgcn_s_monitor_sleep:
87 forEachCall(
88 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitMonitorSleep(I&: *II); });
89 break;
90 }
91 }
92
93 return Changed;
94}
95
96// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
97// barrier intrinsics.
98bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
99 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
100 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
101 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
102 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
103 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
104
105 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
106 bool IsSingleWaveWG = false;
107
108 if (TM.getOptLevel() > CodeGenOptLevel::None) {
109 unsigned WGMaxSize = ST.getFlatWorkGroupSizes(F: *I.getFunction()).second;
110 IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
111 }
112
113 IRBuilder<> B(&I);
114
115 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
116 // hardware instruction in any subtarget.
117 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
118 // The default cluster barrier expects one signal per workgroup. So we need
119 // a workgroup barrier first.
120 if (IsSingleWaveWG) {
121 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
122 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
123 ->copyMetadata(SrcInst: I);
124 } else {
125 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
126 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
127 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
128 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
129 Args: {BarrierID_32});
130 IsFirst->copyMetadata(SrcInst: I);
131 B.CreateIntrinsicWithoutFolding(
132 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
133 ->copyMetadata(SrcInst: I);
134
135 Instruction *ThenTerm =
136 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
137 B.SetInsertPoint(ThenTerm);
138 }
139
140 // Now we can signal the cluster barrier from a single wave and wait for the
141 // barrier in all waves.
142 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
143 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
144 B.CreateIntrinsicWithoutFolding(
145 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
146 ->copyMetadata(SrcInst: I);
147
148 B.SetInsertPoint(&I);
149 B.CreateIntrinsicWithoutFolding(
150 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
151 ->copyMetadata(SrcInst: I);
152
153 I.eraseFromParent();
154 return true;
155 }
156
157 bool IsWorkgroupScope = false;
158
159 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
160 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
161 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
162 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
163 if (BarrierID == AMDGPU::Barrier::TRAP ||
164 BarrierID == AMDGPU::Barrier::WORKGROUP ||
165 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
166 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
167 IsWorkgroupScope = true;
168 else if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst &&
169 BarrierID == AMDGPU::Barrier::CLUSTER) {
170 I.getContext().diagnose(
171 DI: DiagnosticInfoUnsupported(*I.getFunction(),
172 "s_barrier_signal_isfirst does not support "
173 "user_cluster_barrier_id (-3)",
174 I.getDebugLoc()));
175 }
176 } else {
177 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
178 IsWorkgroupScope = true;
179 }
180
181 if (IsWorkgroupScope && IsSingleWaveWG) {
182 // Down-grade waits, remove split signals.
183 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
184 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
185 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
186 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
187 ->copyMetadata(SrcInst: I);
188 } else if (I.getIntrinsicID() ==
189 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
190 // If we're the only wave of the workgroup, we're always first.
191 I.replaceAllUsesWith(V: B.getInt1(V: true));
192 }
193 I.eraseFromParent();
194 return true;
195 }
196
197 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
198 ST.hasSplitBarriers()) {
199 // Lower to split barriers.
200 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
201 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
202 B.CreateIntrinsicWithoutFolding(
203 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
204 ->copyMetadata(SrcInst: I);
205 B.CreateIntrinsicWithoutFolding(
206 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
207 ->copyMetadata(SrcInst: I);
208 I.eraseFromParent();
209 return true;
210 }
211
212 return false;
213}
214
215bool AMDGPULowerIntrinsicsImpl::visitPtrSBufferLoad(IntrinsicInst &I) {
216 assert(I.getIntrinsicID() == Intrinsic::amdgcn_ptr_s_buffer_load);
217
218 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
219 return false;
220
221 I.setMetadata(KindID: LLVMContext::MD_invariant_load,
222 Node: MDNode::get(Context&: I.getContext(), MDs: {}));
223 return true;
224}
225
226bool AMDGPULowerIntrinsicsImpl::visitMonitorSleep(IntrinsicInst &I) {
227 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_monitor_sleep);
228
229 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
230 if (!ST.hasNoSleepForever())
231 return false;
232
233 int Sleep = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
234 if (!(Sleep & 0x8000))
235 return false;
236
237 IRBuilder<> B(&I);
238 Value *NewSleep = B.getInt16(C: 0x2000); // Maximum
239 I.setArgOperand(i: 0, v: NewSleep);
240
241 return true;
242}
243
244PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
245 ModuleAnalysisManager &MAM) {
246 AMDGPULowerIntrinsicsImpl Impl(M, TM);
247 if (!Impl.run())
248 return PreservedAnalyses::all();
249 return PreservedAnalyses::none();
250}
251
252bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
253 auto &TPC = getAnalysis<TargetPassConfig>();
254 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
255
256 AMDGPULowerIntrinsicsImpl Impl(M, TM);
257 return Impl.run();
258}
259
260#define PASS_DESC "AMDGPU lower intrinsics"
261INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
262 false)
263INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
264INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
265 false)
266
267char AMDGPULowerIntrinsicsLegacy::ID = 0;
268
269ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
270 return new AMDGPULowerIntrinsicsLegacy;
271}
272