1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/IR/DiagnosticInfo.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/IntrinsicsAMDGPU.h"
21#include "llvm/InitializePasses.h"
22#include "llvm/Transforms/Utils/BasicBlockUtils.h"
23
24#define DEBUG_TYPE "amdgpu-lower-intrinsics"
25
26using namespace llvm;
27
28namespace {
29
30class AMDGPULowerIntrinsicsImpl {
31public:
32 Module &M;
33 const AMDGPUTargetMachine &TM;
34
35 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
36 : M(M), TM(TM) {}
37
38 bool run();
39
40private:
41 bool visitBarrier(IntrinsicInst &I);
42 bool visitPtrSBufferLoad(IntrinsicInst &I);
43 bool visitMonitorSleep(IntrinsicInst &I);
44 bool visitCvtScale(IntrinsicInst &I);
45 bool visitUnscaledWMMA(IntrinsicInst &I);
46};
47
48class AMDGPULowerIntrinsicsLegacy : public ModulePass {
49public:
50 static char ID;
51
52 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
53
54 bool runOnModule(Module &M) override;
55
56 void getAnalysisUsage(AnalysisUsage &AU) const override {
57 AU.addRequired<TargetPassConfig>();
58 }
59};
60
61template <class T> static void forEachCall(Function &Intrin, T Callback) {
62 for (User *U : make_early_inc_range(Range: Intrin.users())) {
63 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
64 Callback(CI);
65 }
66}
67
68} // anonymous namespace
69
70bool AMDGPULowerIntrinsicsImpl::run() {
71 bool Changed = false;
72
73 for (Function &F : M) {
74 switch (F.getIntrinsicID()) {
75 default:
76 continue;
77 case Intrinsic::amdgcn_s_barrier:
78 case Intrinsic::amdgcn_s_barrier_signal:
79 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
80 case Intrinsic::amdgcn_s_barrier_wait:
81 case Intrinsic::amdgcn_s_cluster_barrier:
82 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
83 break;
84 case Intrinsic::amdgcn_ptr_s_buffer_load:
85 forEachCall(
86 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitPtrSBufferLoad(I&: *II); });
87 break;
88 case Intrinsic::amdgcn_s_monitor_sleep:
89 forEachCall(
90 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitMonitorSleep(I&: *II); });
91 break;
92 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
93 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
94 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
95 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
96 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
97 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
98 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
99 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
100 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
101 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
102 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
103 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
104 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
105 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
106 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
107 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitCvtScale(I&: *II); });
108 break;
109 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
110 forEachCall(
111 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitUnscaledWMMA(I&: *II); });
112 break;
113 }
114 }
115
116 return Changed;
117}
118
119// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
120// barrier intrinsics.
121bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
122 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
123 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
124 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
125 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
126 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
127
128 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
129 bool IsSingleWaveWG = false;
130
131 if (TM.getOptLevel() > CodeGenOptLevel::None) {
132 unsigned WGMaxSize = ST.getFlatWorkGroupSizes(F: *I.getFunction()).second;
133 IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
134 }
135
136 IRBuilder<> B(&I);
137
138 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
139 // hardware instruction in any subtarget.
140 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
141 // The default cluster barrier expects one signal per workgroup. So we need
142 // a workgroup barrier first.
143 if (IsSingleWaveWG) {
144 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
145 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
146 ->copyMetadata(SrcInst: I);
147 } else {
148 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
149 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
150 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
151 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
152 Args: {BarrierID_32});
153 IsFirst->copyMetadata(SrcInst: I);
154 B.CreateIntrinsicWithoutFolding(
155 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
156 ->copyMetadata(SrcInst: I);
157
158 Instruction *ThenTerm =
159 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
160 B.SetInsertPoint(ThenTerm);
161 }
162
163 // Now we can signal the cluster barrier from a single wave and wait for the
164 // barrier in all waves.
165 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
166 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
167 B.CreateIntrinsicWithoutFolding(
168 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
169 ->copyMetadata(SrcInst: I);
170
171 B.SetInsertPoint(&I);
172 B.CreateIntrinsicWithoutFolding(
173 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
174 ->copyMetadata(SrcInst: I);
175
176 I.eraseFromParent();
177 return true;
178 }
179
180 bool IsWorkgroupScope = false;
181
182 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
183 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
184 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
185 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
186 if (BarrierID == AMDGPU::Barrier::TRAP ||
187 BarrierID == AMDGPU::Barrier::WORKGROUP ||
188 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
189 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
190 IsWorkgroupScope = true;
191 else if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst &&
192 BarrierID == AMDGPU::Barrier::CLUSTER) {
193 I.getContext().diagnose(
194 DI: DiagnosticInfoUnsupported(*I.getFunction(),
195 "s_barrier_signal_isfirst does not support "
196 "user_cluster_barrier_id (-3)",
197 I.getDebugLoc()));
198 }
199 } else {
200 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
201 IsWorkgroupScope = true;
202 }
203
204 if (IsWorkgroupScope && IsSingleWaveWG) {
205 // Down-grade waits, remove split signals.
206 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
207 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
208 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
209 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
210 ->copyMetadata(SrcInst: I);
211 } else if (I.getIntrinsicID() ==
212 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
213 // If we're the only wave of the workgroup, we're always first.
214 I.replaceAllUsesWith(V: B.getInt1(V: true));
215 }
216 I.eraseFromParent();
217 return true;
218 }
219
220 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
221 ST.hasSplitBarriers()) {
222 // Lower to split barriers.
223 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
224 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
225 B.CreateIntrinsicWithoutFolding(
226 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
227 ->copyMetadata(SrcInst: I);
228 B.CreateIntrinsicWithoutFolding(
229 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
230 ->copyMetadata(SrcInst: I);
231 I.eraseFromParent();
232 return true;
233 }
234
235 return false;
236}
237
238bool AMDGPULowerIntrinsicsImpl::visitPtrSBufferLoad(IntrinsicInst &I) {
239 assert(I.getIntrinsicID() == Intrinsic::amdgcn_ptr_s_buffer_load);
240
241 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
242 return false;
243
244 I.setMetadata(KindID: LLVMContext::MD_invariant_load,
245 Node: MDNode::get(Context&: I.getContext(), MDs: {}));
246 return true;
247}
248
249bool AMDGPULowerIntrinsicsImpl::visitMonitorSleep(IntrinsicInst &I) {
250 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_monitor_sleep);
251
252 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
253 if (!ST.hasNoSleepForever())
254 return false;
255
256 int Sleep = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
257 if (!(Sleep & 0x8000))
258 return false;
259
260 IRBuilder<> B(&I);
261 Value *NewSleep = B.getInt16(C: 0x2000); // Maximum
262 I.setArgOperand(i: 0, v: NewSleep);
263
264 return true;
265}
266
267bool AMDGPULowerIntrinsicsImpl::visitCvtScale(IntrinsicInst &I) {
268 int MaxSel = 0;
269 switch (I.getIntrinsicID()) {
270 default:
271 llvm_unreachable("expected cvt_scale_* intrinsic");
272 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
273 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
274 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
275 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
276 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
277 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
278 MaxSel = 8;
279 break;
280 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
281 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
282 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
283 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
284 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
285 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
286 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
287 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
288 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
289 MaxSel = 4;
290 break;
291 }
292
293 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
294 if (ST.hasBlock16ConversionScaleInsts())
295 MaxSel *= 2;
296
297 int ScaleSel = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getSExtValue();
298 if (ScaleSel < MaxSel)
299 return false;
300
301 I.getContext().diagnose(DI: DiagnosticInfoUnsupported(
302 *I.getFunction(),
303 I.getCalledFunction()->getName() +
304 Twine(" scale_sel maximum supported value is ") + Twine(MaxSel - 1),
305 I.getDebugLoc()));
306
307 return false;
308}
309
310bool AMDGPULowerIntrinsicsImpl::visitUnscaledWMMA(IntrinsicInst &I) {
311 assert(I.getIntrinsicID() == Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4);
312
313 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
314 if (!ST.hasGFX1250_STRICT())
315 return false;
316
317 IRBuilder<> B(&I);
318
319 // Convert unscaled WMMA into a scaled version with scale factors 0.
320 SmallVector<Value *, 14> Args(I.args());
321 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_a_scale
322 Args.push_back(
323 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_a_scale_fmt
324 Args.push_back(Elt: B.getInt32(C: 0)); // matrix a scale exponential
325 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_b_scale
326 Args.push_back(
327 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_b_scale_fmt
328 Args.push_back(Elt: B.getInt32(C: 0)); // matrix b scale exponential
329 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_a_reuse
330 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_b_reuse
331
332 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
333 RetTy: I.getType(), ID: Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4, Args);
334 I.replaceAllUsesWith(V: NewI);
335 NewI->copyMetadata(SrcInst: I);
336 NewI->takeName(V: &I);
337 I.eraseFromParent();
338
339 return true;
340}
341
342PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
343 ModuleAnalysisManager &MAM) {
344 AMDGPULowerIntrinsicsImpl Impl(M, TM);
345 if (!Impl.run())
346 return PreservedAnalyses::all();
347 return PreservedAnalyses::none();
348}
349
350bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
351 auto &TPC = getAnalysis<TargetPassConfig>();
352 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
353
354 AMDGPULowerIntrinsicsImpl Impl(M, TM);
355 return Impl.run();
356}
357
358#define PASS_DESC "AMDGPU lower intrinsics"
359INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
360 false)
361INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
362INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
363 false)
364
365char AMDGPULowerIntrinsicsLegacy::ID = 0;
366
367ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
368 return new AMDGPULowerIntrinsicsLegacy;
369}
370