1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/Analysis/ValueTracking.h"
18#include "llvm/Analysis/VectorUtils.h"
19#include "llvm/IR/DiagnosticInfo.h"
20#include "llvm/IR/IRBuilder.h"
21#include "llvm/IR/IntrinsicInst.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/InitializePasses.h"
24#include "llvm/TargetParser/AMDGPUTargetParser.h"
25#include "llvm/Transforms/Utils/BasicBlockUtils.h"
26
27#define DEBUG_TYPE "amdgpu-lower-intrinsics"
28
29using namespace llvm;
30
31namespace {
32
33class AMDGPULowerIntrinsicsImpl {
34public:
35 Module &M;
36 const AMDGPUTargetMachine &TM;
37
38 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
39 : M(M), TM(TM) {}
40
41 bool run();
42
43private:
44 bool visitBarrier(IntrinsicInst &I);
45 bool visitPtrSBufferLoad(IntrinsicInst &I);
46 bool visitMonitorSleep(IntrinsicInst &I);
47 bool visitCvtScale(IntrinsicInst &I);
48 bool visitUnscaledWMMA(IntrinsicInst &I);
49 bool visitMcastLoad(IntrinsicInst &I);
50};
51
52class AMDGPULowerIntrinsicsLegacy : public ModulePass {
53public:
54 static char ID;
55
56 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
57
58 bool runOnModule(Module &M) override;
59
60 void getAnalysisUsage(AnalysisUsage &AU) const override {
61 AU.addRequired<TargetPassConfig>();
62 }
63};
64
65template <class T> static void forEachCall(Function &Intrin, T Callback) {
66 for (User *U : make_early_inc_range(Range: Intrin.users())) {
67 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
68 Callback(CI);
69 }
70}
71
72} // anonymous namespace
73
74bool AMDGPULowerIntrinsicsImpl::run() {
75 bool Changed = false;
76 const bool HasMcastFullMaskBug =
77 AMDGPU::getFeatureBitset(SubArch: M.getTargetTriple().getSubArch())
78 .test(I: AMDGPU::FEAT_MCAST_FULL_MASK_BUG);
79
80 for (Function &F : M) {
81 switch (F.getIntrinsicID()) {
82 default:
83 continue;
84 case Intrinsic::amdgcn_s_barrier:
85 case Intrinsic::amdgcn_s_barrier_signal:
86 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
87 case Intrinsic::amdgcn_s_barrier_wait:
88 case Intrinsic::amdgcn_s_cluster_barrier:
89 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
90 break;
91 case Intrinsic::amdgcn_ptr_s_buffer_load:
92 forEachCall(
93 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitPtrSBufferLoad(I&: *II); });
94 break;
95 case Intrinsic::amdgcn_s_monitor_sleep:
96 forEachCall(
97 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitMonitorSleep(I&: *II); });
98 break;
99 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
100 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
101 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
102 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
103 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
104 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
105 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
106 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
107 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
108 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
109 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
110 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
111 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
112 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
113 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
114 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitCvtScale(I&: *II); });
115 break;
116 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
117 forEachCall(
118 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitUnscaledWMMA(I&: *II); });
119 break;
120 case Intrinsic::amdgcn_cluster_load_b32:
121 case Intrinsic::amdgcn_cluster_load_b64:
122 case Intrinsic::amdgcn_cluster_load_b128:
123 case Intrinsic::amdgcn_cluster_load_async_to_lds_b8:
124 case Intrinsic::amdgcn_cluster_load_async_to_lds_b32:
125 case Intrinsic::amdgcn_cluster_load_async_to_lds_b64:
126 case Intrinsic::amdgcn_cluster_load_async_to_lds_b128:
127 case Intrinsic::amdgcn_tensor_load_to_lds:
128 if (!HasMcastFullMaskBug)
129 break;
130 forEachCall(Intrin&: F,
131 Callback: [&](IntrinsicInst *II) { Changed |= visitMcastLoad(I&: *II); });
132 break;
133 }
134 }
135
136 return Changed;
137}
138
139// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
140// barrier intrinsics.
141bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
142 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
143 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
144 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
145 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
146 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
147
148 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
149 bool IsSingleWaveWG = false;
150
151 if (TM.getOptLevel() > CodeGenOptLevel::None)
152 IsSingleWaveWG = ST.isSingleWavefrontWorkgroup(F: *I.getFunction());
153
154 IRBuilder<> B(&I);
155
156 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
157 // hardware instruction in any subtarget.
158 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
159 // The default cluster barrier expects one signal per workgroup. So we need
160 // a workgroup barrier first.
161 if (IsSingleWaveWG) {
162 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
163 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
164 ->copyMetadata(SrcInst: I);
165 } else {
166 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
167 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
168 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
169 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
170 Args: {BarrierID_32});
171 IsFirst->copyMetadata(SrcInst: I);
172 B.CreateIntrinsicWithoutFolding(
173 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
174 ->copyMetadata(SrcInst: I);
175
176 Instruction *ThenTerm =
177 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
178 B.SetInsertPoint(ThenTerm);
179 }
180
181 // Now we can signal the cluster barrier from a single wave and wait for the
182 // barrier in all waves.
183 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
184 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
185 B.CreateIntrinsicWithoutFolding(
186 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
187 ->copyMetadata(SrcInst: I);
188
189 B.SetInsertPoint(&I);
190 B.CreateIntrinsicWithoutFolding(
191 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
192 ->copyMetadata(SrcInst: I);
193
194 I.eraseFromParent();
195 return true;
196 }
197
198 bool IsWorkgroupScope = false;
199
200 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
201 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
202 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
203 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
204 if (BarrierID == AMDGPU::Barrier::TRAP ||
205 BarrierID == AMDGPU::Barrier::WORKGROUP ||
206 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
207 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
208 IsWorkgroupScope = true;
209 else if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst &&
210 BarrierID == AMDGPU::Barrier::CLUSTER) {
211 I.getContext().diagnose(
212 DI: DiagnosticInfoUnsupported(*I.getFunction(),
213 "s_barrier_signal_isfirst does not support "
214 "user_cluster_barrier_id (-3)",
215 I.getDebugLoc()));
216 }
217 } else {
218 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
219 IsWorkgroupScope = true;
220 }
221
222 if (IsWorkgroupScope && IsSingleWaveWG) {
223 // Down-grade waits, remove split signals.
224 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
225 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
226 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
227 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
228 ->copyMetadata(SrcInst: I);
229 } else if (I.getIntrinsicID() ==
230 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
231 // If we're the only wave of the workgroup, we're always first.
232 I.replaceAllUsesWith(V: B.getInt1(V: true));
233 }
234 I.eraseFromParent();
235 return true;
236 }
237
238 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
239 ST.hasSplitBarriers()) {
240 // Lower to split barriers.
241 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
242 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
243 B.CreateIntrinsicWithoutFolding(
244 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
245 ->copyMetadata(SrcInst: I);
246 B.CreateIntrinsicWithoutFolding(
247 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
248 ->copyMetadata(SrcInst: I);
249 I.eraseFromParent();
250 return true;
251 }
252
253 return false;
254}
255
256bool AMDGPULowerIntrinsicsImpl::visitPtrSBufferLoad(IntrinsicInst &I) {
257 assert(I.getIntrinsicID() == Intrinsic::amdgcn_ptr_s_buffer_load);
258
259 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
260 return false;
261
262 I.setMetadata(KindID: LLVMContext::MD_invariant_load,
263 Node: MDNode::get(Context&: I.getContext(), MDs: {}));
264 return true;
265}
266
267bool AMDGPULowerIntrinsicsImpl::visitMonitorSleep(IntrinsicInst &I) {
268 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_monitor_sleep);
269
270 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
271 if (!ST.hasNoSleepForever())
272 return false;
273
274 int Sleep = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
275 if (!(Sleep & 0x8000))
276 return false;
277
278 IRBuilder<> B(&I);
279 Value *NewSleep = B.getInt16(C: 127); // Maximum
280 I.setArgOperand(i: 0, v: NewSleep);
281
282 return true;
283}
284
285bool AMDGPULowerIntrinsicsImpl::visitCvtScale(IntrinsicInst &I) {
286 int MaxSel = 0;
287 switch (I.getIntrinsicID()) {
288 default:
289 llvm_unreachable("expected cvt_scale_* intrinsic");
290 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
291 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
292 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
293 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
294 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
295 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
296 MaxSel = 8;
297 break;
298 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
299 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
300 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
301 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
302 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
303 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
304 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
305 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
306 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
307 MaxSel = 4;
308 break;
309 }
310
311 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
312 if (ST.hasBlock16ConversionScaleInsts())
313 MaxSel *= 2;
314
315 int ScaleSel = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getSExtValue();
316 if (ScaleSel < MaxSel)
317 return false;
318
319 I.getContext().diagnose(DI: DiagnosticInfoUnsupported(
320 *I.getFunction(),
321 I.getCalledFunction()->getName() +
322 Twine(" scale_sel maximum supported value is ") + Twine(MaxSel - 1),
323 I.getDebugLoc()));
324
325 return false;
326}
327
328bool AMDGPULowerIntrinsicsImpl::visitUnscaledWMMA(IntrinsicInst &I) {
329 assert(I.getIntrinsicID() == Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4);
330
331 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
332 if (!ST.hasGFX1250_STRICT())
333 return false;
334
335 IRBuilder<> B(&I);
336
337 // Convert unscaled WMMA into a scaled version with scale factors 0.
338 SmallVector<Value *, 14> Args(I.args());
339 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_a_scale
340 Args.push_back(
341 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_a_scale_fmt
342 Args.push_back(Elt: B.getInt32(C: 0)); // matrix a scale exponential
343 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_b_scale
344 Args.push_back(
345 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_b_scale_fmt
346 Args.push_back(Elt: B.getInt32(C: 0)); // matrix b scale exponential
347 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_a_reuse
348 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_b_reuse
349
350 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
351 RetTy: I.getType(), ID: Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4, Args);
352 I.replaceAllUsesWith(V: NewI);
353 NewI->copyMetadata(SrcInst: I);
354 NewI->takeName(V: &I);
355 I.eraseFromParent();
356
357 return true;
358}
359
360// A multicast load whose workgroup mask names all 16 workgroups of a cluster
361// may never return to some of them. Clamp the mask to 0x7fff: workgroups 0-14
362// still multicast to each other, and workgroup 15 does an ordinary load (mask
363// 0). It is still a single load per wave, and every workgroup still receives
364// the data. Only a 16-workgroup cluster has a workgroup 15, so in a smaller
365// cluster the clamped mask names the same workgroups.
366bool AMDGPULowerIntrinsicsImpl::visitMcastLoad(IntrinsicInst &I) {
367 // A cluster size fixed at compile time cannot change at run time.
368 AMDGPU::ClusterDimsAttr ClusterDims =
369 AMDGPU::ClusterDimsAttr::get(F: *I.getFunction());
370 if (ClusterDims.isNoCluster())
371 return false;
372 if (ClusterDims.isFixedDims()) {
373 const std::array<unsigned, 3> &Dims = ClusterDims.getDims();
374 if (Dims[0] * Dims[1] * Dims[2] < 16)
375 return false;
376 }
377
378 // A tensor load has the mask in bits [15:0] of dword 0 of D# group 1. A
379 // cluster load has it in bits [15:0] of its last operand, which goes to M0.
380 constexpr uint32_t FullMask = 0xffff;
381 bool IsTensor = I.getIntrinsicID() == Intrinsic::amdgcn_tensor_load_to_lds;
382 unsigned MaskIdx = IsTensor ? 1 : I.arg_size() - 1;
383 Value *MaskOp = I.getArgOperand(i: MaskIdx);
384 SimplifyQuery Q(M.getDataLayout(), &I);
385 KnownBits Known;
386 if (IsTensor) {
387 unsigned NumElts =
388 cast<FixedVectorType>(Val: MaskOp->getType())->getNumElements();
389 Known = computeKnownBits(V: MaskOp, DemandedElts: APInt::getOneBitSet(numBits: NumElts, BitNo: 0), Q);
390 } else {
391 Known = computeKnownBits(V: MaskOp, Q);
392 }
393 KnownBits KnownMask = Known.trunc(BitWidth: 16);
394 if (!KnownMask.getMaxValue().isAllOnes())
395 return false;
396 bool IsKnownFull = KnownMask.isAllOnes();
397
398 IRBuilder<> B(&I);
399 Value *Mask = MaskOp;
400 if (IsTensor) {
401 Mask = findScalarElement(V: MaskOp, EltNo: 0);
402 if (!Mask)
403 Mask = B.CreateExtractElement(Vec: MaskOp, Idx: uint64_t(0));
404 }
405
406 // Compute the clamp mask in the entry block so it is not recomputed in loops.
407 IRBuilder<> EntryB(M);
408 EntryB.SetInsertPointPastAllocas(I.getFunction());
409 Value *FlatID =
410 EntryB.CreateIntrinsic(ID: Intrinsic::amdgcn_cluster_workgroup_flat_id, Args: {});
411 Value *IsLast = EntryB.CreateICmpEQ(LHS: FlatID, RHS: EntryB.getInt32(C: 15));
412 Value *ClampMask = EntryB.CreateSelect(C: IsLast, True: EntryB.getInt32(C: 0xffff0000),
413 False: EntryB.getInt32(C: 0xffff7fff));
414 Value *NewMask = B.CreateAnd(LHS: ClampMask, RHS: Mask);
415 if (!IsKnownFull) {
416 Value *IsFull =
417 B.CreateICmpEQ(LHS: B.CreateAnd(LHS: Mask, RHS: FullMask), RHS: B.getInt32(C: FullMask));
418 NewMask = B.CreateSelect(C: IsFull, True: NewMask, False: Mask);
419 }
420
421 if (IsTensor)
422 NewMask = B.CreateInsertElement(Vec: MaskOp, NewElt: NewMask, Idx: uint64_t(0));
423 I.setArgOperand(i: MaskIdx, v: NewMask);
424 return true;
425}
426
427PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
428 ModuleAnalysisManager &MAM) {
429 AMDGPULowerIntrinsicsImpl Impl(M, TM);
430 if (!Impl.run())
431 return PreservedAnalyses::all();
432 return PreservedAnalyses::none();
433}
434
435bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
436 auto &TPC = getAnalysis<TargetPassConfig>();
437 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
438
439 AMDGPULowerIntrinsicsImpl Impl(M, TM);
440 return Impl.run();
441}
442
443#define PASS_DESC "AMDGPU lower intrinsics"
444INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
445 false)
446INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
447INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
448 false)
449
450char AMDGPULowerIntrinsicsLegacy::ID = 0;
451
452ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
453 return new AMDGPULowerIntrinsicsLegacy;
454}
455