1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/IR/DiagnosticInfo.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/IntrinsicsAMDGPU.h"
21#include "llvm/InitializePasses.h"
22#include "llvm/Transforms/Utils/BasicBlockUtils.h"
23
24#define DEBUG_TYPE "amdgpu-lower-intrinsics"
25
26using namespace llvm;
27
28namespace {
29
30class AMDGPULowerIntrinsicsImpl {
31public:
32 Module &M;
33 const AMDGPUTargetMachine &TM;
34
35 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
36 : M(M), TM(TM) {}
37
38 bool run();
39
40private:
41 bool visitBarrier(IntrinsicInst &I);
42 bool visitPtrSBufferLoad(IntrinsicInst &I);
43 bool visitMonitorSleep(IntrinsicInst &I);
44 bool visitCvtScale(IntrinsicInst &I);
45 bool visitUnscaledWMMA(IntrinsicInst &I);
46};
47
48class AMDGPULowerIntrinsicsLegacy : public ModulePass {
49public:
50 static char ID;
51
52 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
53
54 bool runOnModule(Module &M) override;
55
56 void getAnalysisUsage(AnalysisUsage &AU) const override {
57 AU.addRequired<TargetPassConfig>();
58 }
59};
60
61template <class T> static void forEachCall(Function &Intrin, T Callback) {
62 for (User *U : make_early_inc_range(Range: Intrin.users())) {
63 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
64 Callback(CI);
65 }
66}
67
68} // anonymous namespace
69
70bool AMDGPULowerIntrinsicsImpl::run() {
71 bool Changed = false;
72
73 for (Function &F : M) {
74 switch (F.getIntrinsicID()) {
75 default:
76 continue;
77 case Intrinsic::amdgcn_s_barrier:
78 case Intrinsic::amdgcn_s_barrier_signal:
79 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
80 case Intrinsic::amdgcn_s_barrier_wait:
81 case Intrinsic::amdgcn_s_cluster_barrier:
82 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
83 break;
84 case Intrinsic::amdgcn_ptr_s_buffer_load:
85 forEachCall(
86 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitPtrSBufferLoad(I&: *II); });
87 break;
88 case Intrinsic::amdgcn_s_monitor_sleep:
89 forEachCall(
90 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitMonitorSleep(I&: *II); });
91 break;
92 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
93 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
94 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
95 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
96 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
97 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
98 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
99 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
100 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
101 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
102 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
103 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
104 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
105 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
106 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
107 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitCvtScale(I&: *II); });
108 break;
109 case Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4:
110 forEachCall(
111 Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitUnscaledWMMA(I&: *II); });
112 break;
113 }
114 }
115
116 return Changed;
117}
118
119// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
120// barrier intrinsics.
121bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
122 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
123 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
124 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
125 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
126 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
127
128 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
129 bool IsSingleWaveWG = false;
130
131 if (TM.getOptLevel() > CodeGenOptLevel::None)
132 IsSingleWaveWG = ST.isSingleWavefrontWorkgroup(F: *I.getFunction());
133
134 IRBuilder<> B(&I);
135
136 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
137 // hardware instruction in any subtarget.
138 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
139 // The default cluster barrier expects one signal per workgroup. So we need
140 // a workgroup barrier first.
141 if (IsSingleWaveWG) {
142 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
143 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
144 ->copyMetadata(SrcInst: I);
145 } else {
146 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
147 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
148 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
149 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
150 Args: {BarrierID_32});
151 IsFirst->copyMetadata(SrcInst: I);
152 B.CreateIntrinsicWithoutFolding(
153 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
154 ->copyMetadata(SrcInst: I);
155
156 Instruction *ThenTerm =
157 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
158 B.SetInsertPoint(ThenTerm);
159 }
160
161 // Now we can signal the cluster barrier from a single wave and wait for the
162 // barrier in all waves.
163 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
164 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
165 B.CreateIntrinsicWithoutFolding(
166 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
167 ->copyMetadata(SrcInst: I);
168
169 B.SetInsertPoint(&I);
170 B.CreateIntrinsicWithoutFolding(
171 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
172 ->copyMetadata(SrcInst: I);
173
174 I.eraseFromParent();
175 return true;
176 }
177
178 bool IsWorkgroupScope = false;
179
180 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
181 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
182 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
183 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
184 if (BarrierID == AMDGPU::Barrier::TRAP ||
185 BarrierID == AMDGPU::Barrier::WORKGROUP ||
186 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
187 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
188 IsWorkgroupScope = true;
189 else if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst &&
190 BarrierID == AMDGPU::Barrier::CLUSTER) {
191 I.getContext().diagnose(
192 DI: DiagnosticInfoUnsupported(*I.getFunction(),
193 "s_barrier_signal_isfirst does not support "
194 "user_cluster_barrier_id (-3)",
195 I.getDebugLoc()));
196 }
197 } else {
198 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
199 IsWorkgroupScope = true;
200 }
201
202 if (IsWorkgroupScope && IsSingleWaveWG) {
203 // Down-grade waits, remove split signals.
204 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
205 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
206 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
207 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
208 ->copyMetadata(SrcInst: I);
209 } else if (I.getIntrinsicID() ==
210 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
211 // If we're the only wave of the workgroup, we're always first.
212 I.replaceAllUsesWith(V: B.getInt1(V: true));
213 }
214 I.eraseFromParent();
215 return true;
216 }
217
218 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
219 ST.hasSplitBarriers()) {
220 // Lower to split barriers.
221 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
222 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
223 B.CreateIntrinsicWithoutFolding(
224 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
225 ->copyMetadata(SrcInst: I);
226 B.CreateIntrinsicWithoutFolding(
227 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
228 ->copyMetadata(SrcInst: I);
229 I.eraseFromParent();
230 return true;
231 }
232
233 return false;
234}
235
236bool AMDGPULowerIntrinsicsImpl::visitPtrSBufferLoad(IntrinsicInst &I) {
237 assert(I.getIntrinsicID() == Intrinsic::amdgcn_ptr_s_buffer_load);
238
239 if (I.hasMetadata(KindID: LLVMContext::MD_invariant_load))
240 return false;
241
242 I.setMetadata(KindID: LLVMContext::MD_invariant_load,
243 Node: MDNode::get(Context&: I.getContext(), MDs: {}));
244 return true;
245}
246
247bool AMDGPULowerIntrinsicsImpl::visitMonitorSleep(IntrinsicInst &I) {
248 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_monitor_sleep);
249
250 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
251 if (!ST.hasNoSleepForever())
252 return false;
253
254 int Sleep = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
255 if (!(Sleep & 0x8000))
256 return false;
257
258 IRBuilder<> B(&I);
259 Value *NewSleep = B.getInt16(C: 127); // Maximum
260 I.setArgOperand(i: 0, v: NewSleep);
261
262 return true;
263}
264
265bool AMDGPULowerIntrinsicsImpl::visitCvtScale(IntrinsicInst &I) {
266 int MaxSel = 0;
267 switch (I.getIntrinsicID()) {
268 default:
269 llvm_unreachable("expected cvt_scale_* intrinsic");
270 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp8:
271 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp8:
272 case Intrinsic::amdgcn_cvt_scale_pk8_f16_bf8:
273 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_bf8:
274 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp8:
275 case Intrinsic::amdgcn_cvt_scale_pk8_f32_bf8:
276 MaxSel = 8;
277 break;
278 case Intrinsic::amdgcn_cvt_scale_pk8_f16_fp4:
279 case Intrinsic::amdgcn_cvt_scale_pk8_bf16_fp4:
280 case Intrinsic::amdgcn_cvt_scale_pk8_f32_fp4:
281 case Intrinsic::amdgcn_cvt_scale_pk16_f16_bf6:
282 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_bf6:
283 case Intrinsic::amdgcn_cvt_scale_pk16_f16_fp6:
284 case Intrinsic::amdgcn_cvt_scale_pk16_bf16_fp6:
285 case Intrinsic::amdgcn_cvt_scale_pk16_f32_fp6:
286 case Intrinsic::amdgcn_cvt_scale_pk16_f32_bf6:
287 MaxSel = 4;
288 break;
289 }
290
291 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
292 if (ST.hasBlock16ConversionScaleInsts())
293 MaxSel *= 2;
294
295 int ScaleSel = cast<ConstantInt>(Val: I.getArgOperand(i: 2))->getSExtValue();
296 if (ScaleSel < MaxSel)
297 return false;
298
299 I.getContext().diagnose(DI: DiagnosticInfoUnsupported(
300 *I.getFunction(),
301 I.getCalledFunction()->getName() +
302 Twine(" scale_sel maximum supported value is ") + Twine(MaxSel - 1),
303 I.getDebugLoc()));
304
305 return false;
306}
307
308bool AMDGPULowerIntrinsicsImpl::visitUnscaledWMMA(IntrinsicInst &I) {
309 assert(I.getIntrinsicID() == Intrinsic::amdgcn_wmma_f32_16x16x128_f8f6f4);
310
311 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
312 if (!ST.hasGFX1250_STRICT())
313 return false;
314
315 IRBuilder<> B(&I);
316
317 // Convert unscaled WMMA into a scaled version with scale factors 0.
318 SmallVector<Value *, 14> Args(I.args());
319 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_a_scale
320 Args.push_back(
321 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_a_scale_fmt
322 Args.push_back(Elt: B.getInt32(C: 0)); // matrix a scale exponential
323 Args.push_back(Elt: B.getInt32(C: 0)); // matrix_b_scale
324 Args.push_back(
325 Elt: B.getInt32(C: AMDGPU::WMMA::MATRIX_SCALE_FMT_E8)); // matrix_b_scale_fmt
326 Args.push_back(Elt: B.getInt32(C: 0)); // matrix b scale exponential
327 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_a_reuse
328 Args.push_back(Elt: B.getInt1(V: 0)); // matrix_b_reuse
329
330 CallInst *NewI = B.CreateIntrinsicWithoutFolding(
331 RetTy: I.getType(), ID: Intrinsic::amdgcn_wmma_scale_f32_16x16x128_f8f6f4, Args);
332 I.replaceAllUsesWith(V: NewI);
333 NewI->copyMetadata(SrcInst: I);
334 NewI->takeName(V: &I);
335 I.eraseFromParent();
336
337 return true;
338}
339
340PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
341 ModuleAnalysisManager &MAM) {
342 AMDGPULowerIntrinsicsImpl Impl(M, TM);
343 if (!Impl.run())
344 return PreservedAnalyses::all();
345 return PreservedAnalyses::none();
346}
347
348bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
349 auto &TPC = getAnalysis<TargetPassConfig>();
350 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
351
352 AMDGPULowerIntrinsicsImpl Impl(M, TM);
353 return Impl.run();
354}
355
356#define PASS_DESC "AMDGPU lower intrinsics"
357INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
358 false)
359INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
360INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
361 false)
362
363char AMDGPULowerIntrinsicsLegacy::ID = 0;
364
365ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
366 return new AMDGPULowerIntrinsicsLegacy;
367}
368