1//===-- AMDGPULowerIntrinsics.cpp -------------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower intrinsics that would otherwise require separate handling in both
10// SelectionDAG and GlobalISel.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "AMDGPUTargetMachine.h"
16#include "GCNSubtarget.h"
17#include "llvm/IR/IRBuilder.h"
18#include "llvm/IR/IntrinsicInst.h"
19#include "llvm/IR/IntrinsicsAMDGPU.h"
20#include "llvm/InitializePasses.h"
21#include "llvm/Transforms/Utils/BasicBlockUtils.h"
22
23#define DEBUG_TYPE "amdgpu-lower-intrinsics"
24
25using namespace llvm;
26
27namespace {
28
29class AMDGPULowerIntrinsicsImpl {
30public:
31 Module &M;
32 const AMDGPUTargetMachine &TM;
33
34 AMDGPULowerIntrinsicsImpl(Module &M, const AMDGPUTargetMachine &TM)
35 : M(M), TM(TM) {}
36
37 bool run();
38
39private:
40 bool visitBarrier(IntrinsicInst &I);
41};
42
43class AMDGPULowerIntrinsicsLegacy : public ModulePass {
44public:
45 static char ID;
46
47 AMDGPULowerIntrinsicsLegacy() : ModulePass(ID) {}
48
49 bool runOnModule(Module &M) override;
50
51 void getAnalysisUsage(AnalysisUsage &AU) const override {
52 AU.addRequired<TargetPassConfig>();
53 }
54};
55
56template <class T> static void forEachCall(Function &Intrin, T Callback) {
57 for (User *U : make_early_inc_range(Range: Intrin.users())) {
58 if (auto *CI = dyn_cast<IntrinsicInst>(Val: U))
59 Callback(CI);
60 }
61}
62
63} // anonymous namespace
64
65bool AMDGPULowerIntrinsicsImpl::run() {
66 bool Changed = false;
67
68 for (Function &F : M) {
69 switch (F.getIntrinsicID()) {
70 default:
71 continue;
72 case Intrinsic::amdgcn_s_barrier:
73 case Intrinsic::amdgcn_s_barrier_signal:
74 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
75 case Intrinsic::amdgcn_s_barrier_wait:
76 case Intrinsic::amdgcn_s_cluster_barrier:
77 forEachCall(Intrin&: F, Callback: [&](IntrinsicInst *II) { Changed |= visitBarrier(I&: *II); });
78 break;
79 }
80 }
81
82 return Changed;
83}
84
85// Optimize barriers and lower s_(cluster_)barrier to a sequence of split
86// barrier intrinsics.
87bool AMDGPULowerIntrinsicsImpl::visitBarrier(IntrinsicInst &I) {
88 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
89 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
90 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst ||
91 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
92 I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier);
93
94 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F: *I.getFunction());
95 bool IsSingleWaveWG = false;
96
97 if (TM.getOptLevel() > CodeGenOptLevel::None) {
98 unsigned WGMaxSize = ST.getFlatWorkGroupSizes(F: *I.getFunction()).second;
99 IsSingleWaveWG = WGMaxSize <= ST.getWavefrontSize();
100 }
101
102 IRBuilder<> B(&I);
103
104 // Lower the s_cluster_barrier intrinsic first. There is no corresponding
105 // hardware instruction in any subtarget.
106 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_cluster_barrier) {
107 // The default cluster barrier expects one signal per workgroup. So we need
108 // a workgroup barrier first.
109 if (IsSingleWaveWG) {
110 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
111 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
112 ->copyMetadata(SrcInst: I);
113 } else {
114 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
115 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
116 CallInst *IsFirst = B.CreateIntrinsicWithoutFolding(
117 RetTy: B.getInt1Ty(), ID: Intrinsic::amdgcn_s_barrier_signal_isfirst,
118 Args: {BarrierID_32});
119 IsFirst->copyMetadata(SrcInst: I);
120 B.CreateIntrinsicWithoutFolding(
121 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
122 ->copyMetadata(SrcInst: I);
123
124 Instruction *ThenTerm =
125 SplitBlockAndInsertIfThen(Cond: IsFirst, SplitBefore: I.getIterator(), Unreachable: false);
126 B.SetInsertPoint(ThenTerm);
127 }
128
129 // Now we can signal the cluster barrier from a single wave and wait for the
130 // barrier in all waves.
131 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::CLUSTER);
132 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::CLUSTER);
133 B.CreateIntrinsicWithoutFolding(
134 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
135 ->copyMetadata(SrcInst: I);
136
137 B.SetInsertPoint(&I);
138 B.CreateIntrinsicWithoutFolding(
139 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
140 ->copyMetadata(SrcInst: I);
141
142 I.eraseFromParent();
143 return true;
144 }
145
146 bool IsWorkgroupScope = false;
147
148 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait ||
149 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal ||
150 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_signal_isfirst) {
151 int BarrierID = cast<ConstantInt>(Val: I.getArgOperand(i: 0))->getSExtValue();
152 if (BarrierID == AMDGPU::Barrier::TRAP ||
153 BarrierID == AMDGPU::Barrier::WORKGROUP ||
154 (BarrierID >= AMDGPU::Barrier::NAMED_BARRIER_FIRST &&
155 BarrierID <= AMDGPU::Barrier::NAMED_BARRIER_LAST))
156 IsWorkgroupScope = true;
157 } else {
158 assert(I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier);
159 IsWorkgroupScope = true;
160 }
161
162 if (IsWorkgroupScope && IsSingleWaveWG) {
163 // Down-grade waits, remove split signals.
164 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier ||
165 I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier_wait) {
166 B.CreateIntrinsicWithoutFolding(RetTy: B.getVoidTy(),
167 ID: Intrinsic::amdgcn_wave_barrier, Args: {})
168 ->copyMetadata(SrcInst: I);
169 } else if (I.getIntrinsicID() ==
170 Intrinsic::amdgcn_s_barrier_signal_isfirst) {
171 // If we're the only wave of the workgroup, we're always first.
172 I.replaceAllUsesWith(V: B.getInt1(V: true));
173 }
174 I.eraseFromParent();
175 return true;
176 }
177
178 if (I.getIntrinsicID() == Intrinsic::amdgcn_s_barrier &&
179 ST.hasSplitBarriers()) {
180 // Lower to split barriers.
181 Value *BarrierID_32 = B.getInt32(C: AMDGPU::Barrier::WORKGROUP);
182 Value *BarrierID_16 = B.getInt16(C: AMDGPU::Barrier::WORKGROUP);
183 B.CreateIntrinsicWithoutFolding(
184 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_signal, Args: {BarrierID_32})
185 ->copyMetadata(SrcInst: I);
186 B.CreateIntrinsicWithoutFolding(
187 RetTy: B.getVoidTy(), ID: Intrinsic::amdgcn_s_barrier_wait, Args: {BarrierID_16})
188 ->copyMetadata(SrcInst: I);
189 I.eraseFromParent();
190 return true;
191 }
192
193 return false;
194}
195
196PreservedAnalyses AMDGPULowerIntrinsicsPass::run(Module &M,
197 ModuleAnalysisManager &MAM) {
198 AMDGPULowerIntrinsicsImpl Impl(M, TM);
199 if (!Impl.run())
200 return PreservedAnalyses::all();
201 return PreservedAnalyses::none();
202}
203
204bool AMDGPULowerIntrinsicsLegacy::runOnModule(Module &M) {
205 auto &TPC = getAnalysis<TargetPassConfig>();
206 const AMDGPUTargetMachine &TM = TPC.getTM<AMDGPUTargetMachine>();
207
208 AMDGPULowerIntrinsicsImpl Impl(M, TM);
209 return Impl.run();
210}
211
212#define PASS_DESC "AMDGPU lower intrinsics"
213INITIALIZE_PASS_BEGIN(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
214 false)
215INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
216INITIALIZE_PASS_END(AMDGPULowerIntrinsicsLegacy, DEBUG_TYPE, PASS_DESC, false,
217 false)
218
219char AMDGPULowerIntrinsicsLegacy::ID = 0;
220
221ModulePass *llvm::createAMDGPULowerIntrinsicsLegacyPass() {
222 return new AMDGPULowerIntrinsicsLegacy;
223}
224