1//===-- AMDGPUAtomicOptimizer.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass optimizes atomic operations by using a single lane of a wavefront
11/// to perform the atomic operation, thus reducing contention on that memory
12/// location.
13/// Atomic optimizer uses following strategies to compute scan and reduced
14/// values
15/// 1. DPP -
16/// This is the most efficient implementation for scan. DPP uses Whole Wave
17/// Mode (WWM)
18/// 2. Iterative -
19// An alternative implementation iterates over all active lanes
20/// of Wavefront using llvm.cttz and performs scan using readlane & writelane
21/// intrinsics
22//===----------------------------------------------------------------------===//
23
24#include "AMDGPU.h"
25#include "GCNSubtarget.h"
26#include "llvm/Analysis/DomTreeUpdater.h"
27#include "llvm/Analysis/UniformityAnalysis.h"
28#include "llvm/CodeGen/TargetPassConfig.h"
29#include "llvm/IR/IRBuilder.h"
30#include "llvm/IR/InstVisitor.h"
31#include "llvm/IR/IntrinsicsAMDGPU.h"
32#include "llvm/InitializePasses.h"
33#include "llvm/Target/TargetMachine.h"
34#include "llvm/Transforms/Utils/BasicBlockUtils.h"
35
36#define DEBUG_TYPE "amdgpu-atomic-optimizer"
37
38using namespace llvm;
39using namespace llvm::AMDGPU;
40
41namespace {
42
43struct ReplacementInfo {
44 Instruction *I;
45 AtomicRMWInst::BinOp Op;
46 unsigned ValIdx;
47 bool ValDivergent;
48 bool IsLDS;
49};
50
51class AMDGPUAtomicOptimizer : public FunctionPass {
52public:
53 static char ID;
54 ScanOptions ScanImpl;
55 AMDGPUAtomicOptimizer(ScanOptions ScanImpl)
56 : FunctionPass(ID), ScanImpl(ScanImpl) {}
57
58 bool runOnFunction(Function &F) override;
59
60 void getAnalysisUsage(AnalysisUsage &AU) const override {
61 AU.addPreserved<DominatorTreeWrapperPass>();
62 AU.addRequired<UniformityInfoWrapperPass>();
63 AU.addRequired<TargetPassConfig>();
64 }
65};
66
67class AMDGPUAtomicOptimizerImpl
68 : public InstVisitor<AMDGPUAtomicOptimizerImpl> {
69private:
70 Function &F;
71 SmallVector<ReplacementInfo, 8> ToReplace;
72 const UniformityInfo &UA;
73 const DataLayout &DL;
74 DomTreeUpdater &DTU;
75 const GCNSubtarget &ST;
76 bool IsPixelShader;
77 ScanOptions ScanImpl;
78
79 Value *buildReduction(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
80 Value *const Identity) const;
81 Value *buildScan(IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *V,
82 Value *const Identity) const;
83 Value *buildShiftRight(IRBuilder<> &B, Value *V, Value *const Identity) const;
84
85 std::pair<Value *, Value *>
86 buildScanIteratively(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
87 Value *const Identity, Value *V, Instruction &I,
88 BasicBlock *ComputeLoop, BasicBlock *ComputeEnd) const;
89
90 void optimizeAtomic(Instruction &I, AtomicRMWInst::BinOp Op, unsigned ValIdx,
91 bool ValDivergent, bool IsLDS) const;
92
93public:
94 AMDGPUAtomicOptimizerImpl() = delete;
95
96 AMDGPUAtomicOptimizerImpl(Function &F, const UniformityInfo &UA,
97 DomTreeUpdater &DTU, const GCNSubtarget &ST,
98 ScanOptions ScanImpl)
99 : F(F), UA(UA), DL(F.getDataLayout()), DTU(DTU), ST(ST),
100 IsPixelShader(F.getCallingConv() == CallingConv::AMDGPU_PS),
101 ScanImpl(ScanImpl) {}
102
103 bool run();
104
105 void visitAtomicRMWInst(AtomicRMWInst &I);
106 void visitIntrinsicInst(IntrinsicInst &I);
107};
108
109} // namespace
110
111char AMDGPUAtomicOptimizer::ID = 0;
112
113char &llvm::AMDGPUAtomicOptimizerID = AMDGPUAtomicOptimizer::ID;
114
115bool AMDGPUAtomicOptimizer::runOnFunction(Function &F) {
116 if (skipFunction(F)) {
117 return false;
118 }
119
120 const UniformityInfo &UA =
121 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
122
123 DominatorTreeWrapperPass *DTW =
124 getAnalysisIfAvailable<DominatorTreeWrapperPass>();
125 DomTreeUpdater DTU(DTW ? &DTW->getDomTree() : nullptr,
126 DomTreeUpdater::UpdateStrategy::Lazy);
127
128 const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
129 const TargetMachine &TM = TPC.getTM<TargetMachine>();
130 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
131
132 return AMDGPUAtomicOptimizerImpl(F, UA, DTU, ST, ScanImpl).run();
133}
134
135PreservedAnalyses AMDGPUAtomicOptimizerPass::run(Function &F,
136 FunctionAnalysisManager &AM) {
137 const auto &UA = AM.getResult<UniformityInfoAnalysis>(IR&: F);
138
139 DomTreeUpdater DTU(&AM.getResult<DominatorTreeAnalysis>(IR&: F),
140 DomTreeUpdater::UpdateStrategy::Lazy);
141 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
142
143 bool IsChanged = AMDGPUAtomicOptimizerImpl(F, UA, DTU, ST, ScanImpl).run();
144
145 if (!IsChanged) {
146 return PreservedAnalyses::all();
147 }
148
149 PreservedAnalyses PA;
150 PA.preserve<DominatorTreeAnalysis>();
151 return PA;
152}
153
154bool AMDGPUAtomicOptimizerImpl::run() {
155 // Scan option None disables the Pass
156 if (ScanImpl == ScanOptions::None)
157 return false;
158 if (ST.isSingleLaneExecution(Kernel: F))
159 return false;
160
161 visit(F);
162 if (ToReplace.empty())
163 return false;
164
165 for (auto &[I, Op, ValIdx, ValDivergent, IsLDS] : ToReplace)
166 optimizeAtomic(I&: *I, Op, ValIdx, ValDivergent, IsLDS);
167 ToReplace.clear();
168 return true;
169}
170
171static bool isLegalCrossLaneType(Type *Ty) {
172 switch (Ty->getTypeID()) {
173 case Type::FloatTyID:
174 case Type::DoubleTyID:
175 return true;
176 case Type::IntegerTyID: {
177 unsigned Size = Ty->getIntegerBitWidth();
178 return (Size == 32 || Size == 64);
179 }
180 default:
181 return false;
182 }
183}
184
185void AMDGPUAtomicOptimizerImpl::visitAtomicRMWInst(AtomicRMWInst &I) {
186 if (I.getType()->isVectorTy() || I.isVolatile())
187 return;
188
189 // Early exit for unhandled address space atomic instructions.
190 switch (I.getPointerAddressSpace()) {
191 default:
192 return;
193 case AMDGPUAS::GLOBAL_ADDRESS:
194 case AMDGPUAS::LOCAL_ADDRESS:
195 break;
196 }
197
198 AtomicRMWInst::BinOp Op = I.getOperation();
199
200 switch (Op) {
201 default:
202 return;
203 case AtomicRMWInst::Add:
204 case AtomicRMWInst::Sub:
205 case AtomicRMWInst::And:
206 case AtomicRMWInst::Or:
207 case AtomicRMWInst::Xor:
208 case AtomicRMWInst::Max:
209 case AtomicRMWInst::Min:
210 case AtomicRMWInst::UMax:
211 case AtomicRMWInst::UMin:
212 case AtomicRMWInst::FAdd:
213 case AtomicRMWInst::FSub:
214 case AtomicRMWInst::FMax:
215 case AtomicRMWInst::FMin:
216 break;
217 }
218
219 // Only 32 and 64 bit floating point atomic ops are supported.
220 if (AtomicRMWInst::isFPOperation(Op) &&
221 !(I.getType()->isFloatTy() || I.getType()->isDoubleTy())) {
222 return;
223 }
224
225 const unsigned PtrIdx = 0;
226 const unsigned ValIdx = 1;
227
228 // If the pointer operand is divergent, then each lane is doing an atomic
229 // operation on a different address, and we cannot optimize that.
230 if (UA.isDivergentAtUse(U: I.getOperandUse(i: PtrIdx))) {
231 return;
232 }
233
234 bool ValDivergent = UA.isDivergentAtUse(U: I.getOperandUse(i: ValIdx));
235
236 // If the value operand is divergent, each lane is contributing a different
237 // value to the atomic calculation. We can only optimize divergent values if
238 // we have DPP available on our subtarget (for DPP strategy), and the atomic
239 // operation is 32 or 64 bits.
240 if (ValDivergent) {
241 if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
242 return;
243
244 if (!isLegalCrossLaneType(Ty: I.getType()))
245 return;
246 }
247
248 const bool IsLDS = I.getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
249
250 // The iterative scan runs once per active lane and costs more than the
251 // hardware serialization of a native LDS atomic.
252 if (IsLDS && ValDivergent && ScanImpl == ScanOptions::Iterative &&
253 ST.getTargetLowering()->shouldExpandAtomicRMWInIR(&I) ==
254 TargetLowering::AtomicExpansionKind::None)
255 return;
256
257 // If we get here, we can optimize the atomic using a single wavefront-wide
258 // atomic operation to do the calculation for the entire wavefront, so
259 // remember the instruction so we can come back to it.
260 ToReplace.push_back(Elt: {.I: &I, .Op: Op, .ValIdx: ValIdx, .ValDivergent: ValDivergent, .IsLDS: IsLDS});
261}
262
263void AMDGPUAtomicOptimizerImpl::visitIntrinsicInst(IntrinsicInst &I) {
264 if (I.getType()->isVectorTy())
265 return;
266
267 AtomicRMWInst::BinOp Op;
268
269 switch (I.getIntrinsicID()) {
270 default:
271 return;
272 case Intrinsic::amdgcn_struct_buffer_atomic_add:
273 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_add:
274 case Intrinsic::amdgcn_raw_buffer_atomic_add:
275 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_add:
276 Op = AtomicRMWInst::Add;
277 break;
278 case Intrinsic::amdgcn_struct_buffer_atomic_sub:
279 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_sub:
280 case Intrinsic::amdgcn_raw_buffer_atomic_sub:
281 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_sub:
282 Op = AtomicRMWInst::Sub;
283 break;
284 case Intrinsic::amdgcn_struct_buffer_atomic_and:
285 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_and:
286 case Intrinsic::amdgcn_raw_buffer_atomic_and:
287 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_and:
288 Op = AtomicRMWInst::And;
289 break;
290 case Intrinsic::amdgcn_struct_buffer_atomic_or:
291 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_or:
292 case Intrinsic::amdgcn_raw_buffer_atomic_or:
293 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_or:
294 Op = AtomicRMWInst::Or;
295 break;
296 case Intrinsic::amdgcn_struct_buffer_atomic_xor:
297 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_xor:
298 case Intrinsic::amdgcn_raw_buffer_atomic_xor:
299 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_xor:
300 Op = AtomicRMWInst::Xor;
301 break;
302 case Intrinsic::amdgcn_struct_buffer_atomic_smin:
303 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smin:
304 case Intrinsic::amdgcn_raw_buffer_atomic_smin:
305 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smin:
306 Op = AtomicRMWInst::Min;
307 break;
308 case Intrinsic::amdgcn_struct_buffer_atomic_umin:
309 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umin:
310 case Intrinsic::amdgcn_raw_buffer_atomic_umin:
311 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umin:
312 Op = AtomicRMWInst::UMin;
313 break;
314 case Intrinsic::amdgcn_struct_buffer_atomic_smax:
315 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_smax:
316 case Intrinsic::amdgcn_raw_buffer_atomic_smax:
317 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_smax:
318 Op = AtomicRMWInst::Max;
319 break;
320 case Intrinsic::amdgcn_struct_buffer_atomic_umax:
321 case Intrinsic::amdgcn_struct_ptr_buffer_atomic_umax:
322 case Intrinsic::amdgcn_raw_buffer_atomic_umax:
323 case Intrinsic::amdgcn_raw_ptr_buffer_atomic_umax:
324 Op = AtomicRMWInst::UMax;
325 break;
326 }
327
328 auto *Aux = cast<ConstantInt>(Val: I.getArgOperand(i: I.arg_size() - 1));
329 if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
330 return;
331
332 const unsigned ValIdx = 0;
333
334 const bool ValDivergent = UA.isDivergentAtUse(U: I.getOperandUse(i: ValIdx));
335
336 // If the value operand is divergent, each lane is contributing a different
337 // value to the atomic calculation. We can only optimize divergent values if
338 // we have DPP available on our subtarget (for DPP strategy), and the atomic
339 // operation is 32 or 64 bits.
340 if (ValDivergent) {
341 if (ScanImpl == ScanOptions::DPP && !ST.hasDPP())
342 return;
343
344 if (!isLegalCrossLaneType(Ty: I.getType()))
345 return;
346 }
347
348 // If any of the other arguments to the intrinsic are divergent, we can't
349 // optimize the operation.
350 for (unsigned Idx = 1; Idx < I.getNumOperands(); Idx++) {
351 if (UA.isDivergentAtUse(U: I.getOperandUse(i: Idx)))
352 return;
353 }
354
355 // If we get here, we can optimize the atomic using a single wavefront-wide
356 // atomic operation to do the calculation for the entire wavefront, so
357 // remember the instruction so we can come back to it.
358 // Buffer atomics are never LDS.
359 ToReplace.push_back(Elt: {.I: &I, .Op: Op, .ValIdx: ValIdx, .ValDivergent: ValDivergent, /*IsLDS=*/false});
360}
361
362// Use the builder to create the non-atomic counterpart of the specified
363// atomicrmw binary op.
364static Value *buildNonAtomicBinOp(IRBuilder<> &B, AtomicRMWInst::BinOp Op,
365 Value *LHS, Value *RHS) {
366 CmpInst::Predicate Pred;
367
368 switch (Op) {
369 default:
370 llvm_unreachable("Unhandled atomic op");
371 case AtomicRMWInst::Add:
372 return B.CreateBinOp(Opc: Instruction::Add, LHS, RHS);
373 case AtomicRMWInst::FAdd:
374 return B.CreateFAdd(L: LHS, R: RHS);
375 case AtomicRMWInst::Sub:
376 return B.CreateBinOp(Opc: Instruction::Sub, LHS, RHS);
377 case AtomicRMWInst::FSub:
378 return B.CreateFSub(L: LHS, R: RHS);
379 case AtomicRMWInst::And:
380 return B.CreateBinOp(Opc: Instruction::And, LHS, RHS);
381 case AtomicRMWInst::Or:
382 return B.CreateBinOp(Opc: Instruction::Or, LHS, RHS);
383 case AtomicRMWInst::Xor:
384 return B.CreateBinOp(Opc: Instruction::Xor, LHS, RHS);
385
386 case AtomicRMWInst::Max:
387 Pred = CmpInst::ICMP_SGT;
388 break;
389 case AtomicRMWInst::Min:
390 Pred = CmpInst::ICMP_SLT;
391 break;
392 case AtomicRMWInst::UMax:
393 Pred = CmpInst::ICMP_UGT;
394 break;
395 case AtomicRMWInst::UMin:
396 Pred = CmpInst::ICMP_ULT;
397 break;
398 case AtomicRMWInst::FMax:
399 return B.CreateMaxNum(LHS, RHS);
400 case AtomicRMWInst::FMin:
401 return B.CreateMinNum(LHS, RHS);
402 }
403 Value *Cond = B.CreateICmp(P: Pred, LHS, RHS);
404 return B.CreateSelect(C: Cond, True: LHS, False: RHS);
405}
406
407// Use the builder to create a reduction of V across the wavefront, with all
408// lanes active, returning the same result in all lanes.
409Value *AMDGPUAtomicOptimizerImpl::buildReduction(IRBuilder<> &B,
410 AtomicRMWInst::BinOp Op,
411 Value *V,
412 Value *const Identity) const {
413 Type *AtomicTy = V->getType();
414 Module *M = B.getModule();
415
416 // Reduce within each row of 16 lanes.
417 for (unsigned Idx = 0; Idx < 4; Idx++) {
418 V = buildNonAtomicBinOp(
419 B, Op, LHS: V,
420 RHS: B.CreateIntrinsic(ID: Intrinsic::amdgcn_update_dpp, OverloadTypes: AtomicTy,
421 Args: {Identity, V, B.getInt32(C: DPP::ROW_XMASK0 | 1 << Idx),
422 B.getInt32(C: 0xf), B.getInt32(C: 0xf), B.getFalse()}));
423 }
424
425 // Reduce within each pair of rows (i.e. 32 lanes).
426 assert(ST.hasPermlane16Insts());
427 Value *Permlanex16Call =
428 B.CreateIntrinsic(RetTy: AtomicTy, ID: Intrinsic::amdgcn_permlanex16,
429 Args: {PoisonValue::get(T: AtomicTy), V, B.getInt32(C: 0),
430 B.getInt32(C: 0), B.getFalse(), B.getFalse()});
431 V = buildNonAtomicBinOp(B, Op, LHS: V, RHS: Permlanex16Call);
432 if (ST.isWave32()) {
433 return V;
434 }
435
436 if (ST.hasPermLane64()) {
437 // Reduce across the upper and lower 32 lanes.
438 Value *Permlane64Call =
439 B.CreateIntrinsic(RetTy: AtomicTy, ID: Intrinsic::amdgcn_permlane64, Args: V);
440 return buildNonAtomicBinOp(B, Op, LHS: V, RHS: Permlane64Call);
441 }
442
443 // Pick an arbitrary lane from 0..31 and an arbitrary lane from 32..63 and
444 // combine them with a scalar operation.
445 Function *ReadLane = Intrinsic::getOrInsertDeclaration(
446 M, id: Intrinsic::amdgcn_readlane, OverloadTys: AtomicTy);
447 Value *Lane0 = B.CreateCall(Callee: ReadLane, Args: {V, B.getInt32(C: 0)});
448 Value *Lane32 = B.CreateCall(Callee: ReadLane, Args: {V, B.getInt32(C: 32)});
449 return buildNonAtomicBinOp(B, Op, LHS: Lane0, RHS: Lane32);
450}
451
452// Use the builder to create an inclusive scan of V across the wavefront, with
453// all lanes active.
454Value *AMDGPUAtomicOptimizerImpl::buildScan(IRBuilder<> &B,
455 AtomicRMWInst::BinOp Op, Value *V,
456 Value *Identity) const {
457 Type *AtomicTy = V->getType();
458 Module *M = B.getModule();
459 Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
460 M, id: Intrinsic::amdgcn_update_dpp, OverloadTys: AtomicTy);
461
462 for (unsigned Idx = 0; Idx < 4; Idx++) {
463 V = buildNonAtomicBinOp(
464 B, Op, LHS: V,
465 RHS: B.CreateCall(Callee: UpdateDPP,
466 Args: {Identity, V, B.getInt32(C: DPP::ROW_SHR0 | 1 << Idx),
467 B.getInt32(C: 0xf), B.getInt32(C: 0xf), B.getFalse()}));
468 }
469 if (ST.hasDPPBroadcasts()) {
470 // GFX9 has DPP row broadcast operations.
471 V = buildNonAtomicBinOp(
472 B, Op, LHS: V,
473 RHS: B.CreateCall(Callee: UpdateDPP,
474 Args: {Identity, V, B.getInt32(C: DPP::BCAST15), B.getInt32(C: 0xa),
475 B.getInt32(C: 0xf), B.getFalse()}));
476 V = buildNonAtomicBinOp(
477 B, Op, LHS: V,
478 RHS: B.CreateCall(Callee: UpdateDPP,
479 Args: {Identity, V, B.getInt32(C: DPP::BCAST31), B.getInt32(C: 0xc),
480 B.getInt32(C: 0xf), B.getFalse()}));
481 } else {
482 // On GFX10 all DPP operations are confined to a single row. To get cross-
483 // row operations we have to use permlane or readlane.
484
485 // Combine lane 15 into lanes 16..31 (and, for wave 64, lane 47 into lanes
486 // 48..63).
487 assert(ST.hasPermlane16Insts());
488 Value *PermX =
489 B.CreateIntrinsic(RetTy: AtomicTy, ID: Intrinsic::amdgcn_permlanex16,
490 Args: {PoisonValue::get(T: AtomicTy), V, B.getInt32(C: -1),
491 B.getInt32(C: -1), B.getFalse(), B.getFalse()});
492
493 Value *UpdateDPPCall = B.CreateCall(
494 Callee: UpdateDPP, Args: {Identity, PermX, B.getInt32(C: DPP::QUAD_PERM_ID),
495 B.getInt32(C: 0xa), B.getInt32(C: 0xf), B.getFalse()});
496 V = buildNonAtomicBinOp(B, Op, LHS: V, RHS: UpdateDPPCall);
497
498 if (!ST.isWave32()) {
499 // Combine lane 31 into lanes 32..63.
500 Value *const Lane31 = B.CreateIntrinsic(
501 RetTy: AtomicTy, ID: Intrinsic::amdgcn_readlane, Args: {V, B.getInt32(C: 31)});
502
503 Value *UpdateDPPCall = B.CreateCall(
504 Callee: UpdateDPP, Args: {Identity, Lane31, B.getInt32(C: DPP::QUAD_PERM_ID),
505 B.getInt32(C: 0xc), B.getInt32(C: 0xf), B.getFalse()});
506
507 V = buildNonAtomicBinOp(B, Op, LHS: V, RHS: UpdateDPPCall);
508 }
509 }
510 return V;
511}
512
513// Use the builder to create a shift right of V across the wavefront, with all
514// lanes active, to turn an inclusive scan into an exclusive scan.
515Value *AMDGPUAtomicOptimizerImpl::buildShiftRight(IRBuilder<> &B, Value *V,
516 Value *Identity) const {
517 Type *AtomicTy = V->getType();
518 Module *M = B.getModule();
519 Function *UpdateDPP = Intrinsic::getOrInsertDeclaration(
520 M, id: Intrinsic::amdgcn_update_dpp, OverloadTys: AtomicTy);
521 if (ST.hasDPPWavefrontShifts()) {
522 // GFX9 has DPP wavefront shift operations.
523 V = B.CreateCall(Callee: UpdateDPP,
524 Args: {Identity, V, B.getInt32(C: DPP::WAVE_SHR1), B.getInt32(C: 0xf),
525 B.getInt32(C: 0xf), B.getFalse()});
526 } else {
527 Function *ReadLane = Intrinsic::getOrInsertDeclaration(
528 M, id: Intrinsic::amdgcn_readlane, OverloadTys: AtomicTy);
529 Function *WriteLane = Intrinsic::getOrInsertDeclaration(
530 M, id: Intrinsic::amdgcn_writelane, OverloadTys: AtomicTy);
531
532 // On GFX10 all DPP operations are confined to a single row. To get cross-
533 // row operations we have to use permlane or readlane.
534 Value *Old = V;
535 V = B.CreateCall(Callee: UpdateDPP,
536 Args: {Identity, V, B.getInt32(C: DPP::ROW_SHR0 + 1),
537 B.getInt32(C: 0xf), B.getInt32(C: 0xf), B.getFalse()});
538
539 // Copy the old lane 15 to the new lane 16.
540 V = B.CreateCall(Callee: WriteLane, Args: {B.CreateCall(Callee: ReadLane, Args: {Old, B.getInt32(C: 15)}),
541 B.getInt32(C: 16), V});
542
543 if (!ST.isWave32()) {
544 // Copy the old lane 31 to the new lane 32.
545 V = B.CreateCall(
546 Callee: WriteLane,
547 Args: {B.CreateCall(Callee: ReadLane, Args: {Old, B.getInt32(C: 31)}), B.getInt32(C: 32), V});
548
549 // Copy the old lane 47 to the new lane 48.
550 V = B.CreateCall(
551 Callee: WriteLane,
552 Args: {B.CreateCall(Callee: ReadLane, Args: {Old, B.getInt32(C: 47)}), B.getInt32(C: 48), V});
553 }
554 }
555
556 return V;
557}
558
559// Use the builder to create an exclusive scan and compute the final reduced
560// value using an iterative approach. This provides an alternative
561// implementation to DPP which uses WMM for scan computations. This API iterate
562// over active lanes to read, compute and update the value using
563// readlane and writelane intrinsics.
564std::pair<Value *, Value *> AMDGPUAtomicOptimizerImpl::buildScanIteratively(
565 IRBuilder<> &B, AtomicRMWInst::BinOp Op, Value *const Identity, Value *V,
566 Instruction &I, BasicBlock *ComputeLoop, BasicBlock *ComputeEnd) const {
567 auto *Ty = I.getType();
568 auto *WaveTy = B.getIntNTy(N: ST.getWavefrontSize());
569 auto *EntryBB = I.getParent();
570 auto NeedResult = !I.use_empty();
571
572 auto *Ballot =
573 B.CreateIntrinsic(ID: Intrinsic::amdgcn_ballot, OverloadTypes: WaveTy, Args: B.getTrue());
574
575 // Start inserting instructions for ComputeLoop block
576 B.SetInsertPoint(ComputeLoop);
577 // Phi nodes for Accumulator, Scan results destination, and Active Lanes
578 auto *Accumulator = B.CreatePHI(Ty, NumReservedValues: 2, Name: "Accumulator");
579 Accumulator->addIncoming(V: Identity, BB: EntryBB);
580 PHINode *OldValuePhi = nullptr;
581 if (NeedResult) {
582 OldValuePhi = B.CreatePHI(Ty, NumReservedValues: 2, Name: "OldValuePhi");
583 OldValuePhi->addIncoming(V: PoisonValue::get(T: Ty), BB: EntryBB);
584 }
585 auto *ActiveBits = B.CreatePHI(Ty: WaveTy, NumReservedValues: 2, Name: "ActiveBits");
586 ActiveBits->addIncoming(V: Ballot, BB: EntryBB);
587
588 // Use llvm.cttz intrinsic to find the lowest remaining active lane.
589 auto *FF1 =
590 B.CreateIntrinsic(ID: Intrinsic::cttz, OverloadTypes: WaveTy, Args: {ActiveBits, B.getTrue()});
591
592 auto *LaneIdxInt = B.CreateTrunc(V: FF1, DestTy: B.getInt32Ty());
593
594 // Get the value required for atomic operation
595 Value *LaneValue = B.CreateIntrinsic(RetTy: V->getType(), ID: Intrinsic::amdgcn_readlane,
596 Args: {V, LaneIdxInt});
597
598 // Perform writelane if intermediate scan results are required later in the
599 // kernel computations
600 Value *OldValue = nullptr;
601 if (NeedResult) {
602 OldValue = B.CreateIntrinsic(RetTy: V->getType(), ID: Intrinsic::amdgcn_writelane,
603 Args: {Accumulator, LaneIdxInt, OldValuePhi});
604 OldValuePhi->addIncoming(V: OldValue, BB: ComputeLoop);
605 }
606
607 // Accumulate the results
608 auto *NewAccumulator = buildNonAtomicBinOp(B, Op, LHS: Accumulator, RHS: LaneValue);
609 Accumulator->addIncoming(V: NewAccumulator, BB: ComputeLoop);
610
611 // Set bit to zero of current active lane so that for next iteration llvm.cttz
612 // return the next active lane
613 auto *Mask = B.CreateShl(LHS: ConstantInt::get(Ty: WaveTy, V: 1), RHS: FF1);
614
615 auto *InverseMask = B.CreateXor(LHS: Mask, RHS: ConstantInt::getAllOnesValue(Ty: WaveTy));
616 auto *NewActiveBits = B.CreateAnd(LHS: ActiveBits, RHS: InverseMask);
617 ActiveBits->addIncoming(V: NewActiveBits, BB: ComputeLoop);
618
619 // Branch out of the loop when all lanes are processed.
620 auto *IsEnd = B.CreateICmpEQ(LHS: NewActiveBits, RHS: ConstantInt::get(Ty: WaveTy, V: 0));
621 B.CreateCondBr(Cond: IsEnd, True: ComputeEnd, False: ComputeLoop);
622
623 B.SetInsertPoint(ComputeEnd);
624
625 return {OldValue, NewAccumulator};
626}
627
628static Constant *getIdentityValueForAtomicOp(Type *const Ty,
629 AtomicRMWInst::BinOp Op) {
630 LLVMContext &C = Ty->getContext();
631 const unsigned BitWidth = Ty->getPrimitiveSizeInBits();
632 switch (Op) {
633 default:
634 llvm_unreachable("Unhandled atomic op");
635 case AtomicRMWInst::Add:
636 case AtomicRMWInst::Sub:
637 case AtomicRMWInst::Or:
638 case AtomicRMWInst::Xor:
639 case AtomicRMWInst::UMax:
640 return ConstantInt::get(Context&: C, V: APInt::getMinValue(numBits: BitWidth));
641 case AtomicRMWInst::And:
642 case AtomicRMWInst::UMin:
643 return ConstantInt::get(Context&: C, V: APInt::getMaxValue(numBits: BitWidth));
644 case AtomicRMWInst::Max:
645 return ConstantInt::get(Context&: C, V: APInt::getSignedMinValue(numBits: BitWidth));
646 case AtomicRMWInst::Min:
647 return ConstantInt::get(Context&: C, V: APInt::getSignedMaxValue(numBits: BitWidth));
648 case AtomicRMWInst::FAdd:
649 return ConstantFP::get(Context&: C, V: APFloat::getZero(Sem: Ty->getFltSemantics(), Negative: true));
650 case AtomicRMWInst::FSub:
651 return ConstantFP::get(Context&: C, V: APFloat::getZero(Sem: Ty->getFltSemantics(), Negative: false));
652 case AtomicRMWInst::FMin:
653 case AtomicRMWInst::FMax:
654 // FIXME: atomicrmw fmax/fmin behave like llvm.maxnum/minnum so NaN is the
655 // closest thing they have to an identity, but it still does not preserve
656 // the difference between quiet and signaling NaNs or NaNs with different
657 // payloads.
658 return ConstantFP::get(Context&: C, V: APFloat::getNaN(Sem: Ty->getFltSemantics()));
659 }
660}
661
662static Intrinsic::ID getWaveReductionIntrinsic(AtomicRMWInst::BinOp Op) {
663 switch (Op) {
664 default:
665 llvm_unreachable(
666 "Atomic Op yet to be ported to use Wave Reduction intrinsics.");
667 case AtomicRMWInst::Add:
668 case AtomicRMWInst::Sub:
669 return Intrinsic::amdgcn_wave_reduce_add;
670 case AtomicRMWInst::FAdd:
671 case AtomicRMWInst::FSub:
672 return Intrinsic::amdgcn_wave_reduce_fadd;
673 case AtomicRMWInst::And:
674 return Intrinsic::amdgcn_wave_reduce_and;
675 case AtomicRMWInst::Or:
676 return Intrinsic::amdgcn_wave_reduce_or;
677 case AtomicRMWInst::Xor:
678 return Intrinsic::amdgcn_wave_reduce_xor;
679 case AtomicRMWInst::UMax:
680 return Intrinsic::amdgcn_wave_reduce_umax;
681 case AtomicRMWInst::Max:
682 return Intrinsic::amdgcn_wave_reduce_max;
683 case AtomicRMWInst::FMax:
684 return Intrinsic::amdgcn_wave_reduce_fmax;
685 case AtomicRMWInst::UMin:
686 return Intrinsic::amdgcn_wave_reduce_umin;
687 case AtomicRMWInst::Min:
688 return Intrinsic::amdgcn_wave_reduce_min;
689 case AtomicRMWInst::FMin:
690 return Intrinsic::amdgcn_wave_reduce_fmin;
691 }
692}
693
694static Value *buildMul(IRBuilder<> &B, Value *LHS, Value *RHS) {
695 const ConstantInt *CI = dyn_cast<ConstantInt>(Val: LHS);
696 return (CI && CI->isOne()) ? RHS : B.CreateMul(LHS, RHS);
697}
698
699void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
700 AtomicRMWInst::BinOp Op,
701 unsigned ValIdx,
702 bool ValDivergent,
703 bool IsLDS) const {
704 // Don't generate a DPP scan if !amdgpu.expected.active.lane hint indicates
705 // insufficient lanes to offset fixed overhead.
706
707 // FIXME: The threshold was tuned empirically on gfx11 and gfx12. The DPP scan
708 // overhead differs across subtargets, so the break-even point may differ too;
709 // this may need to become subtarget-dependent.
710 if (IsLDS && ValDivergent && ScanImpl == ScanOptions::DPP) {
711 if (MDNode *MD = I.getMetadata(Kind: "amdgpu.expected.active.lanes")) {
712 auto *CI = mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0));
713 constexpr unsigned ActiveLanesThreshold = 5;
714 if (CI->getValue().ule(RHS: ActiveLanesThreshold))
715 return;
716 }
717 }
718
719 // Start building just before the instruction.
720 IRBuilder<> B(&I);
721
722 if (AtomicRMWInst::isFPOperation(Op)) {
723 B.setIsFPConstrained(I.getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
724 }
725
726 // If we are in a pixel shader, because of how we have to mask out helper
727 // lane invocations, we need to record the entry and exit BB's.
728 BasicBlock *PixelEntryBB = nullptr;
729 BasicBlock *PixelExitBB = nullptr;
730
731 // If we're optimizing an atomic within a pixel shader, we need to wrap the
732 // entire atomic operation in a helper-lane check. We do not want any helper
733 // lanes that are around only for the purposes of derivatives to take part
734 // in any cross-lane communication, and we use a branch on whether the lane is
735 // live to do this.
736 if (IsPixelShader) {
737 // Record I's original position as the entry block.
738 PixelEntryBB = I.getParent();
739
740 Value *const Cond = B.CreateIntrinsic(ID: Intrinsic::amdgcn_ps_live, Args: {});
741 Instruction *const NonHelperTerminator =
742 SplitBlockAndInsertIfThen(Cond, SplitBefore: &I, Unreachable: false, BranchWeights: nullptr, DTU: &DTU, LI: nullptr);
743
744 // Record I's new position as the exit block.
745 PixelExitBB = I.getParent();
746
747 I.moveBefore(InsertPos: NonHelperTerminator->getIterator());
748 B.SetInsertPoint(&I);
749 }
750
751 Type *const Ty = I.getType();
752 Type *Int32Ty = B.getInt32Ty();
753 bool isAtomicFloatingPointTy = Ty->isFloatingPointTy();
754 [[maybe_unused]] const unsigned TyBitWidth = DL.getTypeSizeInBits(Ty);
755
756 // This is the value in the atomic operation we need to combine in order to
757 // reduce the number of atomic operations.
758 Value *V = I.getOperand(i: ValIdx);
759
760 // We need to know how many lanes are active within the wavefront, and we do
761 // this by doing a ballot of active lanes.
762 Type *const WaveTy = B.getIntNTy(N: ST.getWavefrontSize());
763 CallInst *const Ballot = B.CreateIntrinsicWithoutFolding(
764 ID: Intrinsic::amdgcn_ballot, OverloadTypes: WaveTy, Args: B.getTrue());
765
766 // We need to know how many lanes are active within the wavefront that are
767 // below us. If we counted each lane linearly starting from 0, a lane is
768 // below us only if its associated index was less than ours. We do this by
769 // using the mbcnt intrinsic.
770 Value *Mbcnt;
771 if (ST.isWave32()) {
772 Mbcnt =
773 B.CreateIntrinsic(ID: Intrinsic::amdgcn_mbcnt_lo, Args: {Ballot, B.getInt32(C: 0)});
774 } else {
775 Value *const ExtractLo = B.CreateTrunc(V: Ballot, DestTy: Int32Ty);
776 Value *const ExtractHi = B.CreateTrunc(V: B.CreateLShr(LHS: Ballot, RHS: 32), DestTy: Int32Ty);
777 Mbcnt = B.CreateIntrinsic(ID: Intrinsic::amdgcn_mbcnt_lo,
778 Args: {ExtractLo, B.getInt32(C: 0)});
779 Mbcnt = B.CreateIntrinsic(ID: Intrinsic::amdgcn_mbcnt_hi, Args: {ExtractHi, Mbcnt});
780 }
781
782 Function *F = I.getFunction();
783 LLVMContext &C = F->getContext();
784 const bool NeedResult = !I.use_empty();
785 const bool UseWaveReductionIntrinsic = !ValDivergent || !NeedResult;
786
787 // For atomic sub, perform scan with add operation and allow one lane to
788 // subtract the reduced value later.
789 AtomicRMWInst::BinOp ScanOp = Op;
790 if (Op == AtomicRMWInst::Sub) {
791 ScanOp = AtomicRMWInst::Add;
792 } else if (Op == AtomicRMWInst::FSub) {
793 ScanOp = AtomicRMWInst::FAdd;
794 }
795 Value *Identity = getIdentityValueForAtomicOp(Ty, Op: ScanOp);
796
797 Value *ExclScan = nullptr;
798 Value *NewV = nullptr;
799
800 BasicBlock *ComputeLoop = nullptr;
801 BasicBlock *ComputeEnd = nullptr;
802 if (UseWaveReductionIntrinsic) {
803 // Build reductions with wave-reduce intrinsics.
804 unsigned Strategy = ScanImpl == ScanOptions::DPP ? 2 : 1;
805 Intrinsic::ID WaveRedIntrinsic = getWaveReductionIntrinsic(Op);
806 NewV = B.CreateIntrinsic(ID: WaveRedIntrinsic, OverloadTypes: Ty, Args: {V, B.getInt32(C: Strategy)});
807 } else {
808 // If we have a divergent value in each lane, we need to combine the value
809 // using DPP.
810 assert(ValDivergent && NeedResult);
811 if (ScanImpl == ScanOptions::DPP) {
812 // First we need to set all inactive invocations to the identity value,
813 // so that they can correctly contribute to the final result.
814 NewV =
815 B.CreateIntrinsic(ID: Intrinsic::amdgcn_set_inactive, OverloadTypes: Ty, Args: {V, Identity});
816 if (!NeedResult && ST.hasPermlane16Insts()) {
817 // On GFX10 the permlanex16 instruction helps us build a reduction
818 // without too many readlanes and writelanes, which are generally bad
819 // for performance.
820 NewV = buildReduction(B, Op: ScanOp, V: NewV, Identity);
821 } else {
822 NewV = buildScan(B, Op: ScanOp, V: NewV, Identity);
823 if (NeedResult)
824 ExclScan = buildShiftRight(B, V: NewV, Identity);
825 // Read the value from the last lane, which has accumulated the values
826 // of each active lane in the wavefront. This will be our new value
827 // which we will provide to the atomic operation.
828 Value *const LastLaneIdx = B.getInt32(C: ST.getWavefrontSize() - 1);
829 NewV = B.CreateIntrinsic(RetTy: Ty, ID: Intrinsic::amdgcn_readlane,
830 Args: {NewV, LastLaneIdx});
831 }
832 // Finally mark the readlanes in the WWM section.
833 NewV = B.CreateIntrinsic(ID: Intrinsic::amdgcn_strict_wwm, OverloadTypes: Ty, Args: NewV);
834 } else if (ScanImpl == ScanOptions::Iterative) {
835 // Alternative implementation for scan
836 ComputeLoop = BasicBlock::Create(Context&: C, Name: "ComputeLoop", Parent: F);
837 ComputeEnd = BasicBlock::Create(Context&: C, Name: "ComputeEnd", Parent: F);
838 std::tie(args&: ExclScan, args&: NewV) = buildScanIteratively(B, Op: ScanOp, Identity, V, I,
839 ComputeLoop, ComputeEnd);
840 } else {
841 llvm_unreachable("Atomic Optimzer is disabled for None strategy");
842 }
843 }
844
845 // We only want a single lane to enter our new control flow, and we do this
846 // by checking if there are any active lanes below us. Only one lane will
847 // have 0 active lanes below us, so that will be the only one to progress.
848 Value *const Cond = B.CreateICmpEQ(LHS: Mbcnt, RHS: B.getInt32(C: 0));
849
850 // Store I's original basic block before we split the block.
851 BasicBlock *const OriginalBB = I.getParent();
852
853 // We need to introduce some new control flow to force a single lane to be
854 // active. We do this by splitting I's basic block at I, and introducing the
855 // new block such that:
856 // entry --> single_lane -\
857 // \------------------> exit
858 Instruction *const SingleLaneTerminator =
859 SplitBlockAndInsertIfThen(Cond, SplitBefore: &I, Unreachable: false, BranchWeights: nullptr, DTU: &DTU, LI: nullptr);
860
861 // At this point, we have split the I's block to allow one lane in wavefront
862 // to update the precomputed reduced value. Also, completed the codegen for
863 // new control flow i.e. iterative loop which perform reduction and scan using
864 // ComputeLoop and ComputeEnd.
865 // For the new control flow, we need to move branch instruction i.e.
866 // terminator created during SplitBlockAndInsertIfThen from I's block to
867 // ComputeEnd block. We also need to set up predecessor to next block when
868 // single lane done updating the final reduced value.
869 BasicBlock *Predecessor = nullptr;
870 if (NeedResult && ValDivergent && ScanImpl == ScanOptions::Iterative) {
871 // Move terminator from I's block to ComputeEnd block.
872 //
873 // OriginalBB is known to have a branch as terminator because
874 // SplitBlockAndInsertIfThen will have inserted one.
875 CondBrInst *Terminator = cast<CondBrInst>(Val: OriginalBB->getTerminator());
876 B.SetInsertPoint(ComputeEnd);
877 Terminator->removeFromParent();
878 B.Insert(I: Terminator);
879
880 // Branch to ComputeLoop Block unconditionally from the I's block for
881 // iterative approach.
882 B.SetInsertPoint(OriginalBB);
883 B.CreateBr(Dest: ComputeLoop);
884
885 // Update the dominator tree for new control flow.
886 SmallVector<DominatorTree::UpdateType, 6> DomTreeUpdates(
887 {{DominatorTree::Insert, OriginalBB, ComputeLoop},
888 {DominatorTree::Insert, ComputeLoop, ComputeEnd}});
889
890 // We're moving the terminator from EntryBB to ComputeEnd, make sure we move
891 // the DT edges as well.
892 for (auto *Succ : Terminator->successors()) {
893 DomTreeUpdates.push_back(Elt: {DominatorTree::Insert, ComputeEnd, Succ});
894 DomTreeUpdates.push_back(Elt: {DominatorTree::Delete, OriginalBB, Succ});
895 }
896
897 DTU.applyUpdates(Updates: DomTreeUpdates);
898
899 Predecessor = ComputeEnd;
900 } else {
901 Predecessor = OriginalBB;
902 }
903 // Move the IR builder into single_lane next.
904 B.SetInsertPoint(SingleLaneTerminator);
905
906 // Clone the original atomic operation into single lane, replacing the
907 // original value with our newly created one.
908 Instruction *const NewI = I.clone();
909 B.Insert(I: NewI);
910 NewI->setOperand(i: ValIdx, Val: NewV);
911
912 // Move the IR builder into exit next, and start inserting just before the
913 // original instruction.
914 B.SetInsertPoint(&I);
915
916 if (NeedResult) {
917 // Create a PHI node to get our new atomic result into the exit block.
918 PHINode *const PHI = B.CreatePHI(Ty, NumReservedValues: 2);
919 PHI->addIncoming(V: PoisonValue::get(T: Ty), BB: Predecessor);
920 PHI->addIncoming(V: NewI, BB: SingleLaneTerminator->getParent());
921
922 // We need to broadcast the value who was the lowest active lane (the first
923 // lane) to all other lanes in the wavefront.
924
925 Value *ReadlaneVal = PHI;
926 if (TyBitWidth < 32)
927 ReadlaneVal = B.CreateZExt(V: PHI, DestTy: B.getInt32Ty());
928
929 Value *BroadcastI = B.CreateIntrinsic(
930 RetTy: ReadlaneVal->getType(), ID: Intrinsic::amdgcn_readfirstlane, Args: ReadlaneVal);
931 if (TyBitWidth < 32)
932 BroadcastI = B.CreateTrunc(V: BroadcastI, DestTy: Ty);
933
934 // Now that we have the result of our single atomic operation, we need to
935 // get our individual lane's slice into the result. We use the lane offset
936 // we previously calculated combined with the atomic result value we got
937 // from the first lane, to get our lane's index into the atomic result.
938 Value *LaneOffset = nullptr;
939 if (ValDivergent) {
940 if (ScanImpl == ScanOptions::DPP) {
941 LaneOffset =
942 B.CreateIntrinsic(ID: Intrinsic::amdgcn_strict_wwm, OverloadTypes: Ty, Args: ExclScan);
943 } else if (ScanImpl == ScanOptions::Iterative) {
944 LaneOffset = ExclScan;
945 } else {
946 llvm_unreachable("Atomic Optimzer is disabled for None strategy");
947 }
948 } else {
949 Mbcnt = isAtomicFloatingPointTy ? B.CreateUIToFP(V: Mbcnt, DestTy: Ty)
950 : B.CreateIntCast(V: Mbcnt, DestTy: Ty, isSigned: false);
951 switch (Op) {
952 default:
953 llvm_unreachable("Unhandled atomic op");
954 case AtomicRMWInst::Add:
955 case AtomicRMWInst::Sub:
956 LaneOffset = buildMul(B, LHS: V, RHS: Mbcnt);
957 break;
958 case AtomicRMWInst::And:
959 case AtomicRMWInst::Or:
960 case AtomicRMWInst::Max:
961 case AtomicRMWInst::Min:
962 case AtomicRMWInst::UMax:
963 case AtomicRMWInst::UMin:
964 case AtomicRMWInst::FMin:
965 case AtomicRMWInst::FMax:
966 LaneOffset = B.CreateSelect(C: Cond, True: Identity, False: V);
967 break;
968 case AtomicRMWInst::Xor:
969 LaneOffset = buildMul(B, LHS: V, RHS: B.CreateAnd(LHS: Mbcnt, RHS: 1));
970 break;
971 case AtomicRMWInst::FAdd:
972 case AtomicRMWInst::FSub: {
973 LaneOffset = B.CreateFMul(L: V, R: Mbcnt);
974 break;
975 }
976 }
977 }
978 Value *Result = buildNonAtomicBinOp(B, Op, LHS: BroadcastI, RHS: LaneOffset);
979 if (isAtomicFloatingPointTy) {
980 // For fadd/fsub the first active lane of LaneOffset should be the
981 // identity (-0.0 for fadd or +0.0 for fsub) but the value we calculated
982 // is V * +0.0 which might have the wrong sign or might be nan (if V is
983 // inf or nan).
984 //
985 // For all floating point ops if the in-memory value was a nan then the
986 // binop we just built might have quieted it or changed its payload.
987 //
988 // Correct all these problems by using BroadcastI as the result in the
989 // first active lane.
990 Result = B.CreateSelect(C: Cond, True: BroadcastI, False: Result);
991 }
992
993 if (IsPixelShader) {
994 // Need a final PHI to reconverge to above the helper lane branch mask.
995 B.SetInsertPoint(PixelExitBB->getFirstNonPHIIt());
996
997 PHINode *const PHI = B.CreatePHI(Ty, NumReservedValues: 2);
998 PHI->addIncoming(V: PoisonValue::get(T: Ty), BB: PixelEntryBB);
999 PHI->addIncoming(V: Result, BB: I.getParent());
1000 I.replaceAllUsesWith(V: PHI);
1001 } else {
1002 // Replace the original atomic instruction with the new one.
1003 I.replaceAllUsesWith(V: Result);
1004 }
1005 }
1006
1007 // And delete the original.
1008 I.eraseFromParent();
1009}
1010
1011INITIALIZE_PASS_BEGIN(AMDGPUAtomicOptimizer, DEBUG_TYPE,
1012 "AMDGPU atomic optimizations", false, false)
1013INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
1014INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
1015INITIALIZE_PASS_END(AMDGPUAtomicOptimizer, DEBUG_TYPE,
1016 "AMDGPU atomic optimizations", false, false)
1017
1018FunctionPass *llvm::createAMDGPUAtomicOptimizerPass(ScanOptions ScanStrategy) {
1019 return new AMDGPUAtomicOptimizer(ScanStrategy);
1020}
1021