1//===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This pass does misc. AMDGPU optimizations on IR *just* before instruction
11/// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "AMDGPU.h"
16#include "AMDGPUMemoryUtils.h"
17#include "AMDGPUTargetMachine.h"
18#include "llvm/Analysis/AssumptionCache.h"
19#include "llvm/Analysis/Loads.h"
20#include "llvm/Analysis/UniformityAnalysis.h"
21#include "llvm/Analysis/ValueTracking.h"
22#include "llvm/CodeGen/TargetPassConfig.h"
23#include "llvm/IR/IRBuilder.h"
24#include "llvm/IR/InstVisitor.h"
25#include "llvm/IR/IntrinsicsAMDGPU.h"
26#include "llvm/InitializePasses.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Transforms/Utils/Local.h"
29
30#define DEBUG_TYPE "amdgpu-late-codegenprepare"
31
32using namespace llvm;
33
34// Scalar load widening needs running after load-store-vectorizer as that pass
35// doesn't handle overlapping cases. In addition, this pass enhances the
36// widening to handle cases where scalar sub-dword loads are naturally aligned
37// only but not dword aligned.
38static cl::opt<bool>
39 WidenLoads("amdgpu-late-codegenprepare-widen-constant-loads",
40 cl::desc("Widen sub-dword constant address space loads in "
41 "AMDGPULateCodeGenPrepare"),
42 cl::ReallyHidden, cl::init(Val: true));
43
44namespace {
45
46class AMDGPULateCodeGenPrepare
47 : public InstVisitor<AMDGPULateCodeGenPrepare, bool> {
48 Function &F;
49 const DataLayout &DL;
50 const GCNSubtarget &ST;
51
52 AssumptionCache *const AC;
53 UniformityInfo &UA;
54
55 SmallVector<WeakTrackingVH, 8> DeadInsts;
56
57public:
58 AMDGPULateCodeGenPrepare(Function &F, const GCNSubtarget &ST,
59 AssumptionCache *AC, UniformityInfo &UA)
60 : F(F), DL(F.getDataLayout()), ST(ST), AC(AC), UA(UA) {}
61 bool run();
62 bool visitInstruction(Instruction &) { return false; }
63
64 // Widening may read padding bytes past the original access, so require the
65 // whole AccessSize-byte range at Base to be dereferenceable, not just Base
66 // itself aligned.
67 bool isSafeToWidenLoad(const Value *Base, uint64_t AccessSize,
68 const Instruction *CxtI) const {
69 return isDereferenceableAndAlignedPointer(
70 V: Base, Alignment: Align(4),
71 Size: APInt(DL.getIndexTypeSizeInBits(Ty: Base->getType()), AccessSize),
72 Q: SimplifyQuery(DL, /*TLI=*/nullptr, /*DT=*/nullptr, AC, CxtI));
73 }
74
75 bool canWidenScalarExtLoad(LoadInst &LI) const;
76 bool visitLoadInst(LoadInst &LI);
77};
78
79using ValueToValueMap = DenseMap<const Value *, Value *>;
80
81class LiveRegOptimizer {
82private:
83 Module &Mod;
84 const DataLayout &DL;
85 const GCNSubtarget &ST;
86
87 /// The scalar type to convert to
88 Type *const ConvertToScalar;
89 /// Map of Value -> Converted Value
90 ValueToValueMap ValMap;
91 /// Map of containing conversions from Optimal Type -> Original Type per BB.
92 DenseMap<BasicBlock *, ValueToValueMap> BBUseValMap;
93
94public:
95 /// Calculate the and \p return the type to convert to given a problematic \p
96 /// OriginalType. In some instances, we may widen the type (e.g. v2i8 -> i32).
97 Type *calculateConvertType(Type *OriginalType);
98 /// Convert the virtual register defined by \p V to the compatible vector of
99 /// legal type
100 Value *convertToOptType(Instruction *V, BasicBlock::iterator &InstPt);
101 /// Convert the virtual register defined by \p V back to the original type \p
102 /// ConvertType, stripping away the MSBs in cases where there was an imperfect
103 /// fit (e.g. v2i32 -> v7i8)
104 Value *convertFromOptType(Type *ConvertType, Instruction *V,
105 BasicBlock::iterator &InstPt,
106 BasicBlock *InsertBlock);
107 /// Check for problematic PHI nodes or cross-bb values based on the value
108 /// defined by \p I, and coerce to legal types if necessary. For problematic
109 /// PHI node, we coerce all incoming values in a single invocation.
110 bool optimizeLiveType(Instruction *I,
111 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
112
113 // Whether or not the type should be replaced to avoid inefficient
114 // legalization code
115 bool shouldReplace(Type *ITy) {
116 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: ITy);
117 if (!VTy)
118 return false;
119
120 const auto *TLI = ST.getTargetLowering();
121
122 Type *EltTy = VTy->getElementType();
123 // If the element size is not is not a multiple scalar size, then we can't
124 // do any bit packing
125 if (!EltTy->isIntegerTy() ||
126 ConvertToScalar->getScalarSizeInBits() % EltTy->getScalarSizeInBits())
127 return false;
128
129 // Only coerce illegal types
130 TargetLoweringBase::LegalizeKind LK =
131 TLI->getTypeConversion(Context&: EltTy->getContext(), VT: EVT::getEVT(Ty: EltTy, HandleUnknown: false));
132 return LK.first != TargetLoweringBase::TypeLegal;
133 }
134
135 bool isOpLegal(const Instruction *I) {
136 if (isa<IntrinsicInst>(Val: I))
137 return true;
138
139 // Any store is a profitable sink (prevents flip-flopping)
140 if (isa<StoreInst>(Val: I))
141 return true;
142
143 if (auto *BO = dyn_cast<BinaryOperator>(Val: I)) {
144 if (auto *VT = dyn_cast<FixedVectorType>(Val: BO->getType())) {
145 if (const auto *IT = dyn_cast<IntegerType>(Val: VT->getElementType())) {
146 unsigned EB = IT->getBitWidth();
147 unsigned EC = VT->getNumElements();
148 // Check for SDWA-compatible operation
149 if ((EB == 8 || EB == 16) && ST.hasSDWA() && EC * EB <= 32) {
150 switch (BO->getOpcode()) {
151 case Instruction::Add:
152 case Instruction::Sub:
153 case Instruction::And:
154 case Instruction::Or:
155 case Instruction::Xor:
156 return true;
157 default:
158 break;
159 }
160 }
161 }
162 }
163 }
164
165 return false;
166 }
167
168 bool isCoercionProfitable(Instruction *II) {
169 SmallPtrSet<Instruction *, 4> CVisited;
170 SmallVector<Instruction *, 4> UserList;
171
172 // Check users for profitable conditions (across block user which can
173 // natively handle the illegal vector).
174 for (User *V : II->users())
175 if (auto *UseInst = dyn_cast<Instruction>(Val: V))
176 UserList.push_back(Elt: UseInst);
177
178 auto IsLookThru = [](Instruction *II) {
179 if (const auto *Intr = dyn_cast<IntrinsicInst>(Val: II))
180 return Intr->getIntrinsicID() == Intrinsic::amdgcn_perm;
181 return isa<PHINode, ShuffleVectorInst, InsertElementInst,
182 ExtractElementInst, CastInst>(Val: II);
183 };
184
185 while (!UserList.empty()) {
186 auto CII = UserList.pop_back_val();
187 if (!CVisited.insert(Ptr: CII).second)
188 continue;
189
190 // Same-BB filter must look at the *user*; and allow non-lookthrough
191 // users when the def is a PHI (loop-header pattern).
192 if (CII->getParent() == II->getParent() && !IsLookThru(CII) &&
193 !isa<PHINode>(Val: II))
194 continue;
195
196 if (isOpLegal(I: CII))
197 return true;
198
199 if (IsLookThru(CII))
200 for (User *V : CII->users())
201 if (auto *UseInst = dyn_cast<Instruction>(Val: V))
202 UserList.push_back(Elt: UseInst);
203 }
204 return false;
205 }
206
207 LiveRegOptimizer(Module &Mod, const GCNSubtarget &ST)
208 : Mod(Mod), DL(Mod.getDataLayout()), ST(ST),
209 ConvertToScalar(Type::getInt32Ty(C&: Mod.getContext())) {}
210};
211
212} // end anonymous namespace
213
214bool AMDGPULateCodeGenPrepare::run() {
215 // "Optimize" the virtual regs that cross basic block boundaries. When
216 // building the SelectionDAG, vectors of illegal types that cross basic blocks
217 // will be scalarized and widened, with each scalar living in its
218 // own register. To work around this, this optimization converts the
219 // vectors to equivalent vectors of legal type (which are converted back
220 // before uses in subsequent blocks), to pack the bits into fewer physical
221 // registers (used in CopyToReg/CopyFromReg pairs).
222 LiveRegOptimizer LRO(*F.getParent(), ST);
223
224 bool Changed = false;
225
226 bool HasScalarSubwordLoads = ST.hasScalarSubwordLoads();
227
228 for (auto &BB : reverse(C&: F))
229 for (Instruction &I : make_early_inc_range(Range: reverse(C&: BB))) {
230 Changed |= !HasScalarSubwordLoads && visit(I);
231 Changed |= LRO.optimizeLiveType(I: &I, DeadInsts);
232 }
233
234 RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts);
235 return Changed;
236}
237
238Type *LiveRegOptimizer::calculateConvertType(Type *OriginalType) {
239 assert(OriginalType->getScalarSizeInBits() <=
240 ConvertToScalar->getScalarSizeInBits());
241
242 FixedVectorType *VTy = cast<FixedVectorType>(Val: OriginalType);
243
244 TypeSize OriginalSize = DL.getTypeSizeInBits(Ty: VTy);
245 TypeSize ConvertScalarSize = DL.getTypeSizeInBits(Ty: ConvertToScalar);
246 unsigned ConvertEltCount =
247 (OriginalSize + ConvertScalarSize - 1) / ConvertScalarSize;
248
249 if (OriginalSize <= ConvertScalarSize)
250 return IntegerType::get(C&: Mod.getContext(), NumBits: ConvertScalarSize);
251
252 return VectorType::get(ElementType: Type::getIntNTy(C&: Mod.getContext(), N: ConvertScalarSize),
253 NumElements: ConvertEltCount, Scalable: false);
254}
255
256Value *LiveRegOptimizer::convertToOptType(Instruction *V,
257 BasicBlock::iterator &InsertPt) {
258 FixedVectorType *VTy = cast<FixedVectorType>(Val: V->getType());
259 Type *NewTy = calculateConvertType(OriginalType: V->getType());
260
261 TypeSize OriginalSize = DL.getTypeSizeInBits(Ty: VTy);
262 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewTy);
263
264 IRBuilder<> Builder(V->getParent(), InsertPt);
265 // If there is a bitsize match, we can fit the old vector into a new vector of
266 // desired type.
267 if (OriginalSize == NewSize)
268 return Builder.CreateBitCast(V, DestTy: NewTy, Name: V->getName() + ".bc");
269
270 // If there is a bitsize mismatch, we must use a wider vector.
271 assert(NewSize > OriginalSize);
272 uint64_t ExpandedVecElementCount = NewSize / VTy->getScalarSizeInBits();
273
274 SmallVector<int, 8> ShuffleMask;
275 uint64_t OriginalElementCount = VTy->getElementCount().getFixedValue();
276 for (unsigned I = 0; I < OriginalElementCount; I++)
277 ShuffleMask.push_back(Elt: I);
278
279 for (uint64_t I = OriginalElementCount; I < ExpandedVecElementCount; I++)
280 ShuffleMask.push_back(Elt: OriginalElementCount);
281
282 Value *ExpandedVec = Builder.CreateShuffleVector(V, Mask: ShuffleMask);
283 return Builder.CreateBitCast(V: ExpandedVec, DestTy: NewTy, Name: V->getName() + ".bc");
284}
285
286Value *LiveRegOptimizer::convertFromOptType(Type *ConvertType, Instruction *V,
287 BasicBlock::iterator &InsertPt,
288 BasicBlock *InsertBB) {
289 FixedVectorType *NewVTy = cast<FixedVectorType>(Val: ConvertType);
290
291 TypeSize OriginalSize = DL.getTypeSizeInBits(Ty: V->getType());
292 TypeSize NewSize = DL.getTypeSizeInBits(Ty: NewVTy);
293
294 IRBuilder<> Builder(InsertBB, InsertPt);
295 // If there is a bitsize match, we simply convert back to the original type.
296 if (OriginalSize == NewSize)
297 return Builder.CreateBitCast(V, DestTy: NewVTy, Name: V->getName() + ".bc");
298
299 // If there is a bitsize mismatch, then we must have used a wider value to
300 // hold the bits.
301 assert(OriginalSize > NewSize);
302 // For wide scalars, we can just truncate the value.
303 if (!V->getType()->isVectorTy()) {
304 Instruction *Trunc = cast<Instruction>(
305 Val: Builder.CreateTrunc(V, DestTy: IntegerType::get(C&: Mod.getContext(), NumBits: NewSize)));
306 return cast<Instruction>(Val: Builder.CreateBitCast(V: Trunc, DestTy: NewVTy));
307 }
308
309 // For wider vectors, we must strip the MSBs to convert back to the original
310 // type.
311 VectorType *ExpandedVT = VectorType::get(
312 ElementType: Type::getIntNTy(C&: Mod.getContext(), N: NewVTy->getScalarSizeInBits()),
313 NumElements: (OriginalSize / NewVTy->getScalarSizeInBits()), Scalable: false);
314 Instruction *Converted =
315 cast<Instruction>(Val: Builder.CreateBitCast(V, DestTy: ExpandedVT));
316
317 unsigned NarrowElementCount = NewVTy->getElementCount().getFixedValue();
318 SmallVector<int, 8> ShuffleMask(NarrowElementCount);
319 std::iota(first: ShuffleMask.begin(), last: ShuffleMask.end(), value: 0);
320
321 return Builder.CreateShuffleVector(V: Converted, Mask: ShuffleMask);
322}
323
324bool LiveRegOptimizer::optimizeLiveType(
325 Instruction *I, SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
326 SmallVector<Instruction *, 4> Worklist;
327 SmallPtrSet<PHINode *, 4> PhiNodes;
328 SmallPtrSet<Instruction *, 4> Defs;
329 SmallPtrSet<Instruction *, 4> Uses;
330 SmallPtrSet<Instruction *, 4> Visited;
331
332 Worklist.push_back(Elt: cast<Instruction>(Val: I));
333 while (!Worklist.empty()) {
334 Instruction *II = Worklist.pop_back_val();
335
336 if (!Visited.insert(Ptr: II).second)
337 continue;
338
339 if (!shouldReplace(ITy: II->getType()))
340 continue;
341
342 if (!isCoercionProfitable(II))
343 continue;
344
345 if (PHINode *Phi = dyn_cast<PHINode>(Val: II)) {
346 PhiNodes.insert(Ptr: Phi);
347 // Collect all the incoming values of problematic PHI nodes.
348 for (Value *V : Phi->incoming_values()) {
349 // Repeat the collection process for newly found PHI nodes.
350 if (PHINode *OpPhi = dyn_cast<PHINode>(Val: V)) {
351 if (!PhiNodes.count(Ptr: OpPhi) && !Visited.count(Ptr: OpPhi))
352 Worklist.push_back(Elt: OpPhi);
353 continue;
354 }
355
356 Instruction *IncInst = dyn_cast<Instruction>(Val: V);
357 // Other incoming value types (e.g. vector literals) are unhandled
358 if (!IncInst && !isa<ConstantAggregateZero>(Val: V))
359 return false;
360
361 // Collect all other incoming values for coercion.
362 if (IncInst && !IncInst->isTerminator())
363 Defs.insert(Ptr: IncInst);
364 }
365 }
366
367 // Collect all relevant uses.
368 for (User *V : II->users()) {
369 // Repeat the collection process for problematic PHI nodes.
370 if (PHINode *OpPhi = dyn_cast<PHINode>(Val: V)) {
371 if (!PhiNodes.count(Ptr: OpPhi) && !Visited.count(Ptr: OpPhi))
372 Worklist.push_back(Elt: OpPhi);
373 continue;
374 }
375
376 Instruction *UseInst = cast<Instruction>(Val: V);
377 // Collect all uses of PHINodes and any use the crosses BB boundaries.
378 if (UseInst->getParent() != II->getParent() || isa<PHINode>(Val: II)) {
379 Uses.insert(Ptr: UseInst);
380 if (!isa<PHINode>(Val: II) && !II->isTerminator())
381 Defs.insert(Ptr: II);
382 }
383 }
384 }
385
386 // Coerce and track the defs.
387 for (Instruction *D : Defs) {
388 if (!ValMap.contains(Val: D)) {
389 BasicBlock::iterator InsertPt = std::next(x: D->getIterator());
390 Value *ConvertVal = convertToOptType(V: D, InsertPt);
391 assert(ConvertVal);
392 ValMap[D] = ConvertVal;
393 }
394 }
395
396 // Construct new-typed PHI nodes.
397 for (PHINode *Phi : PhiNodes) {
398 ValMap[Phi] = PHINode::Create(Ty: calculateConvertType(OriginalType: Phi->getType()),
399 NumReservedValues: Phi->getNumIncomingValues(),
400 NameStr: Phi->getName() + ".tc", InsertBefore: Phi->getIterator());
401 }
402
403 // Connect all the PHI nodes with their new incoming values.
404 for (PHINode *Phi : PhiNodes) {
405 PHINode *NewPhi = cast<PHINode>(Val: ValMap[Phi]);
406 bool MissingIncVal = false;
407 for (int I = 0, E = Phi->getNumIncomingValues(); I < E; I++) {
408 Value *IncVal = Phi->getIncomingValue(i: I);
409 if (isa<ConstantAggregateZero>(Val: IncVal)) {
410 Type *NewType = calculateConvertType(OriginalType: Phi->getType());
411 NewPhi->addIncoming(V: ConstantInt::get(Ty: NewType, V: 0, IsSigned: false),
412 BB: Phi->getIncomingBlock(i: I));
413 } else if (Value *Val = ValMap.lookup(Val: IncVal))
414 NewPhi->addIncoming(V: Val, BB: Phi->getIncomingBlock(i: I));
415 else
416 MissingIncVal = true;
417 }
418 if (MissingIncVal) {
419 Value *DeadVal = ValMap[Phi];
420 // The coercion chain of the PHI is broken. Delete the Phi
421 // from the ValMap and any connected / user Phis.
422 SmallVector<Value *, 4> PHIWorklist;
423 SmallPtrSet<Value *, 4> VisitedPhis;
424 PHIWorklist.push_back(Elt: DeadVal);
425 while (!PHIWorklist.empty()) {
426 Value *NextDeadValue = PHIWorklist.pop_back_val();
427 VisitedPhis.insert(Ptr: NextDeadValue);
428 auto OriginalPhi =
429 llvm::find_if(Range&: PhiNodes, P: [this, &NextDeadValue](PHINode *CandPhi) {
430 return ValMap[CandPhi] == NextDeadValue;
431 });
432 // This PHI may have already been removed from maps when
433 // unwinding a previous Phi
434 if (OriginalPhi != PhiNodes.end())
435 ValMap.erase(Val: *OriginalPhi);
436
437 for (User *U : NextDeadValue->users()) {
438 if (!VisitedPhis.contains(Ptr: cast<PHINode>(Val: U)))
439 PHIWorklist.push_back(Elt: U);
440 }
441 NextDeadValue->replaceAllUsesWith(
442 V: PoisonValue::get(T: NextDeadValue->getType()));
443
444 DeadInsts.emplace_back(Args: cast<Instruction>(Val: NextDeadValue));
445 }
446 } else {
447 DeadInsts.emplace_back(Args: cast<Instruction>(Val: Phi));
448 }
449 }
450 // Coerce back to the original type and replace the uses.
451 for (Instruction *U : Uses) {
452 // Replace all converted operands for a use.
453 for (auto [OpIdx, Op] : enumerate(First: U->operands())) {
454 if (Value *Val = ValMap.lookup(Val: Op)) {
455 Value *NewVal = nullptr;
456 if (BBUseValMap.contains(Val: U->getParent()) &&
457 BBUseValMap[U->getParent()].contains(Val))
458 NewVal = BBUseValMap[U->getParent()][Val];
459 else {
460 // Not getFirstNonPHIIt, which would insert in front of a landingpad.
461 BasicBlock::iterator InsertPt = U->getParent()->getFirstInsertionPt();
462 // We may pick up ops that were previously converted for users in
463 // other blocks. If there is an originally typed definition of the Op
464 // already in this block, simply reuse it.
465 if (isa<Instruction>(Val: Op) && !isa<PHINode>(Val: Op) &&
466 U->getParent() == cast<Instruction>(Val&: Op)->getParent()) {
467 NewVal = Op;
468 } else {
469 NewVal =
470 convertFromOptType(ConvertType: Op->getType(), V: cast<Instruction>(Val: ValMap[Op]),
471 InsertPt, InsertBB: U->getParent());
472 BBUseValMap[U->getParent()][ValMap[Op]] = NewVal;
473 }
474 }
475 assert(NewVal);
476 U->setOperand(i: OpIdx, Val: NewVal);
477 }
478 }
479 }
480
481 return true;
482}
483
484bool AMDGPULateCodeGenPrepare::canWidenScalarExtLoad(LoadInst &LI) const {
485 unsigned AS = LI.getPointerAddressSpace();
486 // Skip non-constant address space.
487 if (AS != AMDGPUAS::CONSTANT_ADDRESS &&
488 AS != AMDGPUAS::CONSTANT_ADDRESS_32BIT)
489 return false;
490 // Skip non-simple loads.
491 if (!LI.isSimple())
492 return false;
493 Type *Ty = LI.getType();
494 // Skip aggregate types.
495 if (Ty->isAggregateType())
496 return false;
497 unsigned TySize = DL.getTypeStoreSize(Ty);
498 // Only handle sub-DWORD loads.
499 if (TySize >= 4)
500 return false;
501 // That load must be at least naturally aligned.
502 if (LI.getAlign() < DL.getABITypeAlign(Ty))
503 return false;
504 // It should be uniform, i.e. a scalar load.
505 return UA.isUniformAtDef(V: &LI);
506}
507
508bool AMDGPULateCodeGenPrepare::visitLoadInst(LoadInst &LI) {
509 if (!WidenLoads)
510 return false;
511
512 // Skip if that load is already aligned on DWORD at least as it's handled in
513 // SDAG.
514 if (LI.getAlign() >= 4)
515 return false;
516
517 if (!canWidenScalarExtLoad(LI))
518 return false;
519
520 int64_t Offset = 0;
521 auto *Base =
522 GetPointerBaseWithConstantOffset(Ptr: LI.getPointerOperand(), Offset, DL);
523
524 int64_t Adjust = Offset & 0x3;
525 int64_t AccessOffset = Offset - Adjust;
526 if (AccessOffset < 0 || !isSafeToWidenLoad(Base, AccessSize: AccessOffset + 4, CxtI: &LI))
527 return false;
528
529 IRBuilder<> IRB(&LI);
530 IRB.SetCurrentDebugLocation(LI.getDebugLoc());
531
532 unsigned LdBits = DL.getTypeStoreSizeInBits(Ty: LI.getType());
533 auto *IntNTy = Type::getIntNTy(C&: LI.getContext(), N: LdBits);
534
535 auto *NewPtr = IRB.CreateConstGEP1_64(
536 Ty: IRB.getInt8Ty(),
537 Ptr: IRB.CreateAddrSpaceCast(V: Base, DestTy: LI.getPointerOperand()->getType()),
538 Idx0: Offset - Adjust);
539
540 LoadInst *NewLd = IRB.CreateAlignedLoad(Ty: IRB.getInt32Ty(), Ptr: NewPtr, Align: Align(4));
541 AMDGPU::copyMetadataForWidenedLoad(Dest&: *NewLd, Source: LI);
542
543 unsigned ShAmt = Adjust * 8;
544 Value *Shifted = ShAmt ? IRB.CreateLShr(LHS: NewLd, RHS: ShAmt) : NewLd;
545 Value *NewVal = IRB.CreateBitCast(
546 V: IRB.CreateTrunc(V: Shifted, DestTy: DL.typeSizeEqualsStoreSize(Ty: LI.getType())
547 ? IntNTy
548 : LI.getType()),
549 DestTy: LI.getType());
550 LI.replaceAllUsesWith(V: NewVal);
551 DeadInsts.emplace_back(Args: &LI);
552
553 return true;
554}
555
556PreservedAnalyses
557AMDGPULateCodeGenPreparePass::run(Function &F, FunctionAnalysisManager &FAM) {
558 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
559 AssumptionCache &AC = FAM.getResult<AssumptionAnalysis>(IR&: F);
560 UniformityInfo &UI = FAM.getResult<UniformityInfoAnalysis>(IR&: F);
561
562 bool Changed = AMDGPULateCodeGenPrepare(F, ST, &AC, UI).run();
563
564 if (!Changed)
565 return PreservedAnalyses::all();
566 PreservedAnalyses PA = PreservedAnalyses::none();
567 PA.preserveSet<CFGAnalyses>();
568 return PA;
569}
570
571class AMDGPULateCodeGenPrepareLegacy : public FunctionPass {
572public:
573 static char ID;
574
575 AMDGPULateCodeGenPrepareLegacy() : FunctionPass(ID) {}
576
577 StringRef getPassName() const override {
578 return "AMDGPU IR late optimizations";
579 }
580
581 void getAnalysisUsage(AnalysisUsage &AU) const override {
582 AU.addRequired<TargetPassConfig>();
583 AU.addRequired<AssumptionCacheTracker>();
584 AU.addRequired<UniformityInfoWrapperPass>();
585 // Invalidates UniformityInfo
586 AU.setPreservesCFG();
587 }
588
589 bool runOnFunction(Function &F) override;
590};
591
592bool AMDGPULateCodeGenPrepareLegacy::runOnFunction(Function &F) {
593 if (skipFunction(F))
594 return false;
595
596 const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
597 const TargetMachine &TM = TPC.getTM<TargetMachine>();
598 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
599
600 AssumptionCache &AC =
601 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
602 UniformityInfo &UI =
603 getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
604
605 return AMDGPULateCodeGenPrepare(F, ST, &AC, UI).run();
606}
607
608INITIALIZE_PASS_BEGIN(AMDGPULateCodeGenPrepareLegacy, DEBUG_TYPE,
609 "AMDGPU IR late optimizations", false, false)
610INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
611INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
612INITIALIZE_PASS_DEPENDENCY(UniformityInfoWrapperPass)
613INITIALIZE_PASS_END(AMDGPULateCodeGenPrepareLegacy, DEBUG_TYPE,
614 "AMDGPU IR late optimizations", false, false)
615
616char AMDGPULateCodeGenPrepareLegacy::ID = 0;
617
618FunctionPass *llvm::createAMDGPULateCodeGenPrepareLegacyPass() {
619 return new AMDGPULateCodeGenPrepareLegacy();
620}
621