1//===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetTransformInfo.h"
18#include "AMDGPUSubtarget.h"
19#include "AMDGPUTargetMachine.h"
20#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21#include "SIModeRegisterDefaults.h"
22#include "llvm/ADT/SmallBitVector.h"
23#include "llvm/Analysis/InlineCost.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/CodeGen/Analysis.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/Support/KnownBits.h"
32#include <optional>
33
34using namespace llvm;
35
36#define DEBUG_TYPE "AMDGPUtti"
37
38static cl::opt<unsigned> UnrollThresholdPrivate(
39 "amdgpu-unroll-threshold-private",
40 cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),
41 cl::init(Val: 2700), cl::Hidden);
42
43static cl::opt<unsigned> UnrollThresholdLocal(
44 "amdgpu-unroll-threshold-local",
45 cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),
46 cl::init(Val: 1000), cl::Hidden);
47
48static cl::opt<unsigned> UnrollThresholdIf(
49 "amdgpu-unroll-threshold-if",
50 cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),
51 cl::init(Val: 200), cl::Hidden);
52
53static cl::opt<bool> UnrollRuntimeLocal(
54 "amdgpu-unroll-runtime-local",
55 cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
56 cl::init(Val: true), cl::Hidden);
57
58static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
59 "amdgpu-unroll-max-block-to-analyze",
60 cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),
61 cl::init(Val: 32), cl::Hidden);
62
63static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost",
64 cl::Hidden, cl::init(Val: 4000),
65 cl::desc("Cost of alloca argument"));
66
67// If the amount of scratch memory to eliminate exceeds our ability to allocate
68// it into registers we gain nothing by aggressively inlining functions for that
69// heuristic.
70static cl::opt<unsigned>
71 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden,
72 cl::init(Val: 256),
73 cl::desc("Maximum alloca size to use for inline cost"));
74
75// Inliner constraint to achieve reasonable compilation time.
76static cl::opt<size_t> InlineMaxBB(
77 "amdgpu-inline-max-bb", cl::Hidden, cl::init(Val: 1100),
78 cl::desc("Maximum number of BBs allowed in a function after inlining"
79 " (compile time constraint)"));
80
81// This default unroll factor is based on microbenchmarks on gfx1030.
82static cl::opt<unsigned> MemcpyLoopUnroll(
83 "amdgpu-memcpy-loop-unroll",
84 cl::desc("Unroll factor (affecting 4x32-bit operations) to use for memory "
85 "operations when lowering statically-sized memcpy, memmove, or"
86 "memset as a loop"),
87 cl::init(Val: 16), cl::Hidden);
88
89static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
90 unsigned Depth = 0) {
91 const Instruction *I = dyn_cast<Instruction>(Val: Cond);
92 if (!I)
93 return false;
94
95 if (!L->contains(Inst: I))
96 return false;
97 for (const Value *V : I->operand_values()) {
98 if (const PHINode *PHI = dyn_cast<PHINode>(Val: V)) {
99 if (llvm::none_of(Range: L->getSubLoops(), P: [PHI](const Loop* SubLoop) {
100 return SubLoop->contains(Inst: PHI); }))
101 return true;
102 } else if (Depth < 10 && dependsOnLocalPhi(L, Cond: V, Depth: Depth+1))
103 return true;
104 }
105 return false;
106}
107
108AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
109 : BaseT(TM, F.getDataLayout()),
110 TargetTriple(TM->getTargetTriple()),
111 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
112 TLI(ST->getTargetLowering()) {}
113
114void AMDGPUTTIImpl::getUnrollingPreferences(
115 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
116 OptimizationRemarkEmitter *ORE) const {
117 const Function &F = *L->getHeader()->getParent();
118 UP.Threshold =
119 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-unroll-threshold", Default: 300);
120 UP.MaxCount = std::numeric_limits<unsigned>::max();
121 UP.Partial = true;
122
123 // Conditional branch in a loop back edge needs 3 additional exec
124 // manipulations in average.
125 UP.BEInsns += 3;
126
127 // We want to run unroll even for the loops which have been vectorized.
128 UP.UnrollVectorizedLoop = true;
129
130 // Enable runtime unrolling for loops whose trip count is not known at
131 // compile time.
132 UP.Runtime = true;
133
134 // Maximum alloca size than can fit registers. Reserve 16 registers.
135 const unsigned MaxAlloca = (256 - 16) * 4;
136 unsigned ThresholdPrivate = UnrollThresholdPrivate;
137 unsigned ThresholdLocal = UnrollThresholdLocal;
138
139 // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the
140 // provided threshold value as the default for Threshold
141 if (MDNode *LoopUnrollThreshold =
142 findOptionMDForLoop(TheLoop: L, Name: "amdgpu.loop.unroll.threshold")) {
143 if (LoopUnrollThreshold->getNumOperands() == 2) {
144 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
145 MD: LoopUnrollThreshold->getOperand(I: 1));
146 if (MetaThresholdValue) {
147 // We will also use the supplied value for PartialThreshold for now.
148 // We may introduce additional metadata if it becomes necessary in the
149 // future.
150 UP.Threshold = MetaThresholdValue->getSExtValue();
151 UP.PartialThreshold = UP.Threshold;
152 ThresholdPrivate = std::min(a: ThresholdPrivate, b: UP.Threshold);
153 ThresholdLocal = std::min(a: ThresholdLocal, b: UP.Threshold);
154 }
155 }
156 }
157
158 unsigned MaxBoost = std::max(a: ThresholdPrivate, b: ThresholdLocal);
159 for (const BasicBlock *BB : L->getBlocks()) {
160 const DataLayout &DL = BB->getDataLayout();
161 unsigned LocalGEPsSeen = 0;
162
163 if (llvm::any_of(Range: L->getSubLoops(), P: [BB](const Loop* SubLoop) {
164 return SubLoop->contains(BB); }))
165 continue; // Block belongs to an inner loop.
166
167 for (const Instruction &I : *BB) {
168 // Unroll a loop which contains an "if" statement whose condition
169 // defined by a PHI belonging to the loop. This may help to eliminate
170 // if region and potentially even PHI itself, saving on both divergence
171 // and registers used for the PHI.
172 // Add a small bonus for each of such "if" statements.
173 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: &I)) {
174 if (UP.Threshold < MaxBoost) {
175 BasicBlock *Succ0 = Br->getSuccessor(i: 0);
176 BasicBlock *Succ1 = Br->getSuccessor(i: 1);
177 if ((L->contains(BB: Succ0) && L->isLoopExiting(BB: Succ0)) ||
178 (L->contains(BB: Succ1) && L->isLoopExiting(BB: Succ1)))
179 continue;
180 if (dependsOnLocalPhi(L, Cond: Br->getCondition())) {
181 UP.Threshold += UnrollThresholdIf;
182 LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
183 << " for loop:\n"
184 << *L << " due to " << *Br << '\n');
185 if (UP.Threshold >= MaxBoost)
186 return;
187 }
188 }
189 continue;
190 }
191
192 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
193 if (!GEP)
194 continue;
195
196 unsigned AS = GEP->getAddressSpace();
197 unsigned Threshold = 0;
198 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
199 Threshold = ThresholdPrivate;
200 else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
201 Threshold = ThresholdLocal;
202 else
203 continue;
204
205 if (UP.Threshold >= Threshold)
206 continue;
207
208 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
209 const Value *Ptr = GEP->getPointerOperand();
210 const AllocaInst *Alloca =
211 dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Ptr));
212 if (!Alloca || !Alloca->isStaticAlloca())
213 continue;
214 auto AllocaSize = Alloca->getAllocationSize(DL);
215 if (!AllocaSize || AllocaSize->getFixedValue() > MaxAlloca)
216 continue;
217 } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
218 AS == AMDGPUAS::REGION_ADDRESS) {
219 LocalGEPsSeen++;
220 // Inhibit unroll for local memory if we have seen addressing not to
221 // a variable, most likely we will be unable to combine it.
222 // Do not unroll too deep inner loops for local memory to give a chance
223 // to unroll an outer loop for a more important reason.
224 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
225 (!isa<GlobalVariable>(Val: GEP->getPointerOperand()) &&
226 !isa<Argument>(Val: GEP->getPointerOperand())))
227 continue;
228 LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
229 << *L << " due to LDS use.\n");
230 UP.Runtime = UnrollRuntimeLocal;
231 }
232
233 // Check if GEP depends on a value defined by this loop itself.
234 bool HasLoopDef = false;
235 for (const Value *Op : GEP->operands()) {
236 const Instruction *Inst = dyn_cast<Instruction>(Val: Op);
237 if (!Inst || L->isLoopInvariant(V: Op))
238 continue;
239
240 if (llvm::any_of(Range: L->getSubLoops(), P: [Inst](const Loop* SubLoop) {
241 return SubLoop->contains(Inst); }))
242 continue;
243 HasLoopDef = true;
244 break;
245 }
246 if (!HasLoopDef)
247 continue;
248
249 // We want to do whatever we can to limit the number of alloca
250 // instructions that make it through to the code generator. allocas
251 // require us to use indirect addressing, which is slow and prone to
252 // compiler bugs. If this loop does an address calculation on an
253 // alloca ptr, then we want to use a higher than normal loop unroll
254 // threshold. This will give SROA a better chance to eliminate these
255 // allocas.
256 //
257 // We also want to have more unrolling for local memory to let ds
258 // instructions with different offsets combine.
259 //
260 // Don't use the maximum allowed value here as it will make some
261 // programs way too big.
262 UP.Threshold = Threshold;
263 LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
264 << " for loop:\n"
265 << *L << " due to " << *GEP << '\n');
266 if (UP.Threshold >= MaxBoost)
267 return;
268 }
269
270 // If we got a GEP in a small BB from inner loop then increase max trip
271 // count to analyze for better estimation cost in unroll
272 if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze)
273 UP.MaxIterationsCountToAnalyze = 32;
274 }
275}
276
277void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
278 TTI::PeelingPreferences &PP) const {
279 BaseT::getPeelingPreferences(L, SE, PP);
280}
281
282uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
283 return 1024;
284}
285
286GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
287 : BaseT(TM, F.getDataLayout()),
288 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
289 TLI(ST->getTargetLowering()), CommonTTI(TM, F),
290 IsGraphics(AMDGPU::isGraphics(CC: F.getCallingConv())) {
291 SIModeRegisterDefaults Mode(F, *ST);
292 HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
293}
294
295bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
296 return !F || !ST->isSingleLaneExecution(Kernel: *F);
297}
298
299unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
300 // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
301 // registers. See getRegisterClassForType for the implementation.
302 // In this case vector registers are not vector in terms of
303 // VGPRs, but those which can hold multiple values.
304
305 // This is really the number of registers to fill when vectorizing /
306 // interleaving loops, so we lie to avoid trying to use all registers.
307 return 4;
308}
309
310TypeSize
311GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
312 switch (K) {
313 case TargetTransformInfo::RGK_Scalar:
314 return TypeSize::getFixed(ExactSize: 32);
315 case TargetTransformInfo::RGK_FixedWidthVector:
316 return TypeSize::getFixed(
317 ExactSize: (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()) ? 128
318 : ST->hasAnyPackedFP32Ops() ? 64
319 : 32);
320 case TargetTransformInfo::RGK_ScalableVector:
321 return TypeSize::getScalable(MinimumSize: 0);
322 }
323 llvm_unreachable("Unsupported register kind");
324}
325
326unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
327 return 32;
328}
329
330unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
331 if (Opcode == Instruction::Load || Opcode == Instruction::Store)
332 return 32 * 4 / ElemWidth;
333 // For a given width return the max 0number of elements that can be combined
334 // into a wider bit value:
335 return (ElemWidth == 8 && ST->has16BitInsts()) ? 4
336 : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
337 : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2
338 : (ElemWidth == 64 &&
339 (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()))
340 ? 2
341 : 1;
342}
343
344bool GCNTTIImpl::preferSLPInstCountCheck() const {
345 // The integer inst-count heuristic causes regressions on gfx94x and gfx950
346 // because 2-element vector trees that pass the scalar/vector instruction
347 // count comparison still widen scalar moves (e.g. v_mov_b32 to v_mov_b64)
348 // after codegen, increasing register pressure and throughput cost without
349 // reducing the total instruction count.
350 return !ST->hasGFX940Insts() && !ST->hasGFX950Insts();
351}
352
353unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
354 unsigned ChainSizeInBytes,
355 VectorType *VecTy) const {
356 unsigned VecRegBitWidth = VF * LoadSize;
357 if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
358 // TODO: Support element-size less than 32bit?
359 return 128 / LoadSize;
360
361 return VF;
362}
363
364unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
365 unsigned ChainSizeInBytes,
366 VectorType *VecTy) const {
367 unsigned VecRegBitWidth = VF * StoreSize;
368 if (VecRegBitWidth > 128)
369 return 128 / StoreSize;
370
371 return VF;
372}
373
374unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
375 if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
376 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
377 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
378 AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
379 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
380 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
381 return 512;
382 }
383
384 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
385 return 8 * ST->getMaxPrivateElementSize();
386
387 // Common to flat, global, local and region. Assume for unknown addrspace.
388 return 128;
389}
390
391bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
392 Align Alignment,
393 unsigned AddrSpace) const {
394 // We allow vectorization of flat stores, even though we may need to decompose
395 // them later if they may access private memory. We don't have enough context
396 // here, and legalization can handle it.
397 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
398 return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&
399 ChainSizeInBytes <= ST->getMaxPrivateElementSize();
400 }
401 return true;
402}
403
404bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
405 Align Alignment,
406 unsigned AddrSpace) const {
407 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
408}
409
410bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
411 Align Alignment,
412 unsigned AddrSpace) const {
413 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
414}
415
416uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
417 return 1024;
418}
419
420Type *GCNTTIImpl::getMemcpyLoopLoweringType(
421 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
422 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
423 std::optional<uint32_t> AtomicElementSize) const {
424
425 if (AtomicElementSize)
426 return Type::getIntNTy(C&: Context, N: *AtomicElementSize * 8);
427
428 // 16-byte accesses achieve the highest copy throughput.
429 // If the operation has a fixed known length that is large enough, it is
430 // worthwhile to return an even wider type and let legalization lower it into
431 // multiple accesses, effectively unrolling the memcpy loop.
432 // We also rely on legalization to decompose into smaller accesses for
433 // subtargets and address spaces where it is necessary.
434 //
435 // Don't unroll if Length is not a constant, since unrolling leads to worse
436 // performance for length values that are smaller or slightly larger than the
437 // total size of the type returned here. Mitigating that would require a more
438 // complex lowering for variable-length memcpy and memmove.
439 unsigned I32EltsInVector = 4;
440 if (MemcpyLoopUnroll > 0 && isa<ConstantInt>(Val: Length))
441 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context),
442 NumElts: MemcpyLoopUnroll * I32EltsInVector);
443
444 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: I32EltsInVector);
445}
446
447void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
448 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
449 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
450 Align SrcAlign, Align DestAlign,
451 std::optional<uint32_t> AtomicCpySize) const {
452
453 if (AtomicCpySize)
454 BaseT::getMemcpyLoopResidualLoweringType(
455 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
456 DestAlign, AtomicCpySize);
457
458 Type *I32x4Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: 4);
459 while (RemainingBytes >= 16) {
460 OpsOut.push_back(Elt: I32x4Ty);
461 RemainingBytes -= 16;
462 }
463
464 Type *I64Ty = Type::getInt64Ty(C&: Context);
465 while (RemainingBytes >= 8) {
466 OpsOut.push_back(Elt: I64Ty);
467 RemainingBytes -= 8;
468 }
469
470 Type *I32Ty = Type::getInt32Ty(C&: Context);
471 while (RemainingBytes >= 4) {
472 OpsOut.push_back(Elt: I32Ty);
473 RemainingBytes -= 4;
474 }
475
476 Type *I16Ty = Type::getInt16Ty(C&: Context);
477 while (RemainingBytes >= 2) {
478 OpsOut.push_back(Elt: I16Ty);
479 RemainingBytes -= 2;
480 }
481
482 Type *I8Ty = Type::getInt8Ty(C&: Context);
483 while (RemainingBytes) {
484 OpsOut.push_back(Elt: I8Ty);
485 --RemainingBytes;
486 }
487}
488
489unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF,
490 bool HasUnorderedReductions) const {
491 // Disable unrolling if the loop is not vectorized.
492 // TODO: Enable this again.
493 if (VF.isScalar())
494 return 1;
495
496 return 8;
497}
498
499bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
500 MemIntrinsicInfo &Info) const {
501 switch (Inst->getIntrinsicID()) {
502 case Intrinsic::amdgcn_ds_ordered_add:
503 case Intrinsic::amdgcn_ds_ordered_swap: {
504 auto *Ordering = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 2));
505 auto *Volatile = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 4));
506 if (!Ordering || !Volatile)
507 return false; // Invalid.
508
509 unsigned OrderingVal = Ordering->getZExtValue();
510 if (OrderingVal > static_cast<unsigned>(AtomicOrdering::SequentiallyConsistent))
511 return false;
512
513 Info.PtrVal = Inst->getArgOperand(i: 0);
514 Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
515 Info.ReadMem = true;
516 Info.WriteMem = true;
517 Info.IsVolatile = !Volatile->isZero();
518 return true;
519 }
520 default:
521 return false;
522 }
523}
524
525/// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
526/// expected to fuse during instruction selection. \p Ty is the type the fused
527/// operation runs on.
528static bool canFuseFMulWithFAddSub(const SITargetLowering &TLI, Type *Ty,
529 const Instruction *FMul,
530 const Instruction *FAddSub) {
531 assert((FAddSub->getOpcode() == Instruction::FAdd ||
532 FAddSub->getOpcode() == Instruction::FSub) &&
533 "Expected an fadd or an fsub");
534
535 // The mad forms fuse exactly without fast-math flags but flush denormals.
536 // An fma forms only when it is not slower than the separate operations.
537 const Function &F = *FAddSub->getFunction();
538 const bool HasFMAD = TLI.isFMADLegal(F, Ty);
539 const bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(F, Ty);
540 if (!HasFMAD && !HasFMA)
541 return false;
542
543 // Without a mad the pair fuses only when both carry contract.
544 return HasFMAD || (FAddSub->hasAllowContract() && FMul->hasAllowContract());
545}
546
547InstructionCost GCNTTIImpl::getArithmeticInstrCost(
548 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
549 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
550 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
551
552 // Legalize the type.
553 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
554 int ISD = TLI->InstructionOpcodeToISD(Opcode);
555
556 // Because we don't have any legal vector operations, but the legal types, we
557 // need to account for split vectors.
558 unsigned NElts = LT.second.isVector() ?
559 LT.second.getVectorNumElements() : 1;
560
561 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
562
563 switch (ISD) {
564 case ISD::SHL:
565 case ISD::SRL:
566 case ISD::SRA:
567 if (SLT == MVT::i64)
568 return get64BitInstrCost(CostKind) * LT.first * NElts;
569
570 if (ST->has16BitInsts() && SLT == MVT::i16)
571 NElts = (NElts + 1) / 2;
572
573 // i32
574 return getFullRateInstrCost() * LT.first * NElts;
575 case ISD::ADD:
576 case ISD::SUB:
577 if (SLT == MVT::i64 && ST->hasAnyPackedU64Ops())
578 NElts = (NElts + 1) / 2;
579 [[fallthrough]];
580 case ISD::AND:
581 case ISD::OR:
582 case ISD::XOR:
583 if (SLT == MVT::i64) {
584 // and, or and xor are typically split into 2 VALU instructions.
585 return 2 * getFullRateInstrCost() * LT.first * NElts;
586 }
587
588 if (ST->has16BitInsts() && SLT == MVT::i16)
589 NElts = (NElts + 1) / 2;
590
591 return LT.first * NElts * getFullRateInstrCost();
592 case ISD::MUL: {
593 const int QuarterRateCost = getQuarterRateInstrCost(CostKind);
594 if (SLT == MVT::i64) {
595 const int FullRateCost = getFullRateInstrCost();
596 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
597 }
598
599 if (ST->has16BitInsts() && SLT == MVT::i16)
600 NElts = (NElts + 1) / 2;
601
602 // i32
603 return QuarterRateCost * NElts * LT.first;
604 }
605 case ISD::FMUL:
606 // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
607 // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
608 // fused operation.
609 if (CxtI && CxtI->hasOneUse()) {
610 const auto *FAddSub = dyn_cast<BinaryOperator>(Val: *CxtI->user_begin());
611 if (FAddSub &&
612 (FAddSub->getOpcode() == Instruction::FAdd ||
613 FAddSub->getOpcode() == Instruction::FSub) &&
614 canFuseFMulWithFAddSub(TLI: *TLI, Ty, FMul: CxtI, FAddSub))
615 return TargetTransformInfo::TCC_Free;
616 }
617 [[fallthrough]];
618 case ISD::FADD:
619 case ISD::FSUB:
620 if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32)
621 NElts = (NElts + 1) / 2;
622 if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)
623 NElts = (NElts + 1) / 2;
624 if (SLT == MVT::f64) {
625 if (ST->hasAnyPackedFP64Ops())
626 NElts = (NElts + 1) / 2;
627 return LT.first * NElts * get64BitInstrCost(CostKind);
628 }
629
630 if (ST->has16BitInsts() && SLT == MVT::f16)
631 NElts = (NElts + 1) / 2;
632
633 if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16)
634 return LT.first * NElts * getFullRateInstrCost();
635 break;
636 case ISD::FDIV:
637 case ISD::FREM:
638 // FIXME: frem should be handled separately. The fdiv in it is most of it,
639 // but the current lowering is also not entirely correct.
640 if (SLT == MVT::f64) {
641 int Cost = 7 * get64BitInstrCost(CostKind) +
642 getQuarterRateInstrCost(CostKind) +
643 3 * getHalfRateInstrCost(CostKind);
644 // Add cost of workaround.
645 if (!ST->hasUsableDivScaleConditionOutput())
646 Cost += 3 * getFullRateInstrCost();
647
648 return LT.first * Cost * NElts;
649 }
650
651 if (!Args.empty() && match(V: Args[0], P: PatternMatch::m_FPOne())) {
652 // TODO: This is more complicated, unsafe flags etc.
653 if ((SLT == MVT::f32 && !HasFP32Denormals) ||
654 (SLT == MVT::f16 && ST->has16BitInsts())) {
655 return LT.first * getTransInstrCost(CostKind) * NElts;
656 }
657 }
658
659 if (SLT == MVT::f16 && ST->has16BitInsts()) {
660 // 2 x v_cvt_f32_f16
661 // f32 rcp
662 // f32 fmul
663 // v_cvt_f16_f32
664 // f16 div_fixup
665 int Cost = 4 * getFullRateInstrCost() + 2 * getTransInstrCost(CostKind);
666 return LT.first * Cost * NElts;
667 }
668
669 if (SLT == MVT::f32 && (CxtI && CxtI->hasApproxFunc())) {
670 // Fast unsafe fdiv lowering:
671 // f32 rcp
672 // f32 fmul
673 int Cost = getTransInstrCost(CostKind) + getFullRateInstrCost();
674 return LT.first * Cost * NElts;
675 }
676
677 if (SLT == MVT::f32 || SLT == MVT::f16) {
678 // 4 more v_cvt_* insts without f16 insts support
679 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
680 1 * getTransInstrCost(CostKind);
681
682 if (!HasFP32Denormals) {
683 // FP mode switches.
684 Cost += 2 * getFullRateInstrCost();
685 }
686
687 return LT.first * NElts * Cost;
688 }
689 break;
690 case ISD::FNEG:
691 // Use the backend' estimation. If fneg is not free each element will cost
692 // one additional instruction.
693 return TLI->isFNegFree(VT: SLT) ? 0 : NElts;
694 default:
695 break;
696 }
697
698 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
699 Args, CxtI);
700}
701
702// Return true if there's a potential benefit from using v2f16/v2i16
703// instructions for an intrinsic, even if it requires nontrivial legalization.
704static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
705 switch (ID) {
706 case Intrinsic::fma:
707 case Intrinsic::fmuladd:
708 case Intrinsic::copysign:
709 case Intrinsic::minimumnum:
710 case Intrinsic::maximumnum:
711 case Intrinsic::canonicalize:
712 // There's a small benefit to using vector ops in the legalized code.
713 case Intrinsic::round:
714 case Intrinsic::uadd_sat:
715 case Intrinsic::usub_sat:
716 case Intrinsic::sadd_sat:
717 case Intrinsic::ssub_sat:
718 case Intrinsic::abs:
719 return true;
720 default:
721 return false;
722 }
723}
724
725InstructionCost
726GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
727 TTI::TargetCostKind CostKind) const {
728 switch (ICA.getID()) {
729 case Intrinsic::fabs:
730 // Free source modifier in the common case.
731 return 0;
732 case Intrinsic::amdgcn_workitem_id_x:
733 case Intrinsic::amdgcn_workitem_id_y:
734 case Intrinsic::amdgcn_workitem_id_z:
735 // TODO: If hasPackedTID, or if the calling context is not an entry point
736 // there may be a bit instruction.
737 return 0;
738 case Intrinsic::amdgcn_workgroup_id_x:
739 case Intrinsic::amdgcn_workgroup_id_y:
740 case Intrinsic::amdgcn_workgroup_id_z:
741 case Intrinsic::amdgcn_lds_kernel_id:
742 case Intrinsic::amdgcn_dispatch_ptr:
743 case Intrinsic::amdgcn_dispatch_id:
744 case Intrinsic::amdgcn_implicitarg_ptr:
745 case Intrinsic::amdgcn_queue_ptr:
746 // Read from an argument register.
747 return 0;
748 default:
749 break;
750 }
751
752 Type *RetTy = ICA.getReturnType();
753
754 Intrinsic::ID IID = ICA.getID();
755 switch (IID) {
756 case Intrinsic::exp:
757 case Intrinsic::exp2:
758 case Intrinsic::exp10: {
759 // Legalize the type.
760 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
761 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
762 unsigned NElts =
763 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
764
765 if (SLT == MVT::f64) {
766 unsigned NumOps = 20;
767 if (IID == Intrinsic::exp)
768 ++NumOps;
769 else if (IID == Intrinsic::exp10)
770 NumOps += 3;
771
772 return LT.first * NElts * NumOps * get64BitInstrCost(CostKind);
773 }
774
775 if (SLT == MVT::f32) {
776 unsigned NumFullRateOps = 0;
777 // v_exp_f32 (transcendental).
778 unsigned NumTransOps = 1;
779
780 if (!ICA.getFlags().approxFunc() && IID != Intrinsic::exp2) {
781 // Non-AFN exp/exp10: range reduction + v_exp_f32 + ldexp +
782 // overflow/underflow checks (lowerFEXP). Denorm is also handled.
783 // FMA preamble: ~13 full-rate ops; non-FMA: ~17.
784 NumFullRateOps = ST->hasFastFMAF32() ? 13 : 17;
785 } else {
786 if (IID == Intrinsic::exp) {
787 // lowerFEXPUnsafe: fmul (base conversion) + v_exp_f32.
788 NumFullRateOps = 1;
789 } else if (IID == Intrinsic::exp10) {
790 // lowerFEXP10Unsafe: 3 fmul + 2 v_exp_f32 (double-exp2).
791 NumFullRateOps = 3;
792 NumTransOps = 2;
793 }
794 // Denorm scaling adds setcc + select + fadd + select + fmul.
795 if (HasFP32Denormals)
796 NumFullRateOps += 5;
797 }
798
799 InstructionCost Cost = NumFullRateOps * getFullRateInstrCost() +
800 NumTransOps * getTransInstrCost(CostKind);
801 return LT.first * NElts * Cost;
802 }
803
804 break;
805 }
806 case Intrinsic::log:
807 case Intrinsic::log2:
808 case Intrinsic::log10: {
809 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
810 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
811 unsigned NElts =
812 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
813
814 if (SLT == MVT::f32) {
815 unsigned NumFullRateOps = 0;
816
817 if (IID == Intrinsic::log2) {
818 // LowerFLOG2: just v_log_f32.
819 } else if (ICA.getFlags().approxFunc()) {
820 // LowerFLOGUnsafe: v_log_f32 + fmul (base conversion).
821 NumFullRateOps = 1;
822 } else {
823 // LowerFLOGCommon non-AFN: v_log_f32 + extended-precision
824 // multiply + finite check.
825 NumFullRateOps = ST->hasFastFMAF32() ? 8 : 11;
826 }
827
828 if (HasFP32Denormals)
829 NumFullRateOps += 5;
830
831 InstructionCost Cost =
832 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
833 return LT.first * NElts * Cost;
834 }
835
836 break;
837 }
838 case Intrinsic::sin:
839 case Intrinsic::cos: {
840 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
841 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
842 unsigned NElts =
843 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
844
845 if (SLT == MVT::f32) {
846 // LowerTrig: fmul(1/2pi) + v_sin/v_cos.
847 unsigned NumFullRateOps = ST->hasTrigReducedRange() ? 2 : 1;
848
849 InstructionCost Cost =
850 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
851 return LT.first * NElts * Cost;
852 }
853
854 break;
855 }
856 case Intrinsic::sqrt: {
857 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
858 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
859 unsigned NElts =
860 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
861
862 if (SLT == MVT::f32) {
863 unsigned NumFullRateOps = 0;
864
865 if (!ICA.getFlags().approxFunc()) {
866 // lowerFSQRTF32 non-AFN: v_sqrt_f32 + refinement + scale fixup.
867 NumFullRateOps = HasFP32Denormals ? 17 : 16;
868 }
869
870 InstructionCost Cost =
871 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
872 return LT.first * NElts * Cost;
873 }
874
875 break;
876 }
877 default:
878 break;
879 }
880
881 if (!intrinsicHasPackedVectorBenefit(ID: ICA.getID()))
882 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
883
884 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
885 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
886 unsigned NElts = LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
887
888 if ((ST->hasVOP3PInsts() &&
889 (SLT == MVT::f16 || SLT == MVT::i16 ||
890 (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) ||
891 (ST->hasAnyPackedFP64Ops() && SLT == MVT::f64) ||
892 (ST->hasAnyPackedU64Ops() && SLT == MVT::i64)) {
893 NElts = (NElts + 1) / 2;
894 } else if (SLT == MVT::f32) {
895 bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() &&
896 IID != Intrinsic::minimumnum &&
897 IID != Intrinsic::maximumnum;
898 NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts;
899 }
900
901 // TODO: Get more refined intrinsic costs?
902 unsigned InstRate = getQuarterRateInstrCost(CostKind);
903
904 switch (ICA.getID()) {
905 case Intrinsic::fma:
906 case Intrinsic::fmuladd:
907 if (SLT == MVT::f64) {
908 InstRate = get64BitInstrCost(CostKind);
909 break;
910 }
911
912 if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16)
913 InstRate = getFullRateInstrCost();
914 else {
915 InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)
916 : getQuarterRateInstrCost(CostKind);
917 }
918 break;
919 case Intrinsic::copysign:
920 return NElts * getFullRateInstrCost();
921 case Intrinsic::minimumnum:
922 case Intrinsic::maximumnum: {
923 // Instruction + 2 canonicalizes. For cases that need type promotion, we the
924 // promotion takes the place of the canonicalize.
925 unsigned NumOps = 3;
926 if (const IntrinsicInst *II = ICA.getInst()) {
927 // Directly legal with ieee=0
928 // TODO: Not directly legal with strictfp
929 if (fpenvIEEEMode(I: *II) == KnownIEEEMode::Off)
930 NumOps = 1;
931 }
932
933 unsigned BaseRate =
934 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
935 InstRate = BaseRate * NumOps;
936 break;
937 }
938 case Intrinsic::canonicalize: {
939 InstRate =
940 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
941 break;
942 }
943 case Intrinsic::uadd_sat:
944 case Intrinsic::usub_sat:
945 case Intrinsic::sadd_sat:
946 case Intrinsic::ssub_sat: {
947 if (SLT == MVT::i16 || SLT == MVT::i32)
948 InstRate = getFullRateInstrCost();
949
950 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
951 if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second)))
952 NElts = 1;
953 break;
954 }
955 case Intrinsic::abs:
956 // Expansion takes 2 instructions for VALU
957 if (SLT == MVT::i16 || SLT == MVT::i32)
958 InstRate = 2 * getFullRateInstrCost();
959 break;
960 default:
961 break;
962 }
963
964 return LT.first * NElts * InstRate;
965}
966
967InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
968 TTI::TargetCostKind CostKind,
969 const Instruction *I) const {
970 assert((I == nullptr || I->getOpcode() == Opcode) &&
971 "Opcode should reflect passed instruction.");
972 const bool SCost =
973 (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);
974 const int CBrCost = SCost ? 5 : 7;
975 switch (Opcode) {
976 case Instruction::UncondBr:
977 // Branch instruction takes about 4 slots on gfx900.
978 return SCost ? 1 : 4;
979 case Instruction::CondBr:
980 // Suppose conditional branch takes additional 3 exec manipulations
981 // instructions in average.
982 return CBrCost;
983 case Instruction::Switch: {
984 const auto *SI = dyn_cast_or_null<SwitchInst>(Val: I);
985 // Each case (including default) takes 1 cmp + 1 cbr instructions in
986 // average.
987 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
988 }
989 case Instruction::Ret:
990 return SCost ? 1 : 10;
991 }
992 return BaseT::getCFInstrCost(Opcode, CostKind, I);
993}
994
995// Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with
996// true16; unpacking is 2.6 to 2.9.
997static constexpr unsigned MaskPackCostPerElt = 4;
998static constexpr unsigned MaskUnpackCostPerElt = 3;
999
1000static std::optional<unsigned> getNumberOfPackedMaskElts(Type *Ty) {
1001 auto *FVT = dyn_cast<FixedVectorType>(Val: Ty);
1002 if (FVT && FVT->getElementType()->isIntegerTy(BitWidth: 1) && FVT->getNumElements() > 1)
1003 return FVT->getNumElements();
1004 return std::nullopt;
1005}
1006
1007InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1008 Type *Src,
1009 TTI::CastContextHint CCH,
1010 TTI::TargetCostKind CostKind,
1011 const Instruction *I) const {
1012 // A bitcast between a vector of i1 and an integer packs or unpacks a mask.
1013 if (Opcode == Instruction::BitCast) {
1014 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Src);
1015 Elts && Dst->isIntegerTy(BitWidth: *Elts))
1016 return InstructionCost(MaskPackCostPerElt) * *Elts *
1017 getFullRateInstrCost();
1018 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Dst);
1019 Elts && Src->isIntegerTy(BitWidth: *Elts))
1020 return InstructionCost(MaskUnpackCostPerElt) * *Elts *
1021 getFullRateInstrCost();
1022 }
1023
1024 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1025}
1026
1027InstructionCost
1028GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
1029 std::optional<FastMathFlags> FMF,
1030 TTI::TargetCostKind CostKind) const {
1031 if (TTI::requiresOrderedReduction(FMF))
1032 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1033
1034 // An add or xor reduction over a vector of i1 becomes a bit count over the
1035 // packed mask; the generic model prices a shuffle tree and misses that.
1036 if (Opcode == Instruction::Add || Opcode == Instruction::Xor) {
1037 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty))
1038 return InstructionCost(MaskPackCostPerElt) * *Elts *
1039 getFullRateInstrCost();
1040 }
1041
1042 EVT OrigTy = TLI->getValueType(DL, Ty);
1043
1044 // Computes cost on targets that have packed math instructions(which support
1045 // 16-bit types only).
1046 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1047 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1048
1049 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1050 return LT.first * getFullRateInstrCost();
1051}
1052
1053InstructionCost
1054GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1055 FastMathFlags FMF,
1056 TTI::TargetCostKind CostKind) const {
1057 EVT OrigTy = TLI->getValueType(DL, Ty);
1058
1059 // Computes cost on targets that have packed math instructions(which support
1060 // 16-bit types only).
1061 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1062 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1063
1064 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1065 return LT.first * getHalfRateInstrCost(CostKind);
1066}
1067
1068InstructionCost GCNTTIImpl::getVectorInstrCost(
1069 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
1070 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
1071 switch (Opcode) {
1072 case Instruction::ExtractElement:
1073 case Instruction::InsertElement: {
1074 unsigned EltSize
1075 = DL.getTypeSizeInBits(Ty: cast<VectorType>(Val: ValTy)->getElementType());
1076 // Dynamic indexing isn't free and is best avoided.
1077 if (Index == ~0u)
1078 return 2;
1079 if (EltSize < 32) {
1080 if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
1081 return 0;
1082 // Inserts of booleans are free.
1083 // TODO: Extracts are free too.
1084 if (EltSize == 1 && Opcode == Instruction::InsertElement)
1085 return TargetTransformInfo::TCC_Free;
1086 // Extract element sequences of consecutive i8 values that match a
1087 // register size are free most likely. It is not possible to know
1088 // if this extract is part of a consecutive sequence so this may
1089 // apply more generally.
1090 if (Opcode == Instruction::ExtractElement && EltSize == 8) {
1091 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: ValTy)) {
1092 unsigned NumElts = FVTy->getNumElements();
1093 if (NumElts >= 4 && isPowerOf2_32(Value: NumElts))
1094 return 0;
1095 }
1096 }
1097 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1098 VIC);
1099 }
1100
1101 // Extracts are just reads of a subregister, so are free. Inserts are
1102 // considered free because we don't want to have any cost for scalarizing
1103 // operations, and we don't have to copy into a different register class.
1104 return 0;
1105 }
1106 default:
1107 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1108 VIC);
1109 }
1110}
1111
1112/// Analyze if the results of inline asm are divergent. If \p Indices is empty,
1113/// this is analyzing the collective result of all output registers. Otherwise,
1114/// this is only querying a specific result index if this returns multiple
1115/// registers in a struct.
1116bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
1117 const CallInst *CI, ArrayRef<unsigned> Indices) const {
1118 // TODO: Handle complex extract indices
1119 if (Indices.size() > 1)
1120 return true;
1121
1122 const DataLayout &DL = CI->getDataLayout();
1123 const SIRegisterInfo *TRI = ST->getRegisterInfo();
1124 TargetLowering::AsmOperandInfoVector TargetConstraints =
1125 TLI->ParseConstraints(DL, TRI: ST->getRegisterInfo(), Call: *CI);
1126
1127 const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];
1128
1129 int OutputIdx = 0;
1130 for (auto &TC : TargetConstraints) {
1131 if (TC.Type != InlineAsm::isOutput)
1132 continue;
1133
1134 // Skip outputs we don't care about.
1135 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
1136 continue;
1137
1138 TLI->ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
1139
1140 const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(
1141 TRI, Constraint: TC.ConstraintCode, VT: TC.ConstraintVT).second;
1142
1143 // For AGPR constraints null is returned on subtargets without AGPRs, so
1144 // assume divergent for null.
1145 if (!RC || !TRI->isSGPRClass(RC))
1146 return true;
1147 }
1148
1149 return false;
1150}
1151
1152bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
1153 const IntrinsicInst *ReadReg) const {
1154 Metadata *MD =
1155 cast<MetadataAsValue>(Val: ReadReg->getArgOperand(i: 0))->getMetadata();
1156 StringRef RegName =
1157 cast<MDString>(Val: cast<MDNode>(Val: MD)->getOperand(I: 0))->getString();
1158
1159 // Special case registers that look like VCC.
1160 MVT VT = MVT::getVT(Ty: ReadReg->getType());
1161 if (VT == MVT::i1)
1162 return true;
1163
1164 // Special case scalar registers that start with 'v'.
1165 if (RegName.starts_with(Prefix: "vcc") || RegName.empty())
1166 return false;
1167
1168 // VGPR or AGPR is divergent. There aren't any specially named vector
1169 // registers.
1170 return RegName[0] == 'v' || RegName[0] == 'a';
1171}
1172
1173/// \returns true if the result of the value could potentially be
1174/// different across workitems in a wavefront.
1175bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
1176 if (const Argument *A = dyn_cast<Argument>(Val: V))
1177 return !AMDGPU::isArgPassedInSGPR(Arg: A);
1178
1179 // Loads from the private and flat address spaces are divergent, because
1180 // threads can execute the load instruction with the same inputs and get
1181 // different results.
1182 //
1183 // All other loads are not divergent, because if threads issue loads with the
1184 // same arguments, they will always get the same result.
1185 if (const LoadInst *Load = dyn_cast<LoadInst>(Val: V))
1186 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
1187 Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
1188
1189 // Atomics are divergent because they are executed sequentially: when an
1190 // atomic operation refers to the same address in each thread, then each
1191 // thread after the first sees the value written by the previous thread as
1192 // original value.
1193 if (isa<AtomicRMWInst, AtomicCmpXchgInst>(Val: V))
1194 return true;
1195
1196 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
1197 Intrinsic::ID IID = Intrinsic->getIntrinsicID();
1198 switch (IID) {
1199 case Intrinsic::read_register:
1200 return isReadRegisterSourceOfDivergence(ReadReg: Intrinsic);
1201 case Intrinsic::amdgcn_workitem_id_y:
1202 case Intrinsic::amdgcn_workitem_id_z: {
1203 const Function *F = Intrinsic->getFunction();
1204 bool HasUniformYZ =
1205 ST->hasWavefrontsEvenlySplittingXDim(F: *F, /*RequitezUniformYZ=*/REquiresUniformYZ: true);
1206 std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize(
1207 F: *F, Dim: IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2);
1208 return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1);
1209 }
1210 default:
1211 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID);
1212 }
1213 }
1214
1215 // Assume all function calls are a source of divergence.
1216 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
1217 if (CI->isInlineAsm())
1218 return isInlineAsmSourceOfDivergence(CI);
1219 return true;
1220 }
1221
1222 // Assume all function calls are a source of divergence.
1223 if (isa<InvokeInst>(Val: V))
1224 return true;
1225
1226 // If the target supports globally addressable scratch, the mapping from
1227 // scratch memory to the flat aperture changes therefore an address space cast
1228 // is no longer uniform.
1229 if (auto *CastI = dyn_cast<AddrSpaceCastInst>(Val: V)) {
1230 return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
1231 CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
1232 ST->hasGloballyAddressableScratch();
1233 }
1234
1235 return false;
1236}
1237
1238bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
1239 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V))
1240 return AMDGPU::isIntrinsicAlwaysUniform(IntrID: Intrinsic->getIntrinsicID());
1241
1242 if (const CallInst *CI = dyn_cast<CallInst>(Val: V)) {
1243 if (CI->isInlineAsm())
1244 return !isInlineAsmSourceOfDivergence(CI);
1245 return false;
1246 }
1247
1248 // In most cases TID / wavefrontsize is uniform.
1249 //
1250 // However, if a kernel has uneven dimesions we can have a value of
1251 // workitem-id-x divided by the wavefrontsize non-uniform. For example
1252 // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)
1253 // packed into a same wave which gives 1 and 0 after the division by 64
1254 // respectively.
1255 //
1256 // The X dimension doesn't reset within a wave if either both the Y
1257 // and Z dimensions are of length 1, or if the X dimension's required
1258 // size is a power of 2. Note, however, if the X dimension's maximum
1259 // size is a power of 2 < the wavefront size, division by the wavefront
1260 // size is guaranteed to yield 0, so this is also a no-reset case.
1261 bool XDimDoesntResetWithinWaves = false;
1262 if (auto *I = dyn_cast<Instruction>(Val: V)) {
1263 const Function *F = I->getFunction();
1264 XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(F: *F);
1265 }
1266 using namespace llvm::PatternMatch;
1267 uint64_t C;
1268 if (match(V, P: m_LShr(L: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
1269 R: m_ConstantInt(V&: C))) ||
1270 match(V, P: m_AShr(L: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
1271 R: m_ConstantInt(V&: C)))) {
1272 return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves;
1273 }
1274
1275 Value *Mask;
1276 if (match(V, P: m_c_And(L: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>(),
1277 R: m_Value(V&: Mask)))) {
1278 return computeKnownBits(V: Mask, DL).countMinTrailingZeros() >=
1279 ST->getWavefrontSizeLog2() &&
1280 XDimDoesntResetWithinWaves;
1281 }
1282
1283 const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(Val: V);
1284 if (!ExtValue)
1285 return false;
1286
1287 const CallInst *CI = dyn_cast<CallInst>(Val: ExtValue->getOperand(i_nocapture: 0));
1288 if (!CI)
1289 return false;
1290
1291 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: CI)) {
1292 switch (Intrinsic->getIntrinsicID()) {
1293 default:
1294 return false;
1295 case Intrinsic::amdgcn_if:
1296 case Intrinsic::amdgcn_else: {
1297 ArrayRef<unsigned> Indices = ExtValue->getIndices();
1298 return Indices.size() == 1 && Indices[0] == 1;
1299 }
1300 }
1301 }
1302
1303 // If we have inline asm returning mixed SGPR and VGPR results, we inferred
1304 // divergent for the overall struct return. We need to override it in the
1305 // case we're extracting an SGPR component here.
1306 if (CI->isInlineAsm())
1307 return !isInlineAsmSourceOfDivergence(CI, Indices: ExtValue->getIndices());
1308
1309 return false;
1310}
1311
1312bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
1313 Intrinsic::ID IID) const {
1314 switch (IID) {
1315 case Intrinsic::amdgcn_is_shared:
1316 case Intrinsic::amdgcn_is_private:
1317 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1318 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1319 case Intrinsic::amdgcn_load_to_lds:
1320 case Intrinsic::amdgcn_make_buffer_rsrc:
1321 OpIndexes.push_back(Elt: 0);
1322 return true;
1323 default:
1324 return false;
1325 }
1326}
1327
1328Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
1329 Value *OldV,
1330 Value *NewV) const {
1331 auto IntrID = II->getIntrinsicID();
1332 switch (IntrID) {
1333 case Intrinsic::amdgcn_is_shared:
1334 case Intrinsic::amdgcn_is_private: {
1335 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
1336 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;
1337 unsigned NewAS = NewV->getType()->getPointerAddressSpace();
1338 LLVMContext &Ctx = NewV->getType()->getContext();
1339 ConstantInt *NewVal = (TrueAS == NewAS) ?
1340 ConstantInt::getTrue(Context&: Ctx) : ConstantInt::getFalse(Context&: Ctx);
1341 return NewVal;
1342 }
1343 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1344 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
1345 Type *DestTy = II->getType();
1346 Type *SrcTy = NewV->getType();
1347 unsigned NewAS = SrcTy->getPointerAddressSpace();
1348 if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS))
1349 return nullptr;
1350 Module *M = II->getModule();
1351 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1352 M, id: II->getIntrinsicID(), OverloadTys: {DestTy, SrcTy, DestTy});
1353 II->setArgOperand(i: 0, v: NewV);
1354 II->setCalledFunction(NewDecl);
1355 return II;
1356 }
1357 case Intrinsic::amdgcn_load_to_lds: {
1358 Type *SrcTy = NewV->getType();
1359 Module *M = II->getModule();
1360 Function *NewDecl =
1361 Intrinsic::getOrInsertDeclaration(M, id: II->getIntrinsicID(), OverloadTys: {SrcTy});
1362 II->setArgOperand(i: 0, v: NewV);
1363 II->setCalledFunction(NewDecl);
1364 return II;
1365 }
1366 case Intrinsic::amdgcn_make_buffer_rsrc: {
1367 Type *SrcTy = NewV->getType();
1368 Type *DstTy = II->getType();
1369 Type *NumRecordsTy = II->getArgOperand(i: 2)->getType();
1370 Module *M = II->getModule();
1371 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1372 M, id: II->getIntrinsicID(), OverloadTys: {DstTy, SrcTy, NumRecordsTy});
1373 II->setArgOperand(i: 0, v: NewV);
1374 II->setCalledFunction(NewDecl);
1375 return II;
1376 }
1377 default:
1378 return nullptr;
1379 }
1380}
1381
1382InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
1383 VectorType *DstTy, VectorType *SrcTy,
1384 TTI::TargetCostKind CostKind,
1385 ArrayRef<int> Mask, int Index,
1386 VectorType *SubTp,
1387 ArrayRef<const Value *> Args,
1388 const Instruction *CxtI) const {
1389 if (!isa<FixedVectorType>(Val: SrcTy))
1390 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1391 SubTp);
1392
1393 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1394
1395 unsigned ScalarSize = DL.getTypeSizeInBits(Ty: SrcTy->getElementType());
1396 if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1397 (ScalarSize == 16 || ScalarSize == 8)) {
1398 // Larger vector widths may require additional instructions, but are
1399 // typically cheaper than scalarized versions.
1400 //
1401 // We assume that shuffling at a register granularity can be done for free.
1402 // This is not true for vectors fed into memory instructions, but it is
1403 // effectively true for all other shuffling. The emphasis of the logic here
1404 // is to assist generic transform in cleaning up / canonicalizing those
1405 // shuffles.
1406
1407 // With op_sel VOP3P instructions freely can access the low half or high
1408 // half of a register, so any swizzle of two elements is free.
1409 if (auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy)) {
1410 unsigned NumSrcElts = SrcVecTy->getNumElements();
1411 if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 &&
1412 (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse ||
1413 Kind == TTI::SK_PermuteSingleSrc))
1414 return 0;
1415 }
1416
1417 unsigned EltsPerReg = 32 / ScalarSize;
1418 switch (Kind) {
1419 case TTI::SK_Broadcast:
1420 // A single v_perm_b32 can be re-used for all destination registers.
1421 return 1;
1422 case TTI::SK_Reverse:
1423 // One instruction per register.
1424 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1425 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1426 return InstructionCost::getInvalid();
1427 case TTI::SK_ExtractSubvector:
1428 if (Index % EltsPerReg == 0)
1429 return 0; // Shuffling at register granularity
1430 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1431 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1432 return InstructionCost::getInvalid();
1433 case TTI::SK_InsertSubvector: {
1434 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1435 if (!DstVecTy)
1436 return InstructionCost::getInvalid();
1437 unsigned NumDstElts = DstVecTy->getNumElements();
1438 unsigned NumInsertElts = cast<FixedVectorType>(Val: SubTp)->getNumElements();
1439 unsigned EndIndex = Index + NumInsertElts;
1440 unsigned BeginSubIdx = Index % EltsPerReg;
1441 unsigned EndSubIdx = EndIndex % EltsPerReg;
1442 unsigned Cost = 0;
1443
1444 if (BeginSubIdx != 0) {
1445 // Need to shift the inserted vector into place. The cost is the number
1446 // of destination registers overlapped by the inserted vector.
1447 Cost = divideCeil(Numerator: EndIndex, Denominator: EltsPerReg) - (Index / EltsPerReg);
1448 }
1449
1450 // If the last register overlap is partial, there may be three source
1451 // registers feeding into it; that takes an extra instruction.
1452 if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx)
1453 Cost += 1;
1454
1455 return Cost;
1456 }
1457 case TTI::SK_Splice: {
1458 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1459 if (!DstVecTy)
1460 return InstructionCost::getInvalid();
1461 unsigned NumElts = DstVecTy->getNumElements();
1462 assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements());
1463 // Determine the sub-region of the result vector that requires
1464 // sub-register shuffles / mixing.
1465 unsigned EltsFromLHS = NumElts - Index;
1466 bool LHSIsAligned = (Index % EltsPerReg) == 0;
1467 bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0;
1468 if (LHSIsAligned && RHSIsAligned)
1469 return 0;
1470 if (LHSIsAligned && !RHSIsAligned)
1471 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg) - (EltsFromLHS / EltsPerReg);
1472 if (!LHSIsAligned && RHSIsAligned)
1473 return divideCeil(Numerator: EltsFromLHS, Denominator: EltsPerReg);
1474 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg);
1475 }
1476 default:
1477 break;
1478 }
1479
1480 if (!Mask.empty()) {
1481 unsigned NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements();
1482
1483 // Generically estimate the cost by assuming that each destination
1484 // register is derived from sources via v_perm_b32 instructions if it
1485 // can't be copied as-is.
1486 //
1487 // For each destination register, derive the cost of obtaining it based
1488 // on the number of source registers that feed into it.
1489 unsigned Cost = 0;
1490 for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) {
1491 SmallVector<int, 4> Regs;
1492 bool Aligned = true;
1493 for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) {
1494 int SrcIdx = Mask[DstIdx + I];
1495 if (SrcIdx == -1)
1496 continue;
1497 int Reg;
1498 if (SrcIdx < (int)NumSrcElts) {
1499 Reg = SrcIdx / EltsPerReg;
1500 if (SrcIdx % EltsPerReg != I)
1501 Aligned = false;
1502 } else {
1503 Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg;
1504 if ((SrcIdx - NumSrcElts) % EltsPerReg != I)
1505 Aligned = false;
1506 }
1507 if (!llvm::is_contained(Range&: Regs, Element: Reg))
1508 Regs.push_back(Elt: Reg);
1509 }
1510 if (Regs.size() >= 2)
1511 Cost += Regs.size() - 1;
1512 else if (!Aligned)
1513 Cost += 1;
1514 }
1515 return Cost;
1516 }
1517 }
1518
1519 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1520 SubTp);
1521}
1522
1523/// Whether it is profitable to sink the operands of an
1524/// Instruction I to the basic block of I.
1525/// This helps using several modifiers (like abs and neg) more often.
1526bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
1527 SmallVectorImpl<Use *> &Ops) const {
1528 using namespace PatternMatch;
1529
1530 // The cost model prices this fmul as free assuming it fuses with its
1531 // fadd/fsub user, which needs them in one block. Sink a stranded
1532 // loop-invariant fmul back to the user when they would fuse. Single use only,
1533 // so this stays a move.
1534 if (I->getOpcode() == Instruction::FAdd ||
1535 I->getOpcode() == Instruction::FSub) {
1536 for (Use &Op : I->operands()) {
1537 auto *FMul = dyn_cast<Instruction>(Val: Op.get());
1538 if (!FMul || FMul->getOpcode() != Instruction::FMul ||
1539 !FMul->hasOneUse() ||
1540 !canFuseFMulWithFAddSub(TLI: *TLI, Ty: I->getType(), FMul, FAddSub: I))
1541 continue;
1542 // The fused operand. Sink it when it sits in another block, then stop.
1543 if (FMul->getParent() != I->getParent())
1544 Ops.push_back(Elt: &Op);
1545 break;
1546 }
1547 }
1548
1549 for (auto &Op : I->operands()) {
1550 // Ensure we are not already sinking this operand.
1551 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op.get(); }))
1552 continue;
1553
1554 if (match(V: &Op, P: m_FAbs(Op0: m_Value())) || match(V: &Op, P: m_FNeg(X: m_Value()))) {
1555 Ops.push_back(Elt: &Op);
1556 continue;
1557 }
1558
1559 // Check for zero-cost multiple use InsertElement/ExtractElement
1560 // instructions
1561 if (Instruction *OpInst = dyn_cast<Instruction>(Val: Op.get())) {
1562 if (OpInst->getType()->isVectorTy() && OpInst->getNumOperands() > 1) {
1563 Instruction *VecOpInst = dyn_cast<Instruction>(Val: OpInst->getOperand(i: 0));
1564 if (VecOpInst && VecOpInst->hasOneUse())
1565 continue;
1566
1567 if (getVectorInstrCost(Opcode: OpInst->getOpcode(), ValTy: OpInst->getType(),
1568 CostKind: TTI::TCK_RecipThroughput, Index: 0,
1569 Op0: OpInst->getOperand(i: 0),
1570 Op1: OpInst->getOperand(i: 1)) == 0) {
1571 Ops.push_back(Elt: &Op);
1572 continue;
1573 }
1574 }
1575 }
1576
1577 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: Op.get())) {
1578
1579 unsigned EltSize = DL.getTypeSizeInBits(
1580 Ty: cast<VectorType>(Val: Shuffle->getType())->getElementType());
1581
1582 // For i32 (or greater) shufflevectors, these will be lowered into a
1583 // series of insert / extract elements, which will be coalesced away.
1584 if (EltSize < 16 || !ST->has16BitInsts())
1585 continue;
1586
1587 int NumSubElts, SubIndex;
1588 if (Shuffle->changesLength()) {
1589 if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) {
1590 Ops.push_back(Elt: &Op);
1591 continue;
1592 }
1593
1594 if ((Shuffle->isExtractSubvectorMask(Index&: SubIndex) ||
1595 Shuffle->isInsertSubvectorMask(NumSubElts, Index&: SubIndex)) &&
1596 !(SubIndex & 0x1)) {
1597 Ops.push_back(Elt: &Op);
1598 continue;
1599 }
1600 }
1601
1602 if (Shuffle->isReverse() || Shuffle->isZeroEltSplat() ||
1603 Shuffle->isSingleSource()) {
1604 Ops.push_back(Elt: &Op);
1605 continue;
1606 }
1607 }
1608 }
1609
1610 return !Ops.empty();
1611}
1612
1613bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
1614 const Function *Callee) const {
1615 const TargetMachine &TM = getTLI()->getTargetMachine();
1616 const GCNSubtarget *CallerST
1617 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));
1618 const GCNSubtarget *CalleeST
1619 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));
1620
1621 if (!BaseT::areInlineCompatible(Caller, Callee))
1622 return false;
1623
1624 // FIXME: dx10_clamp can just take the caller setting, but there seems to be
1625 // no way to support merge for backend defined attributes.
1626 SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
1627 SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
1628 if (!CallerMode.isInlineCompatible(CalleeMode))
1629 return false;
1630
1631 if (Callee->hasFnAttribute(Kind: Attribute::AlwaysInline) ||
1632 Callee->hasFnAttribute(Kind: Attribute::InlineHint))
1633 return true;
1634
1635 // Hack to make compile times reasonable.
1636 if (InlineMaxBB) {
1637 // Single BB does not increase total BB amount.
1638 if (Callee->size() == 1)
1639 return true;
1640 size_t BBSize = Caller->size() + Callee->size() - 1;
1641 if (BBSize > InlineMaxBB) {
1642 LLVM_DEBUG(dbgs() << "AMDGPU inline max-BB rejected inlining "
1643 << Callee->getName() << " into " << Caller->getName()
1644 << ": caller BBs=" << Caller->size() << ", callee BBs="
1645 << Callee->size() << ", combined BBs=" << BBSize
1646 << ", max BBs=" << InlineMaxBB << '\n');
1647 return false;
1648 }
1649 }
1650
1651 return true;
1652}
1653
1654static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
1655 const SITargetLowering *TLI,
1656 const GCNTTIImpl *TTIImpl) {
1657 const int NrOfSGPRUntilSpill = 26;
1658 const int NrOfVGPRUntilSpill = 32;
1659
1660 const DataLayout &DL = TTIImpl->getDataLayout();
1661
1662 unsigned adjustThreshold = 0;
1663 int SGPRsInUse = 0;
1664 int VGPRsInUse = 0;
1665 for (const Use &A : CB->args()) {
1666 SmallVector<EVT, 4> ValueVTs;
1667 ComputeValueVTs(TLI: *TLI, DL, Ty: A.get()->getType(), ValueVTs);
1668 for (auto ArgVT : ValueVTs) {
1669 unsigned CCRegNum = TLI->getNumRegistersForCallingConv(
1670 Context&: CB->getContext(), CC: CB->getCallingConv(), VT: ArgVT);
1671 if (AMDGPU::isArgPassedInSGPR(CB, ArgNo: CB->getArgOperandNo(U: &A)))
1672 SGPRsInUse += CCRegNum;
1673 else
1674 VGPRsInUse += CCRegNum;
1675 }
1676 }
1677
1678 // The cost of passing function arguments through the stack:
1679 // 1 instruction to put a function argument on the stack in the caller.
1680 // 1 instruction to take a function argument from the stack in callee.
1681 // 1 instruction is explicitly take care of data dependencies in callee
1682 // function.
1683 InstructionCost ArgStackCost(1);
1684 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1685 Opcode: Instruction::Store, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1686 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1687 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1688 Opcode: Instruction::Load, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1689 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1690
1691 // The penalty cost is computed relative to the cost of instructions and does
1692 // not model any storage costs.
1693 adjustThreshold += std::max(a: 0, b: SGPRsInUse - NrOfSGPRUntilSpill) *
1694 ArgStackCost.getValue() * InlineConstants::getInstrCost();
1695 adjustThreshold += std::max(a: 0, b: VGPRsInUse - NrOfVGPRUntilSpill) *
1696 ArgStackCost.getValue() * InlineConstants::getInstrCost();
1697 return adjustThreshold;
1698}
1699
1700static unsigned getCallArgsTotalAllocaSize(const CallBase *CB,
1701 const DataLayout &DL) {
1702 // If we have a pointer to a private array passed into a function
1703 // it will not be optimized out, leaving scratch usage.
1704 // This function calculates the total size in bytes of the memory that would
1705 // end in scratch if the call was not inlined.
1706 unsigned AllocaSize = 0;
1707 SmallPtrSet<const AllocaInst *, 8> AIVisited;
1708 for (Value *PtrArg : CB->args()) {
1709 PointerType *Ty = dyn_cast<PointerType>(Val: PtrArg->getType());
1710 if (!Ty)
1711 continue;
1712
1713 unsigned AddrSpace = Ty->getAddressSpace();
1714 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&
1715 AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)
1716 continue;
1717
1718 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: PtrArg));
1719 if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(Ptr: AI).second)
1720 continue;
1721
1722 if (auto Size = AI->getAllocationSize(DL))
1723 AllocaSize += Size->getFixedValue();
1724 }
1725 return AllocaSize;
1726}
1727
1728int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
1729 return BaseT::getInliningLastCallToStaticBonus() *
1730 getInliningThresholdMultiplier();
1731}
1732
1733unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
1734 unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, TTIImpl: this);
1735
1736 // Private object passed as arguments may end up in scratch usage if the call
1737 // is not inlined. Increase the inline threshold to promote inlining.
1738 unsigned AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
1739 if (AllocaSize > 0)
1740 Threshold += ArgAllocaCost;
1741 return Threshold;
1742}
1743
1744unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
1745 const AllocaInst *AI) const {
1746
1747 // Below the cutoff, assume that the private memory objects would be
1748 // optimized
1749 auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
1750 if (AllocaSize <= ArgAllocaCutoff)
1751 return 0;
1752
1753 // Above the cutoff, we give a cost to each private memory object
1754 // depending its size. If the array can be optimized by SROA this cost is not
1755 // added to the total-cost in the inliner cost analysis.
1756 //
1757 // We choose the total cost of the alloca such that their sum cancels the
1758 // bonus given in the threshold (ArgAllocaCost).
1759 //
1760 // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost
1761 //
1762 // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,
1763 // the single-bb bonus and the vector-bonus.
1764 //
1765 // We compensate the first two multipliers, by repeating logic from the
1766 // inliner-cost in here. The vector-bonus is 0 on AMDGPU.
1767 static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");
1768 unsigned Threshold = ArgAllocaCost * getInliningThresholdMultiplier();
1769
1770 bool SingleBB = none_of(Range&: *CB->getCalledFunction(), P: [](const BasicBlock &BB) {
1771 return BB.getTerminator()->getNumSuccessors() > 1;
1772 });
1773 if (SingleBB) {
1774 Threshold += Threshold / 2;
1775 }
1776
1777 auto ArgAllocaSize = AI->getAllocationSize(DL);
1778 if (!ArgAllocaSize)
1779 return 0;
1780
1781 // Attribute the bonus proportionally to the alloca size
1782 unsigned AllocaThresholdBonus =
1783 (Threshold * ArgAllocaSize->getFixedValue()) / AllocaSize;
1784
1785 return AllocaThresholdBonus;
1786}
1787
1788void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
1789 TTI::UnrollingPreferences &UP,
1790 OptimizationRemarkEmitter *ORE) const {
1791 CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
1792}
1793
1794void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
1795 TTI::PeelingPreferences &PP) const {
1796 CommonTTI.getPeelingPreferences(L, SE, PP);
1797}
1798
1799int GCNTTIImpl::getTransInstrCost(TTI::TargetCostKind CostKind) const {
1800 return getQuarterRateInstrCost(CostKind);
1801}
1802
1803int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {
1804 return ST->hasFullRate64Ops()
1805 ? getFullRateInstrCost()
1806 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)
1807 : getQuarterRateInstrCost(CostKind);
1808}
1809
1810std::pair<InstructionCost, MVT>
1811GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {
1812 std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);
1813 auto Size = DL.getTypeSizeInBits(Ty);
1814 // Maximum load or store can handle 8 dwords for scalar and 4 for
1815 // vector ALU. Let's assume anything above 8 dwords is expensive
1816 // even if legal.
1817 if (Size <= 256)
1818 return Cost;
1819
1820 Cost.first += (Size + 255) / 256;
1821 return Cost;
1822}
1823
1824unsigned GCNTTIImpl::getCacheLineSize() const {
1825 if (ST->hasVmemPrefInsts() || ST->hasSmemPrefetchInsts())
1826 return ST->getDataCacheLineSize();
1827 return 0;
1828}
1829
1830unsigned GCNTTIImpl::getPrefetchDistance() const {
1831 return ST->hasPrefetch() ? 128 : 0;
1832}
1833
1834bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
1835 return AMDGPU::isFlatGlobalAddrSpace(AS);
1836}
1837
1838void GCNTTIImpl::collectKernelLaunchBounds(
1839 const Function &F,
1840 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
1841 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F);
1842 LB.push_back(Elt: {"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});
1843 LB.push_back(Elt: {"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});
1844 LB.push_back(Elt: {"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});
1845 std::pair<unsigned, unsigned> FlatWorkGroupSize =
1846 ST->getFlatWorkGroupSizes(F);
1847 LB.push_back(Elt: {"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});
1848 LB.push_back(Elt: {"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});
1849 std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);
1850 LB.push_back(Elt: {"amdgpu-waves-per-eu[0]", WavesPerEU.first});
1851 LB.push_back(Elt: {"amdgpu-waves-per-eu[1]", WavesPerEU.second});
1852}
1853
1854GCNTTIImpl::KnownIEEEMode
1855GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
1856 if (!ST->hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
1857 return KnownIEEEMode::On; // Only mode on gfx1170+
1858
1859 const Function *F = I.getFunction();
1860 if (!F)
1861 return KnownIEEEMode::Unknown;
1862
1863 Attribute IEEEAttr = F->getFnAttribute(Kind: "amdgpu-ieee");
1864 if (IEEEAttr.isValid())
1865 return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off;
1866
1867 return AMDGPU::isShader(CC: F->getCallingConv()) ? KnownIEEEMode::Off
1868 : KnownIEEEMode::On;
1869}
1870
1871InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1872 Align Alignment,
1873 unsigned AddressSpace,
1874 TTI::TargetCostKind CostKind,
1875 TTI::OperandValueInfo OpInfo,
1876 const Instruction *I) const {
1877 if (VectorType *VecTy = dyn_cast<VectorType>(Val: Src)) {
1878 if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
1879 CostKind != TTI::TCK_Latency &&
1880 VecTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1881 return divideCeil(Numerator: DL.getTypeSizeInBits(Ty: VecTy) - 1,
1882 Denominator: getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace));
1883 }
1884 }
1885 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
1886 OpInfo, I);
1887}
1888
1889unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
1890 if (VectorType *VecTy = dyn_cast<VectorType>(Val: Tp)) {
1891 if (VecTy->getElementType()->isIntegerTy(BitWidth: 8)) {
1892 unsigned ElementCount = VecTy->getElementCount().getFixedValue();
1893 return divideCeil(Numerator: ElementCount - 1, Denominator: 4);
1894 }
1895 }
1896 return BaseT::getNumberOfParts(Tp);
1897}
1898
1899ValueUniformity GCNTTIImpl::getValueUniformity(const Value *V) const {
1900 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
1901 switch (Intrinsic->getIntrinsicID()) {
1902 case Intrinsic::amdgcn_wave_shuffle:
1903 return ValueUniformity::Custom;
1904 default:
1905 break;
1906 }
1907 }
1908
1909 if (isAlwaysUniform(V))
1910 return ValueUniformity::AlwaysUniform;
1911
1912 if (isSourceOfDivergence(V))
1913 return ValueUniformity::NeverUniform;
1914
1915 return ValueUniformity::Default;
1916}
1917
1918InstructionCost GCNTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
1919 StackOffset BaseOffset,
1920 bool HasBaseReg, int64_t Scale,
1921 unsigned AddrSpace) const {
1922 if (HasBaseReg && Scale != 0) {
1923 // gfx1250+ can fold base+scale*index when scale matches the memory access
1924 // size (scale_offset bit). Supported for flat/global/constant/scratch
1925 // (VMEM, max 128 bits) and constant_32bit (SMRD, capped to 128 bits here).
1926 if (getST()->hasScaleOffset() && Ty && Ty->isSized() &&
1927 (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace) ||
1928 AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
1929 AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)) {
1930 TypeSize StoreSize = getDataLayout().getTypeStoreSize(Ty);
1931 if (TypeSize::isKnownLE(LHS: StoreSize, RHS: TypeSize::getFixed(ExactSize: 16)) &&
1932 static_cast<int64_t>(StoreSize.getFixedValue()) == Scale)
1933 return 0;
1934 }
1935 return 1;
1936 }
1937 return BaseT::getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
1938 AddrSpace);
1939}
1940
1941bool GCNTTIImpl::isLSRCostLess(const TTI::LSRCost &A,
1942 const TTI::LSRCost &B) const {
1943 // Favor lower per-iteration work over preheader/setup costs.
1944 // AMDGPU lacks rich addressing modes, so ScaleCost is folded into the
1945 // effective instruction count (base+scale*index requires a separate ADD).
1946 unsigned EffInsnsA = A.Insns + A.ScaleCost;
1947 unsigned EffInsnsB = B.Insns + B.ScaleCost;
1948
1949 return std::tie(args&: EffInsnsA, args: A.NumIVMuls, args: A.AddRecCost, args: A.NumBaseAdds,
1950 args: A.SetupCost, args: A.ImmCost, args: A.NumRegs) <
1951 std::tie(args&: EffInsnsB, args: B.NumIVMuls, args: B.AddRecCost, args: B.NumBaseAdds,
1952 args: B.SetupCost, args: B.ImmCost, args: B.NumRegs);
1953}
1954
1955bool GCNTTIImpl::isNumRegsMajorCostOfLSR() const {
1956 // isLSRCostLess de-prioritizes register count; keep consistent.
1957 return false;
1958}
1959
1960bool GCNTTIImpl::shouldDropLSRSolutionIfLessProfitable() const {
1961 // Prefer the baseline when LSR cannot clearly reduce per-iteration work.
1962 return true;
1963}
1964
1965bool GCNTTIImpl::isUniform(const Instruction *I,
1966 const SmallBitVector &UniformArgs) const {
1967 const IntrinsicInst *Intrinsic = cast<IntrinsicInst>(Val: I);
1968 switch (Intrinsic->getIntrinsicID()) {
1969 case Intrinsic::amdgcn_wave_shuffle:
1970 // wave_shuffle(Value, Index): result is uniform when either Value or Index
1971 // is uniform.
1972 return UniformArgs[0] || UniformArgs[1];
1973 default:
1974 llvm_unreachable("unexpected intrinsic in isUniform");
1975 }
1976}
1977