1//===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetTransformInfo.h"
18#include "AMDGPUSubtarget.h"
19#include "AMDGPUTargetMachine.h"
20#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21#include "SIModeRegisterDefaults.h"
22#include "llvm/ADT/SmallBitVector.h"
23#include "llvm/Analysis/InlineCost.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/CodeGen/Analysis.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/Support/KnownBits.h"
32#include <optional>
33
34using namespace llvm;
35
36#define DEBUG_TYPE "AMDGPUtti"
37
38static cl::opt<unsigned> UnrollThresholdPrivate(
39 "amdgpu-unroll-threshold-private",
40 cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"),
41 cl::init(Val: 2700), cl::Hidden);
42
43static cl::opt<unsigned> UnrollThresholdLocal(
44 "amdgpu-unroll-threshold-local",
45 cl::desc("Unroll threshold for AMDGPU if local memory used in a loop"),
46 cl::init(Val: 1000), cl::Hidden);
47
48static cl::opt<unsigned> UnrollThresholdIf(
49 "amdgpu-unroll-threshold-if",
50 cl::desc("Unroll threshold increment for AMDGPU for each if statement inside loop"),
51 cl::init(Val: 200), cl::Hidden);
52
53static cl::opt<bool> UnrollRuntimeLocal(
54 "amdgpu-unroll-runtime-local",
55 cl::desc("Allow runtime unroll for AMDGPU if local memory used in a loop"),
56 cl::init(Val: true), cl::Hidden);
57
58static cl::opt<unsigned> UnrollMaxBlockToAnalyze(
59 "amdgpu-unroll-max-block-to-analyze",
60 cl::desc("Inner loop block size threshold to analyze in unroll for AMDGPU"),
61 cl::init(Val: 32), cl::Hidden);
62
63static cl::opt<unsigned> ArgAllocaCost("amdgpu-inline-arg-alloca-cost",
64 cl::Hidden, cl::init(Val: 4000),
65 cl::desc("Cost of alloca argument"));
66
67// If the amount of scratch memory to eliminate exceeds our ability to allocate
68// it into registers we gain nothing by aggressively inlining functions for that
69// heuristic.
70static cl::opt<unsigned>
71 ArgAllocaCutoff("amdgpu-inline-arg-alloca-cutoff", cl::Hidden,
72 cl::init(Val: 256),
73 cl::desc("Maximum alloca size to use for inline cost"));
74
75// Inliner constraint to achieve reasonable compilation time.
76static cl::opt<size_t> InlineMaxBB(
77 "amdgpu-inline-max-bb", cl::Hidden, cl::init(Val: 1100),
78 cl::desc("Maximum number of BBs allowed in a function after inlining"
79 " (compile time constraint)"));
80
81// This default unroll factor is based on microbenchmarks on gfx1030.
82static cl::opt<unsigned> MemcpyLoopUnroll(
83 "amdgpu-memcpy-loop-unroll",
84 cl::desc("Unroll factor (affecting 4x32-bit operations) to use for memory "
85 "operations when lowering statically-sized memcpy, memmove, or"
86 "memset as a loop"),
87 cl::init(Val: 16), cl::Hidden);
88
89static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
90 unsigned Depth = 0) {
91 const Instruction *I = dyn_cast<Instruction>(Val: Cond);
92 if (!I)
93 return false;
94
95 if (!L->contains(Inst: I))
96 return false;
97 for (const Value *V : I->operand_values()) {
98 if (const PHINode *PHI = dyn_cast<PHINode>(Val: V)) {
99 if (llvm::none_of(Range: L->getSubLoops(), P: [PHI](const Loop* SubLoop) {
100 return SubLoop->contains(Inst: PHI); }))
101 return true;
102 } else if (Depth < 10 && dependsOnLocalPhi(L, Cond: V, Depth: Depth+1))
103 return true;
104 }
105 return false;
106}
107
108AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
109 : BaseT(TM, F.getDataLayout()),
110 TargetTriple(F.getParent()->getTargetTriple()),
111 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
112 TLI(ST->getTargetLowering()) {}
113
114void AMDGPUTTIImpl::getUnrollingPreferences(
115 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
116 OptimizationRemarkEmitter *ORE) const {
117 const Function &F = *L->getHeader()->getParent();
118 UP.Threshold = static_cast<unsigned>(
119 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-unroll-threshold", Default: 300));
120 UP.PartialThreshold = static_cast<unsigned>(
121 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-partial-unroll-threshold", Default: 150));
122 UP.MaxCount = std::numeric_limits<unsigned>::max();
123 UP.Partial = true;
124
125 // Conditional branch in a loop back edge needs 3 additional exec
126 // manipulations in average.
127 UP.BEInsns += 3;
128
129 // We want to run unroll even for the loops which have been vectorized.
130 UP.UnrollVectorizedLoop = true;
131
132 // Enable runtime unrolling for loops whose trip count is not known at
133 // compile time.
134 UP.Runtime = true;
135
136 // Maximum alloca size than can fit registers. Reserve 16 registers.
137 const unsigned MaxAlloca = (256 - 16) * 4;
138 unsigned ThresholdPrivate = UnrollThresholdPrivate;
139 unsigned ThresholdLocal = UnrollThresholdLocal;
140
141 // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the
142 // provided threshold value as the default for Threshold
143 if (MDNode *LoopUnrollThreshold =
144 findOptionMDForLoop(TheLoop: L, Name: "amdgpu.loop.unroll.threshold")) {
145 if (LoopUnrollThreshold->getNumOperands() == 2) {
146 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
147 MD: LoopUnrollThreshold->getOperand(I: 1));
148 if (MetaThresholdValue) {
149 // We will also use the supplied value for PartialThreshold for now.
150 // We may introduce additional metadata if it becomes necessary in the
151 // future.
152 UP.Threshold =
153 static_cast<unsigned>(MetaThresholdValue->getSExtValue());
154 UP.PartialThreshold = UP.Threshold;
155 ThresholdPrivate = std::min(a: ThresholdPrivate, b: UP.Threshold);
156 ThresholdLocal = std::min(a: ThresholdLocal, b: UP.Threshold);
157 }
158 }
159 }
160
161 unsigned MaxBoost = std::max(a: ThresholdPrivate, b: ThresholdLocal);
162 for (const BasicBlock *BB : L->getBlocks()) {
163 const DataLayout &DL = BB->getDataLayout();
164 unsigned LocalGEPsSeen = 0;
165
166 if (llvm::any_of(Range: L->getSubLoops(), P: [BB](const Loop* SubLoop) {
167 return SubLoop->contains(BB); }))
168 continue; // Block belongs to an inner loop.
169
170 for (const Instruction &I : *BB) {
171 // Unroll a loop which contains an "if" statement whose condition
172 // defined by a PHI belonging to the loop. This may help to eliminate
173 // if region and potentially even PHI itself, saving on both divergence
174 // and registers used for the PHI.
175 // Add a small bonus for each of such "if" statements.
176 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: &I)) {
177 if (UP.Threshold < MaxBoost) {
178 BasicBlock *Succ0 = Br->getSuccessor(i: 0);
179 BasicBlock *Succ1 = Br->getSuccessor(i: 1);
180 if ((L->contains(BB: Succ0) && L->isLoopExiting(BB: Succ0)) ||
181 (L->contains(BB: Succ1) && L->isLoopExiting(BB: Succ1)))
182 continue;
183 if (dependsOnLocalPhi(L, Cond: Br->getCondition())) {
184 UP.Threshold += UnrollThresholdIf;
185 LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
186 << " for loop:\n"
187 << *L << " due to " << *Br << '\n');
188 if (UP.Threshold >= MaxBoost)
189 return;
190 }
191 }
192 continue;
193 }
194
195 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
196 if (!GEP)
197 continue;
198
199 unsigned AS = GEP->getAddressSpace();
200 unsigned Threshold = 0;
201 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
202 Threshold = ThresholdPrivate;
203 else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
204 Threshold = ThresholdLocal;
205 else
206 continue;
207
208 if (UP.Threshold >= Threshold)
209 continue;
210
211 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
212 const Value *Ptr = GEP->getPointerOperand();
213 const AllocaInst *Alloca =
214 dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Ptr));
215 if (!Alloca || !Alloca->isStaticAlloca())
216 continue;
217 auto AllocaSize = Alloca->getAllocationSize(DL);
218 if (!AllocaSize || AllocaSize->getFixedValue() > MaxAlloca)
219 continue;
220 } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
221 AS == AMDGPUAS::REGION_ADDRESS) {
222 LocalGEPsSeen++;
223 // Inhibit unroll for local memory if we have seen addressing not to
224 // a variable, most likely we will be unable to combine it.
225 // Do not unroll too deep inner loops for local memory to give a chance
226 // to unroll an outer loop for a more important reason.
227 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
228 (!isa<GlobalVariable>(Val: GEP->getPointerOperand()) &&
229 !isa<Argument>(Val: GEP->getPointerOperand())))
230 continue;
231 LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
232 << *L << " due to LDS use.\n");
233 UP.Runtime = UnrollRuntimeLocal;
234 }
235
236 // Check if GEP depends on a value defined by this loop itself.
237 bool HasLoopDef = false;
238 for (const Value *Op : GEP->operands()) {
239 const Instruction *Inst = dyn_cast<Instruction>(Val: Op);
240 if (!Inst || L->isLoopInvariant(V: Op))
241 continue;
242
243 if (llvm::any_of(Range: L->getSubLoops(), P: [Inst](const Loop* SubLoop) {
244 return SubLoop->contains(Inst); }))
245 continue;
246 HasLoopDef = true;
247 break;
248 }
249 if (!HasLoopDef)
250 continue;
251
252 // We want to do whatever we can to limit the number of alloca
253 // instructions that make it through to the code generator. allocas
254 // require us to use indirect addressing, which is slow and prone to
255 // compiler bugs. If this loop does an address calculation on an
256 // alloca ptr, then we want to use a higher than normal loop unroll
257 // threshold. This will give SROA a better chance to eliminate these
258 // allocas.
259 //
260 // We also want to have more unrolling for local memory to let ds
261 // instructions with different offsets combine.
262 //
263 // Don't use the maximum allowed value here as it will make some
264 // programs way too big.
265 UP.Threshold = Threshold;
266 LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
267 << " for loop:\n"
268 << *L << " due to " << *GEP << '\n');
269 if (UP.Threshold >= MaxBoost)
270 return;
271 }
272
273 // If we got a GEP in a small BB from inner loop then increase max trip
274 // count to analyze for better estimation cost in unroll
275 if (L->isInnermost() && BB->size() < UnrollMaxBlockToAnalyze)
276 UP.MaxIterationsCountToAnalyze = 32;
277 }
278}
279
280void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
281 TTI::PeelingPreferences &PP) const {
282 BaseT::getPeelingPreferences(L, SE, PP);
283}
284
285uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
286 return 1024;
287}
288
289GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
290 : BaseT(TM, F.getDataLayout()),
291 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
292 TLI(ST->getTargetLowering()), CommonTTI(TM, F),
293 IsGraphics(AMDGPU::isGraphics(CC: F.getCallingConv())) {
294 SIModeRegisterDefaults Mode(F, *ST);
295 HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
296}
297
298bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
299 return !F || !ST->isSingleLaneExecution(Kernel: *F);
300}
301
302unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
303 // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
304 // registers. See getRegisterClassForType for the implementation.
305 // In this case vector registers are not vector in terms of
306 // VGPRs, but those which can hold multiple values.
307
308 // This is really the number of registers to fill when vectorizing /
309 // interleaving loops, so we lie to avoid trying to use all registers.
310 return 4;
311}
312
313TypeSize
314GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
315 switch (K) {
316 case TargetTransformInfo::RGK_Scalar:
317 return TypeSize::getFixed(ExactSize: 32);
318 case TargetTransformInfo::RGK_FixedWidthVector:
319 return TypeSize::getFixed(
320 ExactSize: (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()) ? 128
321 : ST->hasAnyPackedFP32Ops() ? 64
322 : 32);
323 case TargetTransformInfo::RGK_ScalableVector:
324 return TypeSize::getScalable(MinimumSize: 0);
325 }
326 llvm_unreachable("Unsupported register kind");
327}
328
329unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
330 return 32;
331}
332
333unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
334 if (Opcode == Instruction::Load || Opcode == Instruction::Store)
335 return 32 * 4 / ElemWidth;
336 // For a given width return the max 0number of elements that can be combined
337 // into a wider bit value:
338 return (ElemWidth == 8 && ST->has16BitInsts()) ? 4
339 : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
340 : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2
341 : (ElemWidth == 64 &&
342 (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()))
343 ? 2
344 : 1;
345}
346
347bool GCNTTIImpl::preferSLPInstCountCheck() const {
348 // The integer inst-count heuristic causes regressions on gfx94x and gfx950
349 // because 2-element vector trees that pass the scalar/vector instruction
350 // count comparison still widen scalar moves (e.g. v_mov_b32 to v_mov_b64)
351 // after codegen, increasing register pressure and throughput cost without
352 // reducing the total instruction count.
353 return !ST->hasGFX940Insts() && !ST->hasGFX950Insts();
354}
355
356unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
357 unsigned ChainSizeInBytes,
358 VectorType *VecTy) const {
359 unsigned VecRegBitWidth = VF * LoadSize;
360 if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
361 // TODO: Support element-size less than 32bit?
362 return 128 / LoadSize;
363
364 return VF;
365}
366
367unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
368 unsigned ChainSizeInBytes,
369 VectorType *VecTy) const {
370 unsigned VecRegBitWidth = VF * StoreSize;
371 if (VecRegBitWidth > 128)
372 return 128 / StoreSize;
373
374 return VF;
375}
376
377unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
378 if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
379 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
380 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
381 AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
382 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
383 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
384 return 512;
385 }
386
387 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
388 return 8 * ST->getMaxPrivateElementSize();
389
390 // Common to flat, global, local and region. Assume for unknown addrspace.
391 return 128;
392}
393
394bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
395 Align Alignment,
396 unsigned AddrSpace) const {
397 // We allow vectorization of flat stores, even though we may need to decompose
398 // them later if they may access private memory. We don't have enough context
399 // here, and legalization can handle it.
400 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
401 return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&
402 ChainSizeInBytes <= ST->getMaxPrivateElementSize();
403 }
404 return true;
405}
406
407bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
408 Align Alignment,
409 unsigned AddrSpace) const {
410 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
411}
412
413bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
414 Align Alignment,
415 unsigned AddrSpace) const {
416 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
417}
418
419uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
420 return 1024;
421}
422
423Type *GCNTTIImpl::getMemcpyLoopLoweringType(
424 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
425 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
426 std::optional<uint32_t> AtomicElementSize) const {
427
428 if (AtomicElementSize)
429 return Type::getIntNTy(C&: Context, N: *AtomicElementSize * 8);
430
431 // 16-byte accesses achieve the highest copy throughput.
432 // If the operation has a fixed known length that is large enough, it is
433 // worthwhile to return an even wider type and let legalization lower it into
434 // multiple accesses, effectively unrolling the memcpy loop.
435 // We also rely on legalization to decompose into smaller accesses for
436 // subtargets and address spaces where it is necessary.
437 //
438 // Don't unroll if Length is not a constant, since unrolling leads to worse
439 // performance for length values that are smaller or slightly larger than the
440 // total size of the type returned here. Mitigating that would require a more
441 // complex lowering for variable-length memcpy and memmove.
442 unsigned I32EltsInVector = 4;
443 if (MemcpyLoopUnroll > 0 && isa<ConstantInt>(Val: Length))
444 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context),
445 NumElts: MemcpyLoopUnroll * I32EltsInVector);
446
447 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: I32EltsInVector);
448}
449
450void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
451 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
452 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
453 Align SrcAlign, Align DestAlign,
454 std::optional<uint32_t> AtomicCpySize) const {
455
456 if (AtomicCpySize)
457 BaseT::getMemcpyLoopResidualLoweringType(
458 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
459 DestAlign, AtomicCpySize);
460
461 Type *I32x4Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: 4);
462 while (RemainingBytes >= 16) {
463 OpsOut.push_back(Elt: I32x4Ty);
464 RemainingBytes -= 16;
465 }
466
467 Type *I64Ty = Type::getInt64Ty(C&: Context);
468 while (RemainingBytes >= 8) {
469 OpsOut.push_back(Elt: I64Ty);
470 RemainingBytes -= 8;
471 }
472
473 Type *I32Ty = Type::getInt32Ty(C&: Context);
474 while (RemainingBytes >= 4) {
475 OpsOut.push_back(Elt: I32Ty);
476 RemainingBytes -= 4;
477 }
478
479 Type *I16Ty = Type::getInt16Ty(C&: Context);
480 while (RemainingBytes >= 2) {
481 OpsOut.push_back(Elt: I16Ty);
482 RemainingBytes -= 2;
483 }
484
485 Type *I8Ty = Type::getInt8Ty(C&: Context);
486 while (RemainingBytes) {
487 OpsOut.push_back(Elt: I8Ty);
488 --RemainingBytes;
489 }
490}
491
492unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF,
493 bool HasUnorderedReductions) const {
494 // Disable unrolling if the loop is not vectorized.
495 // TODO: Enable this again.
496 if (VF.isScalar())
497 return 1;
498
499 return 8;
500}
501
502bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
503 MemIntrinsicInfo &Info) const {
504 switch (Inst->getIntrinsicID()) {
505 case Intrinsic::amdgcn_ds_ordered_add:
506 case Intrinsic::amdgcn_ds_ordered_swap: {
507 auto *Ordering = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 2));
508 auto *Volatile = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 4));
509 if (!Ordering || !Volatile)
510 return false; // Invalid.
511
512 uint64_t OrderingVal = Ordering->getZExtValue();
513 if (!isValidAtomicOrdering(I: OrderingVal))
514 return false;
515
516 Info.PtrVal = Inst->getArgOperand(i: 0);
517 Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
518 Info.ReadMem = true;
519 Info.WriteMem = true;
520 Info.IsVolatile = !Volatile->isZero();
521 return true;
522 }
523 default:
524 return false;
525 }
526}
527
528/// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
529/// expected to fuse during instruction selection. \p Ty is the type the fused
530/// operation runs on.
531static bool canFuseFMulWithFAddSub(const SITargetLowering &TLI, Type *Ty,
532 const Instruction *FMul,
533 const Instruction *FAddSub) {
534 assert((FAddSub->getOpcode() == Instruction::FAdd ||
535 FAddSub->getOpcode() == Instruction::FSub) &&
536 "Expected an fadd or an fsub");
537
538 // The mad forms fuse exactly without fast-math flags but flush denormals.
539 // An fma forms only when it is not slower than the separate operations.
540 const Function &F = *FAddSub->getFunction();
541 const bool HasFMAD = TLI.isFMADLegal(F, Ty);
542 const bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(F, Ty);
543 if (!HasFMAD && !HasFMA)
544 return false;
545
546 // Without a mad the pair fuses only when both carry contract.
547 return HasFMAD || (FAddSub->hasAllowContract() && FMul->hasAllowContract());
548}
549
550/// An fma holds one multiply, so only one fmul operand fuses with \p FAddSub.
551static const Instruction *getFusedFMul(const SITargetLowering &TLI, Type *Ty,
552 const Instruction *FAddSub) {
553 for (const Value *Op : FAddSub->operands()) {
554 const auto *FMul = dyn_cast<Instruction>(Val: Op);
555 if (FMul && FMul->getOpcode() == Instruction::FMul && FMul->hasOneUse() &&
556 canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub))
557 return FMul;
558 }
559 return nullptr;
560}
561
562static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty,
563 const Instruction *FMul, const Instruction *FAddSub) {
564 const Instruction *Fused = getFusedFMul(TLI, Ty, FAddSub);
565 if (Fused == FMul)
566 return true;
567 // (a * b + c * d) + e becomes fma(a, b, fma(c, d, e)) if the outer fadd has
568 // reassoc.
569 if (!Fused || FAddSub->getOpcode() != Instruction::FAdd ||
570 !FAddSub->hasOneUse())
571 return false;
572 const auto *Outer = dyn_cast<BinaryOperator>(Val: *FAddSub->user_begin());
573 return Outer && Outer->getOpcode() == Instruction::FAdd &&
574 Outer->hasAllowReassoc() &&
575 canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub: Outer);
576}
577
578InstructionCost GCNTTIImpl::getArithmeticInstrCost(
579 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
580 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
581 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
582
583 // Legalize the type.
584 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
585 int ISD = TLI->InstructionOpcodeToISD(Opcode);
586
587 // Because we don't have any legal vector operations, but the legal types, we
588 // need to account for split vectors.
589 unsigned NElts = LT.second.isVector() ?
590 LT.second.getVectorNumElements() : 1;
591
592 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
593
594 switch (ISD) {
595 case ISD::SHL:
596 case ISD::SRL:
597 case ISD::SRA:
598 if (SLT == MVT::i64)
599 return get64BitInstrCost(CostKind) * LT.first * NElts;
600
601 if (ST->has16BitInsts() && SLT == MVT::i16)
602 NElts = (NElts + 1) / 2;
603
604 // i32
605 return getFullRateInstrCost() * LT.first * NElts;
606 case ISD::ADD:
607 case ISD::SUB:
608 if (SLT == MVT::i64 && ST->hasAnyPackedU64Ops())
609 NElts = (NElts + 1) / 2;
610 [[fallthrough]];
611 case ISD::AND:
612 case ISD::OR:
613 case ISD::XOR:
614 if (SLT == MVT::i64) {
615 // and, or and xor are typically split into 2 VALU instructions.
616 return 2 * getFullRateInstrCost() * LT.first * NElts;
617 }
618
619 if (ST->has16BitInsts() && SLT == MVT::i16)
620 NElts = (NElts + 1) / 2;
621
622 return LT.first * NElts * getFullRateInstrCost();
623 case ISD::MUL: {
624 const int QuarterRateCost = getQuarterRateInstrCost(CostKind);
625 if (SLT == MVT::i64) {
626 const int FullRateCost = getFullRateInstrCost();
627 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
628 }
629
630 if (ST->has16BitInsts() && SLT == MVT::i16)
631 NElts = (NElts + 1) / 2;
632
633 // i32
634 return QuarterRateCost * NElts * LT.first;
635 }
636 case ISD::FMUL:
637 // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
638 // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
639 // fused operation. A vector fmul priced with a scalar context fuses with
640 // the scalar fadd|fsub only when it is emitted lane by lane.
641 if (CtxI && CtxI->hasOneUse() &&
642 (CtxI->getType() == Ty ||
643 getMaximumVF(ElemWidth: Ty->getScalarSizeInBits(), Opcode) == 1)) {
644 const auto *FAddSub = dyn_cast<BinaryOperator>(Val: *CtxI->user_begin());
645 if (FAddSub &&
646 (FAddSub->getOpcode() == Instruction::FAdd ||
647 FAddSub->getOpcode() == Instruction::FSub) &&
648 isFusedFMul(TLI: *TLI, Ty, FMul: CtxI, FAddSub))
649 return TargetTransformInfo::TCC_Free;
650 }
651 [[fallthrough]];
652 case ISD::FADD:
653 case ISD::FSUB:
654 if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32)
655 NElts = (NElts + 1) / 2;
656 if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)
657 NElts = (NElts + 1) / 2;
658 if (SLT == MVT::f64) {
659 if (ST->hasAnyPackedFP64Ops())
660 NElts = (NElts + 1) / 2;
661 return LT.first * NElts * get64BitInstrCost(CostKind);
662 }
663
664 if (ST->has16BitInsts() && SLT == MVT::f16)
665 NElts = (NElts + 1) / 2;
666
667 if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16)
668 return LT.first * NElts * getFullRateInstrCost();
669 break;
670 case ISD::FDIV:
671 case ISD::FREM:
672 // FIXME: frem should be handled separately. The fdiv in it is most of it,
673 // but the current lowering is also not entirely correct.
674 if (SLT == MVT::f64) {
675 int Cost = 7 * get64BitInstrCost(CostKind) +
676 getQuarterRateInstrCost(CostKind) +
677 3 * getHalfRateInstrCost(CostKind);
678 // Add cost of workaround.
679 if (!ST->hasUsableDivScaleConditionOutput())
680 Cost += 3 * getFullRateInstrCost();
681
682 return LT.first * Cost * NElts;
683 }
684
685 if (!Args.empty() && match(V: Args[0], P: PatternMatch::m_FPOne())) {
686 // TODO: This is more complicated, unsafe flags etc.
687 if ((SLT == MVT::f32 && !HasFP32Denormals) ||
688 (SLT == MVT::f16 && ST->has16BitInsts())) {
689 return LT.first * getTransInstrCost(CostKind) * NElts;
690 }
691 }
692
693 if (SLT == MVT::f16 && ST->has16BitInsts()) {
694 // 2 x v_cvt_f32_f16
695 // f32 rcp
696 // f32 fmul
697 // v_cvt_f16_f32
698 // f16 div_fixup
699 int Cost = 4 * getFullRateInstrCost() + 2 * getTransInstrCost(CostKind);
700 return LT.first * Cost * NElts;
701 }
702
703 if (SLT == MVT::f32 && (CtxI && CtxI->hasApproxFunc())) {
704 // Fast unsafe fdiv lowering:
705 // f32 rcp
706 // f32 fmul
707 int Cost = getTransInstrCost(CostKind) + getFullRateInstrCost();
708 return LT.first * Cost * NElts;
709 }
710
711 if (SLT == MVT::f32 || SLT == MVT::f16) {
712 // 4 more v_cvt_* insts without f16 insts support
713 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
714 1 * getTransInstrCost(CostKind);
715
716 if (!HasFP32Denormals) {
717 // FP mode switches.
718 Cost += 2 * getFullRateInstrCost();
719 }
720
721 return LT.first * NElts * Cost;
722 }
723 break;
724 case ISD::FNEG:
725 // Use the backend' estimation. If fneg is not free each element will cost
726 // one additional instruction.
727 return TLI->isFNegFree(VT: SLT) ? 0 : NElts;
728 default:
729 break;
730 }
731
732 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
733 Args, CtxI);
734}
735
736// Return true if there's a potential benefit from using v2f16/v2i16
737// instructions for an intrinsic, even if it requires nontrivial legalization.
738static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
739 switch (ID) {
740 case Intrinsic::fma:
741 case Intrinsic::fmuladd:
742 case Intrinsic::copysign:
743 case Intrinsic::minimumnum:
744 case Intrinsic::maximumnum:
745 case Intrinsic::canonicalize:
746 // There's a small benefit to using vector ops in the legalized code.
747 case Intrinsic::round:
748 case Intrinsic::uadd_sat:
749 case Intrinsic::usub_sat:
750 case Intrinsic::sadd_sat:
751 case Intrinsic::ssub_sat:
752 case Intrinsic::abs:
753 return true;
754 default:
755 return false;
756 }
757}
758
759InstructionCost
760GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
761 TTI::TargetCostKind CostKind) const {
762 switch (ICA.getID()) {
763 case Intrinsic::fabs:
764 // Free source modifier in the common case.
765 return 0;
766 case Intrinsic::amdgcn_workitem_id_x:
767 case Intrinsic::amdgcn_workitem_id_y:
768 case Intrinsic::amdgcn_workitem_id_z:
769 // TODO: If hasPackedTID, or if the calling context is not an entry point
770 // there may be a bit instruction.
771 return 0;
772 case Intrinsic::amdgcn_workgroup_id_x:
773 case Intrinsic::amdgcn_workgroup_id_y:
774 case Intrinsic::amdgcn_workgroup_id_z:
775 case Intrinsic::amdgcn_lds_kernel_id:
776 case Intrinsic::amdgcn_dispatch_ptr:
777 case Intrinsic::amdgcn_dispatch_id:
778 case Intrinsic::amdgcn_implicitarg_ptr:
779 case Intrinsic::amdgcn_queue_ptr:
780 // Read from an argument register.
781 return 0;
782 default:
783 break;
784 }
785
786 Type *RetTy = ICA.getReturnType();
787
788 Intrinsic::ID IID = ICA.getID();
789 switch (IID) {
790 case Intrinsic::exp:
791 case Intrinsic::exp2:
792 case Intrinsic::exp10: {
793 // Legalize the type.
794 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
795 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
796 unsigned NElts =
797 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
798
799 if (SLT == MVT::f64) {
800 unsigned NumOps = 20;
801 if (IID == Intrinsic::exp)
802 ++NumOps;
803 else if (IID == Intrinsic::exp10)
804 NumOps += 3;
805
806 return LT.first * NElts * NumOps * get64BitInstrCost(CostKind);
807 }
808
809 if (SLT == MVT::f32) {
810 unsigned NumFullRateOps = 0;
811 // v_exp_f32 (transcendental).
812 unsigned NumTransOps = 1;
813
814 if (!ICA.getFlags().approxFunc() && IID != Intrinsic::exp2) {
815 // Non-AFN exp/exp10: range reduction + v_exp_f32 + ldexp +
816 // overflow/underflow checks (lowerFEXP). Denorm is also handled.
817 // FMA preamble: ~13 full-rate ops; non-FMA: ~17.
818 NumFullRateOps = ST->hasFastFMAF32() ? 13 : 17;
819 } else {
820 if (IID == Intrinsic::exp) {
821 // lowerFEXPUnsafe: fmul (base conversion) + v_exp_f32.
822 NumFullRateOps = 1;
823 } else if (IID == Intrinsic::exp10) {
824 // lowerFEXP10Unsafe: 3 fmul + 2 v_exp_f32 (double-exp2).
825 NumFullRateOps = 3;
826 NumTransOps = 2;
827 }
828 // Denorm scaling adds setcc + select + fadd + select + fmul.
829 if (HasFP32Denormals)
830 NumFullRateOps += 5;
831 }
832
833 InstructionCost Cost = NumFullRateOps * getFullRateInstrCost() +
834 NumTransOps * getTransInstrCost(CostKind);
835 return LT.first * NElts * Cost;
836 }
837
838 break;
839 }
840 case Intrinsic::log:
841 case Intrinsic::log2:
842 case Intrinsic::log10: {
843 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
844 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
845 unsigned NElts =
846 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
847
848 if (SLT == MVT::f32) {
849 unsigned NumFullRateOps = 0;
850
851 if (IID == Intrinsic::log2) {
852 // LowerFLOG2: just v_log_f32.
853 } else if (ICA.getFlags().approxFunc()) {
854 // LowerFLOGUnsafe: v_log_f32 + fmul (base conversion).
855 NumFullRateOps = 1;
856 } else {
857 // LowerFLOGCommon non-AFN: v_log_f32 + extended-precision
858 // multiply + finite check.
859 NumFullRateOps = ST->hasFastFMAF32() ? 8 : 11;
860 }
861
862 if (HasFP32Denormals)
863 NumFullRateOps += 5;
864
865 InstructionCost Cost =
866 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
867 return LT.first * NElts * Cost;
868 }
869
870 break;
871 }
872 case Intrinsic::sin:
873 case Intrinsic::cos: {
874 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
875 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
876 unsigned NElts =
877 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
878
879 if (SLT == MVT::f32) {
880 // LowerTrig: fmul(1/2pi) + v_sin/v_cos.
881 unsigned NumFullRateOps = ST->hasTrigReducedRange() ? 2 : 1;
882
883 InstructionCost Cost =
884 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
885 return LT.first * NElts * Cost;
886 }
887
888 break;
889 }
890 case Intrinsic::sqrt: {
891 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
892 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
893 unsigned NElts =
894 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
895
896 if (SLT == MVT::f32) {
897 unsigned NumFullRateOps = 0;
898
899 if (!ICA.getFlags().approxFunc()) {
900 // lowerFSQRTF32 non-AFN: v_sqrt_f32 + refinement + scale fixup.
901 NumFullRateOps = HasFP32Denormals ? 17 : 16;
902 }
903
904 InstructionCost Cost =
905 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
906 return LT.first * NElts * Cost;
907 }
908
909 break;
910 }
911 default:
912 break;
913 }
914
915 if (!intrinsicHasPackedVectorBenefit(ID: ICA.getID()))
916 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
917
918 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
919 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
920 unsigned NElts = LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
921
922 if ((ST->hasVOP3PInsts() &&
923 (SLT == MVT::f16 || SLT == MVT::i16 ||
924 (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) ||
925 (ST->hasAnyPackedFP64Ops() && SLT == MVT::f64) ||
926 (ST->hasAnyPackedU64Ops() && SLT == MVT::i64)) {
927 NElts = (NElts + 1) / 2;
928 } else if (SLT == MVT::f32) {
929 bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() &&
930 IID != Intrinsic::minimumnum &&
931 IID != Intrinsic::maximumnum;
932 NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts;
933 }
934
935 // TODO: Get more refined intrinsic costs?
936 unsigned InstRate = getQuarterRateInstrCost(CostKind);
937
938 switch (ICA.getID()) {
939 case Intrinsic::fma:
940 case Intrinsic::fmuladd:
941 if (SLT == MVT::f64) {
942 InstRate = get64BitInstrCost(CostKind);
943 break;
944 }
945
946 if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16)
947 InstRate = getFullRateInstrCost();
948 else {
949 InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)
950 : getQuarterRateInstrCost(CostKind);
951 }
952 break;
953 case Intrinsic::copysign:
954 return NElts * getFullRateInstrCost();
955 case Intrinsic::minimumnum:
956 case Intrinsic::maximumnum: {
957 // Instruction + 2 canonicalizes. For cases that need type promotion, we the
958 // promotion takes the place of the canonicalize.
959 unsigned NumOps = 3;
960 if (const IntrinsicInst *II = ICA.getInst()) {
961 // Directly legal with ieee=0
962 // TODO: Not directly legal with strictfp
963 if (fpenvIEEEMode(I: *II) == KnownIEEEMode::Off)
964 NumOps = 1;
965 }
966
967 unsigned BaseRate =
968 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
969 InstRate = BaseRate * NumOps;
970 break;
971 }
972 case Intrinsic::canonicalize: {
973 InstRate =
974 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
975 break;
976 }
977 case Intrinsic::uadd_sat:
978 case Intrinsic::usub_sat:
979 case Intrinsic::sadd_sat:
980 case Intrinsic::ssub_sat: {
981 if (SLT == MVT::i16 || SLT == MVT::i32)
982 InstRate = getFullRateInstrCost();
983
984 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
985 if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second)))
986 NElts = 1;
987 break;
988 }
989 case Intrinsic::abs:
990 // Expansion takes 2 instructions for VALU
991 if (SLT == MVT::i16 || SLT == MVT::i32)
992 InstRate = 2 * getFullRateInstrCost();
993 break;
994 default:
995 break;
996 }
997
998 return LT.first * NElts * InstRate;
999}
1000
1001InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
1002 TTI::TargetCostKind CostKind,
1003 const Instruction *I) const {
1004 assert((I == nullptr || I->getOpcode() == Opcode) &&
1005 "Opcode should reflect passed instruction.");
1006 const bool SCost =
1007 (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);
1008 const int CBrCost = SCost ? 5 : 7;
1009 switch (Opcode) {
1010 case Instruction::UncondBr:
1011 // Branch instruction takes about 4 slots on gfx900.
1012 return SCost ? 1 : 4;
1013 case Instruction::CondBr:
1014 // Suppose conditional branch takes additional 3 exec manipulations
1015 // instructions in average.
1016 return CBrCost;
1017 case Instruction::Switch: {
1018 const auto *SI = dyn_cast_or_null<SwitchInst>(Val: I);
1019 // Each case (including default) takes 1 cmp + 1 cbr instructions in
1020 // average.
1021 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
1022 }
1023 case Instruction::Ret:
1024 return SCost ? 1 : 10;
1025 }
1026 return BaseT::getCFInstrCost(Opcode, CostKind, I);
1027}
1028
1029InstructionCost GCNTTIImpl::getCmpSelInstrCost(
1030 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
1031 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
1032 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
1033 // For size and latency cost kinds, return a low cost independent of vector
1034 // width to enable SimplifyCFG's speculativelyExecuteBB optimization.
1035 if (CostKind == TTI::TCK_SizeAndLatency)
1036 return 1;
1037
1038 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
1039 Op1Info, Op2Info, I);
1040}
1041
1042// Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with
1043// true16; unpacking is 2.6 to 2.9.
1044static constexpr unsigned MaskPackCostPerElt = 4;
1045static constexpr unsigned MaskUnpackCostPerElt = 3;
1046
1047static std::optional<unsigned> getNumberOfPackedMaskElts(Type *Ty) {
1048 auto *FVT = dyn_cast<FixedVectorType>(Val: Ty);
1049 if (FVT && FVT->getElementType()->isIntegerTy(BitWidth: 1) && FVT->getNumElements() > 1)
1050 return FVT->getNumElements();
1051 return std::nullopt;
1052}
1053
1054InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1055 Type *Src,
1056 TTI::CastContextHint CCH,
1057 TTI::TargetCostKind CostKind,
1058 const Instruction *I) const {
1059 // A bitcast between a vector of i1 and an integer packs or unpacks a mask.
1060 if (Opcode == Instruction::BitCast) {
1061 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Src);
1062 Elts && Dst->isIntegerTy(BitWidth: *Elts))
1063 return InstructionCost(MaskPackCostPerElt) * *Elts *
1064 getFullRateInstrCost();
1065 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Dst);
1066 Elts && Src->isIntegerTy(BitWidth: *Elts))
1067 return InstructionCost(MaskUnpackCostPerElt) * *Elts *
1068 getFullRateInstrCost();
1069 }
1070
1071 // Each f32 lane is rounded and the halves are packed in pairs. A packed
1072 // conversion rounds a pair at once and true16 writes either half.
1073 if (auto *SrcVTy = dyn_cast<FixedVectorType>(Val: Src);
1074 SrcVTy && Opcode == Instruction::FPTrunc && ST->has16BitInsts() &&
1075 SrcVTy->getElementType()->isFloatTy() &&
1076 Dst->getScalarType()->isHalfTy()) {
1077 const unsigned NElts = SrcVTy->getNumElements();
1078 const unsigned Ops = ST->hasCvtPkF16F32Inst() ? divideCeil(Numerator: NElts, Denominator: 2)
1079 : ST->useRealTrue16Insts() ? NElts
1080 : NElts + NElts / 2;
1081 return InstructionCost(Ops) * getFullRateInstrCost();
1082 }
1083
1084 const int ISD = TLI->InstructionOpcodeToISD(Opcode);
1085 switch (ISD) {
1086 case ISD::SINT_TO_FP:
1087 case ISD::UINT_TO_FP:
1088 case ISD::FP_TO_SINT:
1089 case ISD::FP_TO_UINT:
1090 break;
1091 default:
1092 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1093 }
1094
1095 const bool IsIntToFP = ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP;
1096 Type *FPTy = (IsIntToFP ? Dst : Src)->getScalarType();
1097 if (!FPTy->isHalfTy() && !FPTy->isBFloatTy() && !FPTy->isFloatTy() &&
1098 !FPTy->isDoubleTy())
1099 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1100
1101 unsigned NElts = 1;
1102 if (auto *VT = dyn_cast<FixedVectorType>(Val: Src))
1103 NElts = VT->getNumElements();
1104
1105 const unsigned SrcBits = Src->getScalarSizeInBits();
1106 const unsigned DstBits = Dst->getScalarSizeInBits();
1107 const bool IsSigned = ISD == ISD::SINT_TO_FP || ISD == ISD::FP_TO_SINT;
1108 const unsigned IntBits = IsIntToFP ? SrcBits : DstBits;
1109 const bool UsesInt64 = IntBits > 32 && IntBits <= 64;
1110
1111 auto Scale = [&](unsigned FullRateOps,
1112 unsigned FP64Ops = 0) -> InstructionCost {
1113 return NElts * (InstructionCost(FullRateOps) * getFullRateInstrCost() +
1114 InstructionCost(FP64Ops) * get64BitInstrCost(CostKind));
1115 };
1116
1117 if (IsIntToFP) {
1118 // A scalar load of a whole number of bytes that is not a power of two is
1119 // split and its high part load extends the source. A constant address
1120 // space or invariant global load aligned to 4 bytes may be widened instead
1121 // and then still needs the extension. So does a buffer fat pointer load.
1122 const auto *Load =
1123 I && Src->isIntegerTy() && I->getOperand(i: 0)->getType() == Src
1124 ? dyn_cast<LoadInst>(Val: I->getOperand(i: 0))
1125 : nullptr;
1126 const unsigned AS = Load ? Load->getPointerAddressSpace() : 0;
1127 const bool LoadMayWiden =
1128 Load && Load->getAlign() >= Align(4) &&
1129 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
1130 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
1131 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
1132 Load->hasMetadata(KindID: LLVMContext::MD_invariant_load)));
1133 const bool LoadExtends = Load && Load->isSimple() && Load->hasOneUse() &&
1134 !LoadMayWiden &&
1135 AS != AMDGPUAS::BUFFER_FAT_POINTER &&
1136 AS != AMDGPUAS::BUFFER_STRIDED_POINTER &&
1137 SrcBits % 8 == 0 && !isPowerOf2_32(Value: SrcBits);
1138 const unsigned ExtOps =
1139 UsesInt64 && SrcBits < 64 && !LoadExtends ? (IsSigned ? 2 : 1) : 0;
1140 if (FPTy->isBFloatTy()) {
1141 if (SrcBits < 8 || (SrcBits > 32 && !UsesInt64))
1142 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1143
1144 // Each integer is converted to f32 first.
1145 InstructionCost FloatCost =
1146 Scale(UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1);
1147 if (SrcBits < 32)
1148 FloatCost = getCastInstrCost(
1149 Opcode, Dst: Dst->getWithNewType(EltTy: Type::getFloatTy(C&: Dst->getContext())),
1150 Src, CCH, CostKind, I);
1151
1152 // Native rounding can convert a pair. With 16 bit instructions the
1153 // expansion extracts the low significand bit, adds the rounding bias,
1154 // preserves NaNs and shifts the result. Without gfx9 instructions the two
1155 // additions cannot use v_add3_u32.
1156 InstructionCost RoundCost =
1157 ST->hasBF16ConversionInsts()
1158 ? InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) * getFullRateInstrCost()
1159 : Scale(!ST->has16BitInsts() ? 1
1160 : ST->hasGFX9Insts() ? 6
1161 : 7);
1162 return FloatCost + RoundCost;
1163 }
1164
1165 // No instruction converts from a 64 bit integer.
1166 if (UsesInt64) {
1167 if (FPTy->isDoubleTy()) {
1168 // Two conversions, ldexp and add, all using the FP64 rate.
1169 return Scale(ExtOps, 4);
1170 }
1171 if (FPTy->isFloatTy())
1172 return Scale(ExtOps + (IsSigned ? 12 : 8));
1173 return Scale(ExtOps + (IsSigned ? 13 : 9));
1174 }
1175
1176 // A narrow vector source is converted lane by lane.
1177 if (SrcBits >= 8 && SrcBits < 32 && isa<FixedVectorType>(Val: Src)) {
1178 if (FPTy->isDoubleTy())
1179 return Scale(1 + (SrcBits < 16 && IsSigned && ST->has16BitInsts()), 1);
1180 if (FPTy->isHalfTy()) {
1181 const InstructionCost PairCost =
1182 InstructionCost(NElts / 2) * getFullRateInstrCost();
1183 // Lanes wider than 16 bits, and every lane without 16 bit instructions,
1184 // are converted to f32 first. With the packed conversion each pair is
1185 // then converted at once. Otherwise each lane is converted, and without
1186 // real true16 each pair of halves is packed.
1187 if (!ST->has16BitInsts() || SrcBits > 16) {
1188 auto *FloatTy =
1189 FixedVectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()), NumElts: NElts);
1190 const InstructionCost FloatCost =
1191 getCastInstrCost(Opcode, Dst: FloatTy, Src, CCH, CostKind);
1192 if (ST->has16BitInsts() && ST->hasCvtPkF16F32Inst())
1193 return FloatCost + InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) *
1194 getFullRateInstrCost();
1195 const unsigned PackOps =
1196 !ST->has16BitInsts() ? 2 : !ST->useRealTrue16Insts();
1197 return FloatCost + Scale(1) + PackOps * PairCost;
1198 }
1199 // Each lane is converted from a 16 bit subword. Lanes narrower than 16
1200 // bits are extended lane by lane, except bytes with SDWA. Signed bytes
1201 // are sign extended a pair at a time. Real true16 reads the high
1202 // subword in place, so only signed lanes narrower than 16 bits pay per
1203 // pair. Otherwise wider lanes shift the high subword of each pair down
1204 // without SDWA, and each pair of halves is packed.
1205 const unsigned PerElt =
1206 1 + (SrcBits == 8 ? !ST->hasSDWA() : SrcBits < 16);
1207 const bool RealTrue16 = ST->useRealTrue16Insts();
1208 const unsigned PerPair =
1209 !RealTrue16 + (SrcBits == 8 || RealTrue16 ? SrcBits < 16 && IsSigned
1210 : !ST->hasSDWA());
1211 return Scale(PerElt) + PerPair * PairCost;
1212 }
1213 // An unsigned byte is converted straight out of its register.
1214 if (SrcBits == 8 && !IsSigned)
1215 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1216 if (SrcBits < 16 && !IsSigned)
1217 return Scale(2);
1218 unsigned PerElt = ST->hasSDWA() && SrcBits <= 16 ? 1 : 2;
1219 if (SrcBits < 16 && ST->has16BitInsts())
1220 ++PerElt;
1221 return Scale(PerElt);
1222 }
1223
1224 // A narrow scalar source is extended before the conversion. An unsigned
1225 // byte is converted straight out of its register, and a signed one with
1226 // SDWA. A source wider than a byte but narrower than 16 bits is masked
1227 // or sign extended first. Without 16 bit instructions a half result is
1228 // rounded from f32.
1229 if (SrcBits >= 8 && SrcBits < 32) {
1230 if (LoadExtends) {
1231 if (FPTy->isDoubleTy())
1232 return Scale(0, 1);
1233 return Scale(FPTy->isHalfTy() ? 2 : 1);
1234 }
1235 const bool Narrow = SrcBits > 8 && SrcBits < 16;
1236 const bool SignExtend16 = IsSigned && Narrow && ST->has16BitInsts();
1237 if (FPTy->isDoubleTy())
1238 return Scale(1 + 2 * SignExtend16, 1);
1239 if (SrcBits > 16)
1240 return Scale(FPTy->isHalfTy() ? 3 : 2);
1241 if (FPTy->isHalfTy() && ST->has16BitInsts())
1242 return Scale(Narrow ? (IsSigned ? 3 : 2)
1243 : SrcBits == 8 && !ST->hasSDWA() ? 2
1244 : 1);
1245 const InstructionCost FloatCost =
1246 SignExtend16 ? Scale(ST->hasSDWA() ? 3 : 4)
1247 : Narrow ? Scale(2)
1248 : SrcBits == 8 && !IsSigned ? Scale(1)
1249 : Scale(ST->hasSDWA() ? 1 : 2);
1250 return FPTy->isHalfTy() ? FloatCost + Scale(1) : FloatCost;
1251 }
1252
1253 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1254 }
1255
1256 // No instruction converts to a 64 bit integer.
1257 if (UsesInt64) {
1258 const bool IsSigned64 = IsSigned || (DstBits < 64 && NElts == 1);
1259 if (FPTy->isDoubleTy()) {
1260 // With native trunc/floor there are six FP64 operations. The expanded
1261 // rounding sequence has seven, plus integer operations and constants.
1262 return ST->haveRoundOpsF64() ? Scale(1, 6) : Scale(22, 7);
1263 }
1264 // The f32 expansion has one fma, which is quarter rate without fast FMA.
1265 const InstructionCost SlowFMACost =
1266 ST->hasFastFMAF32() ? 0
1267 : NElts * (getQuarterRateInstrCost(CostKind) -
1268 getFullRateInstrCost());
1269 if (FPTy->isFloatTy())
1270 return Scale(IsSigned64 ? 13 : 6) + SlowFMACost;
1271 if (FPTy->isBFloatTy()) {
1272 // Unlike half, bf16 does not fit in i32. Extend to f32 and use the full
1273 // i64 expansion.
1274 return Scale(1 + (IsSigned64 ? 13 : 6)) + SlowFMACost;
1275 }
1276 return Scale(3);
1277 }
1278
1279 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1280}
1281
1282InstructionCost
1283GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
1284 std::optional<FastMathFlags> FMF,
1285 TTI::TargetCostKind CostKind) const {
1286 if (TTI::requiresOrderedReduction(FMF))
1287 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1288
1289 // An add or xor reduction over a vector of i1 becomes a bit count over the
1290 // packed mask; the generic model prices a shuffle tree and misses that.
1291 if (Opcode == Instruction::Add || Opcode == Instruction::Xor) {
1292 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty))
1293 return InstructionCost(MaskPackCostPerElt) * *Elts *
1294 getFullRateInstrCost();
1295 }
1296
1297 EVT OrigTy = TLI->getValueType(DL, Ty);
1298
1299 // Computes cost on targets that have packed math instructions(which support
1300 // 16-bit types only).
1301 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1302 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1303
1304 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1305 return LT.first * getFullRateInstrCost();
1306}
1307
1308InstructionCost
1309GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1310 FastMathFlags FMF,
1311 TTI::TargetCostKind CostKind) const {
1312 EVT OrigTy = TLI->getValueType(DL, Ty);
1313
1314 // Computes cost on targets that have packed math instructions(which support
1315 // 16-bit types only).
1316 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1317 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1318
1319 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1320 return LT.first * getHalfRateInstrCost(CostKind);
1321}
1322
1323InstructionCost GCNTTIImpl::getVectorInstrCost(
1324 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
1325 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
1326 switch (Opcode) {
1327 case Instruction::ExtractElement:
1328 case Instruction::InsertElement: {
1329 TypeSize EltSize =
1330 DL.getTypeSizeInBits(Ty: cast<VectorType>(Val: ValTy)->getElementType());
1331 // Dynamic indexing isn't free and is best avoided.
1332 if (Index == ~0u)
1333 return 2;
1334 if (EltSize < 32) {
1335 if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
1336 return 0;
1337 // Inserts of booleans are free.
1338 // TODO: Extracts are free too.
1339 if (EltSize == 1 && Opcode == Instruction::InsertElement)
1340 return TargetTransformInfo::TCC_Free;
1341 // Extract element sequences of consecutive i8 values that match a
1342 // register size are free most likely. It is not possible to know
1343 // if this extract is part of a consecutive sequence so this may
1344 // apply more generally.
1345 if (Opcode == Instruction::ExtractElement && EltSize == 8) {
1346 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: ValTy)) {
1347 unsigned NumElts = FVTy->getNumElements();
1348 if (NumElts >= 4 && isPowerOf2_32(Value: NumElts))
1349 return 0;
1350 }
1351 }
1352 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1353 VIC);
1354 }
1355
1356 // Extracts are just reads of a subregister, so are free. Inserts are
1357 // considered free because we don't want to have any cost for scalarizing
1358 // operations, and we don't have to copy into a different register class.
1359 return 0;
1360 }
1361 default:
1362 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1363 VIC);
1364 }
1365}
1366
1367/// Analyze if the results of inline asm are divergent. If \p Indices is empty,
1368/// this is analyzing the collective result of all output registers. Otherwise,
1369/// this is only querying a specific result index if this returns multiple
1370/// registers in a struct.
1371bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
1372 const CallBase *CB, ArrayRef<unsigned> Indices) const {
1373 // TODO: Handle complex extract indices
1374 if (Indices.size() > 1)
1375 return true;
1376
1377 const DataLayout &DL = CB->getDataLayout();
1378 const SIRegisterInfo *TRI = ST->getRegisterInfo();
1379 TargetLowering::AsmOperandInfoVector TargetConstraints =
1380 TLI->ParseConstraints(DL, TRI: ST->getRegisterInfo(), Call: *CB);
1381
1382 const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];
1383
1384 int OutputIdx = 0;
1385 for (auto &TC : TargetConstraints) {
1386 if (TC.Type != InlineAsm::isOutput)
1387 continue;
1388
1389 // Skip outputs we don't care about.
1390 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
1391 continue;
1392
1393 TLI->ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
1394
1395 const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(
1396 TRI, Constraint: TC.ConstraintCode, VT: TC.ConstraintVT).second;
1397
1398 // For AGPR constraints null is returned on subtargets without AGPRs, so
1399 // assume divergent for null.
1400 if (!RC || !TRI->isSGPRClass(RC))
1401 return true;
1402 }
1403
1404 return false;
1405}
1406
1407bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
1408 const IntrinsicInst *ReadReg) const {
1409 Metadata *MD =
1410 cast<MetadataAsValue>(Val: ReadReg->getArgOperand(i: 0))->getMetadata();
1411 StringRef RegName =
1412 cast<MDString>(Val: cast<MDNode>(Val: MD)->getOperand(I: 0))->getString();
1413
1414 // Special case registers that look like VCC.
1415 MVT VT = MVT::getVT(Ty: ReadReg->getType());
1416 if (VT == MVT::i1)
1417 return true;
1418
1419 // Special case scalar registers that start with 'v'.
1420 if (RegName.starts_with(Prefix: "vcc") || RegName.empty())
1421 return false;
1422
1423 // VGPR or AGPR is divergent. There aren't any specially named vector
1424 // registers.
1425 return RegName[0] == 'v' || RegName[0] == 'a';
1426}
1427
1428/// \returns true if the result of the value could potentially be
1429/// different across workitems in a wavefront.
1430bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
1431 if (const Argument *A = dyn_cast<Argument>(Val: V))
1432 return !AMDGPU::isArgPassedInSGPR(Arg: A);
1433
1434 // Loads from the private and flat address spaces are divergent, because
1435 // threads can execute the load instruction with the same inputs and get
1436 // different results.
1437 //
1438 // All other loads are not divergent, because if threads issue loads with the
1439 // same arguments, they will always get the same result.
1440 if (const LoadInst *Load = dyn_cast<LoadInst>(Val: V))
1441 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
1442 Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
1443
1444 // Atomics are divergent because they are executed sequentially: when an
1445 // atomic operation refers to the same address in each thread, then each
1446 // thread after the first sees the value written by the previous thread as
1447 // original value.
1448 if (isa<AtomicRMWInst, AtomicCmpXchgInst>(Val: V))
1449 return true;
1450
1451 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
1452 Intrinsic::ID IID = Intrinsic->getIntrinsicID();
1453 switch (IID) {
1454 case Intrinsic::read_register:
1455 return isReadRegisterSourceOfDivergence(ReadReg: Intrinsic);
1456 case Intrinsic::amdgcn_workitem_id_y:
1457 case Intrinsic::amdgcn_workitem_id_z: {
1458 const Function *F = Intrinsic->getFunction();
1459 bool HasUniformYZ =
1460 ST->hasWavefrontsEvenlySplittingXDim(F: *F, /*RequiresUniformYZ=*/true);
1461 std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize(
1462 F: *F, Dim: IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2);
1463 return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1);
1464 }
1465 default:
1466 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID);
1467 }
1468 }
1469
1470 // Assume all function calls are a source of divergence.
1471 if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) {
1472 if (CB->isInlineAsm())
1473 return isInlineAsmSourceOfDivergence(CB);
1474 return true;
1475 }
1476
1477 // Assume all function calls are a source of divergence.
1478 if (isa<InvokeInst>(Val: V))
1479 return true;
1480
1481 // If the target supports globally addressable scratch, the mapping from
1482 // scratch memory to the flat aperture changes therefore an address space cast
1483 // is no longer uniform.
1484 if (auto *CastI = dyn_cast<AddrSpaceCastInst>(Val: V)) {
1485 return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
1486 CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
1487 ST->hasGloballyAddressableScratch();
1488 }
1489
1490 return false;
1491}
1492
1493bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
1494 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V))
1495 return AMDGPU::isIntrinsicAlwaysUniform(IntrID: Intrinsic->getIntrinsicID());
1496
1497 if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) {
1498 if (CB->isInlineAsm())
1499 return !isInlineAsmSourceOfDivergence(CB);
1500 return false;
1501 }
1502
1503 // In most cases TID / wavefrontsize is uniform.
1504 //
1505 // However, if a kernel has uneven dimesions we can have a value of
1506 // workitem-id-x divided by the wavefrontsize non-uniform. For example
1507 // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)
1508 // packed into a same wave which gives 1 and 0 after the division by 64
1509 // respectively.
1510 //
1511 // The X dimension doesn't reset within a wave if either both the Y
1512 // and Z dimensions are of length 1, or if the X dimension's required
1513 // size is a power of 2. Note, however, if the X dimension's maximum
1514 // size is a power of 2 < the wavefront size, division by the wavefront
1515 // size is guaranteed to yield 0, so this is also a no-reset case.
1516 bool XDimDoesntResetWithinWaves = false;
1517 if (auto *I = dyn_cast<Instruction>(Val: V)) {
1518 const Function *F = I->getFunction();
1519 XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(F: *F);
1520 }
1521 using namespace llvm::PatternMatch;
1522 uint64_t C;
1523 auto MatchTidXCall = m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>();
1524 auto MaybeMaskedTidX =
1525 m_CombineOr(Ps: m_c_And(L: MatchTidXCall, R: m_Constant()), Ps: MatchTidXCall);
1526 auto MaybeCastTidX = m_CastOrSelf(Op: MaybeMaskedTidX);
1527 auto MaybeMaskedCastTidX =
1528 m_CombineOr(Ps: m_c_And(L: MaybeCastTidX, R: m_Constant()), Ps: MaybeCastTidX);
1529 if (match(V, P: m_LShr(L: MaybeMaskedCastTidX, R: m_ConstantInt(V&: C))))
1530 return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves;
1531
1532 Constant *Mask;
1533 if (match(V, P: m_c_And(
1534 L: m_CastOrSelf(Op: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>()),
1535 R: m_Constant(C&: Mask)))) {
1536 return computeKnownBits(V: Mask, DL).countMinTrailingZeros() >=
1537 ST->getWavefrontSizeLog2() &&
1538 XDimDoesntResetWithinWaves;
1539 }
1540
1541 const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(Val: V);
1542 if (!ExtValue)
1543 return false;
1544
1545 const CallBase *CB = dyn_cast<CallBase>(Val: ExtValue->getOperand(i_nocapture: 0));
1546 if (!CB)
1547 return false;
1548
1549 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: CB)) {
1550 switch (Intrinsic->getIntrinsicID()) {
1551 default:
1552 return false;
1553 case Intrinsic::amdgcn_if:
1554 case Intrinsic::amdgcn_else: {
1555 ArrayRef<unsigned> Indices = ExtValue->getIndices();
1556 return Indices.size() == 1 && Indices[0] == 1;
1557 }
1558 }
1559 }
1560
1561 // If we have inline asm returning mixed SGPR and VGPR results, we inferred
1562 // divergent for the overall struct return. We need to override it in the
1563 // case we're extracting an SGPR component here.
1564 if (CB->isInlineAsm())
1565 return !isInlineAsmSourceOfDivergence(CB, Indices: ExtValue->getIndices());
1566
1567 return false;
1568}
1569
1570bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
1571 Intrinsic::ID IID) const {
1572 switch (IID) {
1573 case Intrinsic::amdgcn_is_shared:
1574 case Intrinsic::amdgcn_is_private:
1575 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1576 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1577 case Intrinsic::amdgcn_load_to_lds:
1578 case Intrinsic::amdgcn_make_buffer_rsrc:
1579 OpIndexes.push_back(Elt: 0);
1580 return true;
1581 default:
1582 return false;
1583 }
1584}
1585
1586Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
1587 Value *OldV,
1588 Value *NewV) const {
1589 auto IntrID = II->getIntrinsicID();
1590 switch (IntrID) {
1591 case Intrinsic::amdgcn_is_shared:
1592 case Intrinsic::amdgcn_is_private: {
1593 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
1594 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;
1595 unsigned NewAS = NewV->getType()->getPointerAddressSpace();
1596 LLVMContext &Ctx = NewV->getType()->getContext();
1597 ConstantInt *NewVal = (TrueAS == NewAS) ?
1598 ConstantInt::getTrue(Context&: Ctx) : ConstantInt::getFalse(Context&: Ctx);
1599 return NewVal;
1600 }
1601 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1602 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
1603 Type *DestTy = II->getType();
1604 Type *SrcTy = NewV->getType();
1605 unsigned NewAS = SrcTy->getPointerAddressSpace();
1606 if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS))
1607 return nullptr;
1608 Module *M = II->getModule();
1609 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1610 M, id: II->getIntrinsicID(), OverloadTys: {DestTy, SrcTy, DestTy});
1611 II->setArgOperand(i: 0, v: NewV);
1612 II->setCalledFunction(NewDecl);
1613 return II;
1614 }
1615 case Intrinsic::amdgcn_load_to_lds: {
1616 Type *SrcTy = NewV->getType();
1617 Module *M = II->getModule();
1618 Function *NewDecl =
1619 Intrinsic::getOrInsertDeclaration(M, id: II->getIntrinsicID(), OverloadTys: {SrcTy});
1620 II->setArgOperand(i: 0, v: NewV);
1621 II->setCalledFunction(NewDecl);
1622 return II;
1623 }
1624 case Intrinsic::amdgcn_make_buffer_rsrc: {
1625 Type *SrcTy = NewV->getType();
1626 Type *DstTy = II->getType();
1627 Type *NumRecordsTy = II->getArgOperand(i: 2)->getType();
1628 Module *M = II->getModule();
1629 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1630 M, id: II->getIntrinsicID(), OverloadTys: {DstTy, SrcTy, NumRecordsTy});
1631 II->setArgOperand(i: 0, v: NewV);
1632 II->setCalledFunction(NewDecl);
1633 return II;
1634 }
1635 default:
1636 return nullptr;
1637 }
1638}
1639
1640InstructionCost GCNTTIImpl::getShuffleCost(
1641 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1642 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1643 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1644 TTI::VectorInstrContext VIC) const {
1645 if (!isa<FixedVectorType>(Val: SrcTy))
1646 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1647 SubTp);
1648
1649 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1650
1651 TypeSize ScalarSize = DL.getTypeSizeInBits(Ty: SrcTy->getElementType());
1652 if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1653 (ScalarSize == 16 || ScalarSize == 8)) {
1654 // Larger vector widths may require additional instructions, but are
1655 // typically cheaper than scalarized versions.
1656 //
1657 // We assume that shuffling at a register granularity can be done for free.
1658 // This is not true for vectors fed into memory instructions, but it is
1659 // effectively true for all other shuffling. The emphasis of the logic here
1660 // is to assist generic transform in cleaning up / canonicalizing those
1661 // shuffles.
1662
1663 // With op_sel VOP3P instructions freely can access the low half or high
1664 // half of a register, so any swizzle of two elements is free.
1665 if (auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy)) {
1666 unsigned NumSrcElts = SrcVecTy->getNumElements();
1667 if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 &&
1668 (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse ||
1669 Kind == TTI::SK_PermuteSingleSrc))
1670 return 0;
1671 }
1672
1673 unsigned EltsPerReg = 32 / static_cast<unsigned>(ScalarSize);
1674 switch (Kind) {
1675 case TTI::SK_Broadcast:
1676 // A single v_perm_b32 can be re-used for all destination registers.
1677 return 1;
1678 case TTI::SK_Reverse:
1679 // One instruction per register.
1680 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1681 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1682 return InstructionCost::getInvalid();
1683 case TTI::SK_ExtractSubvector:
1684 if (Index % EltsPerReg == 0)
1685 return 0; // Shuffling at register granularity
1686 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1687 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1688 return InstructionCost::getInvalid();
1689 case TTI::SK_InsertSubvector: {
1690 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1691 if (!DstVecTy)
1692 return InstructionCost::getInvalid();
1693 unsigned NumDstElts = DstVecTy->getNumElements();
1694 unsigned NumInsertElts = cast<FixedVectorType>(Val: SubTp)->getNumElements();
1695 unsigned EndIndex = Index + NumInsertElts;
1696 unsigned BeginSubIdx = Index % EltsPerReg;
1697 unsigned EndSubIdx = EndIndex % EltsPerReg;
1698 InstructionCost Cost = 0;
1699
1700 if (BeginSubIdx != 0) {
1701 // Need to shift the inserted vector into place. The cost is the number
1702 // of destination registers overlapped by the inserted vector.
1703 Cost = divideCeil(Numerator: EndIndex, Denominator: EltsPerReg) - (Index / EltsPerReg);
1704 }
1705
1706 // If the last register overlap is partial, there may be three source
1707 // registers feeding into it; that takes an extra instruction.
1708 if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx)
1709 Cost += 1;
1710
1711 return Cost;
1712 }
1713 case TTI::SK_Splice: {
1714 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1715 if (!DstVecTy)
1716 return InstructionCost::getInvalid();
1717 unsigned NumElts = DstVecTy->getNumElements();
1718 assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements());
1719 // Determine the sub-region of the result vector that requires
1720 // sub-register shuffles / mixing.
1721 unsigned EltsFromLHS = NumElts - Index;
1722 bool LHSIsAligned = (Index % EltsPerReg) == 0;
1723 bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0;
1724 if (LHSIsAligned && RHSIsAligned)
1725 return 0;
1726 if (LHSIsAligned && !RHSIsAligned)
1727 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg) - (EltsFromLHS / EltsPerReg);
1728 if (!LHSIsAligned && RHSIsAligned)
1729 return divideCeil(Numerator: EltsFromLHS, Denominator: EltsPerReg);
1730 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg);
1731 }
1732 default:
1733 break;
1734 }
1735
1736 if (!Mask.empty()) {
1737 unsigned NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements();
1738
1739 // Generically estimate the cost by assuming that each destination
1740 // register is derived from sources via v_perm_b32 instructions if it
1741 // can't be copied as-is.
1742 //
1743 // For each destination register, derive the cost of obtaining it based
1744 // on the number of source registers that feed into it.
1745 InstructionCost Cost = 0;
1746 for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) {
1747 SmallVector<int, 4> Regs;
1748 bool Aligned = true;
1749 for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) {
1750 int SrcIdx = Mask[DstIdx + I];
1751 if (SrcIdx == -1)
1752 continue;
1753 int Reg;
1754 if (SrcIdx < (int)NumSrcElts) {
1755 Reg = SrcIdx / EltsPerReg;
1756 if (SrcIdx % EltsPerReg != I)
1757 Aligned = false;
1758 } else {
1759 Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg;
1760 if ((SrcIdx - NumSrcElts) % EltsPerReg != I)
1761 Aligned = false;
1762 }
1763 if (!llvm::is_contained(Range&: Regs, Element: Reg))
1764 Regs.push_back(Elt: Reg);
1765 }
1766 if (Regs.size() >= 2)
1767 Cost += Regs.size() - 1;
1768 else if (!Aligned)
1769 Cost += 1;
1770 }
1771 return Cost;
1772 }
1773 }
1774
1775 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1776 SubTp);
1777}
1778
1779/// Whether it is profitable to sink the operands of an
1780/// Instruction I to the basic block of I.
1781/// This helps using several modifiers (like abs and neg) more often.
1782bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
1783 SmallVectorImpl<Use *> &Ops) const {
1784 using namespace PatternMatch;
1785
1786 // The cost model prices this fmul as free assuming it fuses with its
1787 // fadd/fsub user, which needs them in one block. Sink a stranded
1788 // loop-invariant fmul back to the user when they would fuse. Single use only,
1789 // so this stays a move.
1790 if (I->getOpcode() == Instruction::FAdd ||
1791 I->getOpcode() == Instruction::FSub) {
1792 const Instruction *FMul = getFusedFMul(TLI: *TLI, Ty: I->getType(), FAddSub: I);
1793 if (FMul && FMul->getParent() != I->getParent())
1794 Ops.push_back(Elt: &I->getOperandUse(i: I->getOperand(i: 0) == FMul ? 0 : 1));
1795 }
1796
1797 for (auto &Op : I->operands()) {
1798 // Ensure we are not already sinking this operand.
1799 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op.get(); }))
1800 continue;
1801
1802 if (match(V: &Op, P: m_FAbs(Op0: m_Value())) || match(V: &Op, P: m_FNeg(X: m_Value()))) {
1803 Ops.push_back(Elt: &Op);
1804 continue;
1805 }
1806
1807 // Check for zero-cost multiple use InsertElement/ExtractElement
1808 // instructions
1809 if (Instruction *OpInst = dyn_cast<Instruction>(Val: Op.get())) {
1810 if (OpInst->getType()->isVectorTy() && OpInst->getNumOperands() > 1) {
1811 Instruction *VecOpInst = dyn_cast<Instruction>(Val: OpInst->getOperand(i: 0));
1812 if (VecOpInst && VecOpInst->hasOneUse())
1813 continue;
1814
1815 if (getVectorInstrCost(Opcode: OpInst->getOpcode(), ValTy: OpInst->getType(),
1816 CostKind: TTI::TCK_RecipThroughput, Index: 0,
1817 Op0: OpInst->getOperand(i: 0),
1818 Op1: OpInst->getOperand(i: 1)) == 0) {
1819 Ops.push_back(Elt: &Op);
1820 continue;
1821 }
1822 }
1823 }
1824
1825 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: Op.get())) {
1826
1827 TypeSize EltSize = DL.getTypeSizeInBits(
1828 Ty: cast<VectorType>(Val: Shuffle->getType())->getElementType());
1829
1830 // For i32 (or greater) shufflevectors, these will be lowered into a
1831 // series of insert / extract elements, which will be coalesced away.
1832 if (EltSize < 16 || !ST->has16BitInsts())
1833 continue;
1834
1835 int NumSubElts, SubIndex;
1836 if (Shuffle->changesLength()) {
1837 if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) {
1838 Ops.push_back(Elt: &Op);
1839 continue;
1840 }
1841
1842 if ((Shuffle->isExtractSubvectorMask(Index&: SubIndex) ||
1843 Shuffle->isInsertSubvectorMask(NumSubElts, Index&: SubIndex)) &&
1844 !(SubIndex & 0x1)) {
1845 Ops.push_back(Elt: &Op);
1846 continue;
1847 }
1848 }
1849
1850 if (Shuffle->isReverse() || Shuffle->isZeroEltSplat() ||
1851 Shuffle->isSingleSource()) {
1852 Ops.push_back(Elt: &Op);
1853 continue;
1854 }
1855 }
1856 }
1857
1858 return !Ops.empty();
1859}
1860
1861bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
1862 const Function *Callee) const {
1863 const TargetMachine &TM = getTLI()->getTargetMachine();
1864 const GCNSubtarget *CallerST
1865 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));
1866 const GCNSubtarget *CalleeST
1867 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));
1868
1869 if (!BaseT::areInlineCompatible(Caller, Callee))
1870 return false;
1871
1872 // FIXME: dx10_clamp can just take the caller setting, but there seems to be
1873 // no way to support merge for backend defined attributes.
1874 SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
1875 SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
1876 if (!CallerMode.isInlineCompatible(CalleeMode))
1877 return false;
1878
1879 if (Callee->hasFnAttribute(Kind: Attribute::AlwaysInline) ||
1880 Callee->hasFnAttribute(Kind: Attribute::InlineHint))
1881 return true;
1882
1883 // Hack to make compile times reasonable.
1884 if (InlineMaxBB) {
1885 // Single BB does not increase total BB amount.
1886 if (Callee->size() == 1)
1887 return true;
1888 size_t BBSize = Caller->size() + Callee->size() - 1;
1889 if (BBSize > InlineMaxBB) {
1890 LLVM_DEBUG(dbgs() << "AMDGPU inline max-BB rejected inlining "
1891 << Callee->getName() << " into " << Caller->getName()
1892 << ": caller BBs=" << Caller->size() << ", callee BBs="
1893 << Callee->size() << ", combined BBs=" << BBSize
1894 << ", max BBs=" << InlineMaxBB << '\n');
1895 return false;
1896 }
1897 }
1898
1899 return true;
1900}
1901
1902static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
1903 const SITargetLowering *TLI,
1904 const GCNTTIImpl *TTIImpl) {
1905 const int NrOfSGPRUntilSpill = 26;
1906 const int NrOfVGPRUntilSpill = 32;
1907
1908 const DataLayout &DL = TTIImpl->getDataLayout();
1909
1910 unsigned adjustThreshold = 0;
1911 int SGPRsInUse = 0;
1912 int VGPRsInUse = 0;
1913 for (const Use &A : CB->args()) {
1914 SmallVector<EVT, 4> ValueVTs;
1915 ComputeValueVTs(TLI: *TLI, DL, Ty: A.get()->getType(), ValueVTs);
1916 for (auto ArgVT : ValueVTs) {
1917 unsigned CCRegNum = TLI->getNumRegistersForCallingConv(
1918 Context&: CB->getContext(), CC: CB->getCallingConv(), VT: ArgVT);
1919 if (AMDGPU::isArgPassedInSGPR(CB, ArgNo: CB->getArgOperandNo(U: &A)))
1920 SGPRsInUse += CCRegNum;
1921 else
1922 VGPRsInUse += CCRegNum;
1923 }
1924 }
1925
1926 // The cost of passing function arguments through the stack:
1927 // 1 instruction to put a function argument on the stack in the caller.
1928 // 1 instruction to take a function argument from the stack in callee.
1929 // 1 instruction is explicitly take care of data dependencies in callee
1930 // function.
1931 InstructionCost ArgStackCost(1);
1932 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1933 Opcode: Instruction::Store, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1934 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1935 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1936 Opcode: Instruction::Load, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1937 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1938
1939 // The penalty cost is computed relative to the cost of instructions and does
1940 // not model any storage costs.
1941 adjustThreshold += static_cast<unsigned>(
1942 std::max(a: 0, b: SGPRsInUse - NrOfSGPRUntilSpill) * ArgStackCost.getValue() *
1943 InlineConstants::getInstrCost());
1944 adjustThreshold += static_cast<unsigned>(
1945 std::max(a: 0, b: VGPRsInUse - NrOfVGPRUntilSpill) * ArgStackCost.getValue() *
1946 InlineConstants::getInstrCost());
1947 return adjustThreshold;
1948}
1949
1950static TypeSize getCallArgsTotalAllocaSize(const CallBase *CB,
1951 const DataLayout &DL) {
1952 // If we have a pointer to a private array passed into a function
1953 // it will not be optimized out, leaving scratch usage.
1954 // This function calculates the total size in bytes of the memory that would
1955 // end in scratch if the call was not inlined.
1956 TypeSize AllocaSize = TypeSize::getFixed(ExactSize: 0);
1957 SmallPtrSet<const AllocaInst *, 8> AIVisited;
1958 for (Value *PtrArg : CB->args()) {
1959 PointerType *Ty = dyn_cast<PointerType>(Val: PtrArg->getType());
1960 if (!Ty)
1961 continue;
1962
1963 unsigned AddrSpace = Ty->getAddressSpace();
1964 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&
1965 AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)
1966 continue;
1967
1968 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: PtrArg));
1969 if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(Ptr: AI).second)
1970 continue;
1971
1972 if (auto Size = AI->getAllocationSize(DL))
1973 AllocaSize += *Size;
1974 }
1975 return AllocaSize;
1976}
1977
1978int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
1979 return BaseT::getInliningLastCallToStaticBonus() *
1980 getInliningThresholdMultiplier();
1981}
1982
1983unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
1984 unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, TTIImpl: this);
1985
1986 // Private object passed as arguments may end up in scratch usage if the call
1987 // is not inlined. Increase the inline threshold to promote inlining.
1988 TypeSize AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
1989 if (AllocaSize.getFixedValue() > 0)
1990 Threshold += ArgAllocaCost;
1991 return Threshold;
1992}
1993
1994unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
1995 const AllocaInst *AI) const {
1996
1997 // Below the cutoff, assume that the private memory objects would be
1998 // optimized
1999 auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL).getFixedValue();
2000 if (AllocaSize <= ArgAllocaCutoff)
2001 return 0;
2002
2003 // Above the cutoff, we give a cost to each private memory object
2004 // depending its size. If the array can be optimized by SROA this cost is not
2005 // added to the total-cost in the inliner cost analysis.
2006 //
2007 // We choose the total cost of the alloca such that their sum cancels the
2008 // bonus given in the threshold (ArgAllocaCost).
2009 //
2010 // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost
2011 //
2012 // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,
2013 // the single-bb bonus and the vector-bonus.
2014 //
2015 // We compensate the first two multipliers, by repeating logic from the
2016 // inliner-cost in here. The vector-bonus is 0 on AMDGPU.
2017 static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");
2018 unsigned Threshold = ArgAllocaCost * getInliningThresholdMultiplier();
2019
2020 bool SingleBB = none_of(Range&: *CB->getCalledFunction(), P: [](const BasicBlock &BB) {
2021 return BB.getTerminator()->getNumSuccessors() > 1;
2022 });
2023 if (SingleBB) {
2024 Threshold += Threshold / 2;
2025 }
2026
2027 auto ArgAllocaSize = AI->getAllocationSize(DL);
2028 if (!ArgAllocaSize)
2029 return 0;
2030
2031 // Attribute the bonus proportionally to the alloca size
2032 unsigned AllocaThresholdBonus = static_cast<unsigned>(
2033 (Threshold * ArgAllocaSize->getFixedValue()) / AllocaSize);
2034
2035 return AllocaThresholdBonus;
2036}
2037
2038void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2039 TTI::UnrollingPreferences &UP,
2040 OptimizationRemarkEmitter *ORE) const {
2041 CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
2042}
2043
2044void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2045 TTI::PeelingPreferences &PP) const {
2046 CommonTTI.getPeelingPreferences(L, SE, PP);
2047}
2048
2049int GCNTTIImpl::getTransInstrCost(TTI::TargetCostKind CostKind) const {
2050 return getQuarterRateInstrCost(CostKind);
2051}
2052
2053int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {
2054 return ST->hasFullRate64Ops()
2055 ? getFullRateInstrCost()
2056 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)
2057 : getQuarterRateInstrCost(CostKind);
2058}
2059
2060std::pair<InstructionCost, MVT>
2061GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {
2062 std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);
2063 auto Size = DL.getTypeSizeInBits(Ty);
2064 // Maximum load or store can handle 8 dwords for scalar and 4 for
2065 // vector ALU. Let's assume anything above 8 dwords is expensive
2066 // even if legal.
2067 if (Size <= 256)
2068 return Cost;
2069
2070 Cost.first += (Size + 255) / 256;
2071 return Cost;
2072}
2073
2074unsigned GCNTTIImpl::getCacheLineSize() const {
2075 if (ST->hasVmemPrefInsts() || ST->hasSmemPrefetchInsts())
2076 return ST->getDataCacheLineSize();
2077 return 0;
2078}
2079
2080unsigned GCNTTIImpl::getPrefetchDistance() const {
2081 return ST->hasPrefetch() ? 128 : 0;
2082}
2083
2084bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
2085 return AMDGPU::isFlatGlobalAddrSpace(AS);
2086}
2087
2088void GCNTTIImpl::collectKernelLaunchBounds(
2089 const Function &F,
2090 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
2091 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F);
2092 LB.push_back(Elt: {"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});
2093 LB.push_back(Elt: {"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});
2094 LB.push_back(Elt: {"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});
2095 std::pair<unsigned, unsigned> FlatWorkGroupSize =
2096 ST->getFlatWorkGroupSizes(F);
2097 LB.push_back(Elt: {"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});
2098 LB.push_back(Elt: {"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});
2099 std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);
2100 LB.push_back(Elt: {"amdgpu-waves-per-eu[0]", WavesPerEU.first});
2101 LB.push_back(Elt: {"amdgpu-waves-per-eu[1]", WavesPerEU.second});
2102}
2103
2104GCNTTIImpl::KnownIEEEMode
2105GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
2106 if (!ST->hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
2107 return KnownIEEEMode::On; // Only mode on gfx1170+
2108
2109 const Function *F = I.getFunction();
2110 if (!F)
2111 return KnownIEEEMode::Unknown;
2112
2113 Attribute IEEEAttr = F->getFnAttribute(Kind: "amdgpu-ieee");
2114 if (IEEEAttr.isValid())
2115 return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off;
2116
2117 return AMDGPU::isShader(CC: F->getCallingConv()) ? KnownIEEEMode::Off
2118 : KnownIEEEMode::On;
2119}
2120
2121InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2122 Align Alignment,
2123 unsigned AddressSpace,
2124 TTI::TargetCostKind CostKind,
2125 TTI::OperandValueInfo OpInfo,
2126 const Instruction *I) const {
2127 if (VectorType *VecTy = dyn_cast<VectorType>(Val: Src)) {
2128 if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
2129 CostKind != TTI::TCK_Latency &&
2130 VecTy->getElementType()->isIntegerTy(BitWidth: 8)) {
2131 return divideCeil(Numerator: DL.getTypeSizeInBits(Ty: VecTy) - 1,
2132 Denominator: getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace));
2133 }
2134 }
2135 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
2136 OpInfo, I);
2137}
2138
2139unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
2140 if (VectorType *VecTy = dyn_cast<VectorType>(Val: Tp)) {
2141 if (VecTy->getElementType()->isIntegerTy(BitWidth: 8)) {
2142 unsigned ElementCount = VecTy->getElementCount().getFixedValue();
2143 return divideCeil(Numerator: ElementCount - 1, Denominator: 4);
2144 }
2145 }
2146 return BaseT::getNumberOfParts(Tp);
2147}
2148
2149ValueUniformity GCNTTIImpl::getValueUniformity(const Value *V) const {
2150 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
2151 switch (Intrinsic->getIntrinsicID()) {
2152 case Intrinsic::amdgcn_wave_shuffle:
2153 return ValueUniformity::Custom;
2154 default:
2155 break;
2156 }
2157 }
2158
2159 if (isAlwaysUniform(V))
2160 return ValueUniformity::AlwaysUniform;
2161
2162 if (isSourceOfDivergence(V))
2163 return ValueUniformity::NeverUniform;
2164
2165 return ValueUniformity::Default;
2166}
2167
2168InstructionCost GCNTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2169 StackOffset BaseOffset,
2170 bool HasBaseReg, int64_t Scale,
2171 unsigned AddrSpace) const {
2172 if (HasBaseReg && Scale != 0) {
2173 // gfx1250+ can fold base+scale*index when scale matches the memory access
2174 // size (scale_offset bit). Supported for flat/global/constant/scratch
2175 // (VMEM, max 128 bits) and constant_32bit (SMRD, capped to 128 bits here).
2176 if (getST()->hasScaleOffset() && Ty && Ty->isSized() &&
2177 (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace) ||
2178 AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
2179 AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)) {
2180 TypeSize StoreSize = getDataLayout().getTypeStoreSize(Ty);
2181 if (TypeSize::isKnownLE(LHS: StoreSize, RHS: TypeSize::getFixed(ExactSize: 16)) &&
2182 static_cast<int64_t>(StoreSize.getFixedValue()) == Scale)
2183 return 0;
2184 }
2185 return 1;
2186 }
2187 return BaseT::getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
2188 AddrSpace);
2189}
2190
2191bool GCNTTIImpl::isLSRCostLess(const TTI::LSRCost &A,
2192 const TTI::LSRCost &B) const {
2193 // Favor lower per-iteration work over preheader/setup costs.
2194 // AMDGPU lacks rich addressing modes, so ScaleCost is folded into the
2195 // effective instruction count (base+scale*index requires a separate ADD).
2196 unsigned EffInsnsA = A.Insns + A.ScaleCost;
2197 unsigned EffInsnsB = B.Insns + B.ScaleCost;
2198
2199 return std::tie(args&: EffInsnsA, args: A.NumIVMuls, args: A.AddRecCost, args: A.NumBaseAdds,
2200 args: A.SetupCost, args: A.ImmCost, args: A.NumRegs) <
2201 std::tie(args&: EffInsnsB, args: B.NumIVMuls, args: B.AddRecCost, args: B.NumBaseAdds,
2202 args: B.SetupCost, args: B.ImmCost, args: B.NumRegs);
2203}
2204
2205bool GCNTTIImpl::isNumRegsMajorCostOfLSR() const {
2206 // isLSRCostLess de-prioritizes register count; keep consistent.
2207 return false;
2208}
2209
2210bool GCNTTIImpl::shouldDropLSRSolutionIfLessProfitable() const {
2211 // Prefer the baseline when LSR cannot clearly reduce per-iteration work.
2212 return true;
2213}
2214
2215bool GCNTTIImpl::isUniform(const Instruction *I,
2216 const SmallBitVector &UniformArgs) const {
2217 const IntrinsicInst *Intrinsic = cast<IntrinsicInst>(Val: I);
2218 switch (Intrinsic->getIntrinsicID()) {
2219 case Intrinsic::amdgcn_wave_shuffle:
2220 // wave_shuffle(Value, Index): result is uniform when either Value or Index
2221 // is uniform.
2222 return UniformArgs[0] || UniformArgs[1];
2223 default:
2224 llvm_unreachable("unexpected intrinsic in isUniform");
2225 }
2226}
2227