1//===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements a TargetTransformInfo analysis pass specific to the
11// AMDGPU target machine. It uses the target's detailed information to provide
12// more precise answers to certain TTI queries, while letting the target
13// independent and default TTI implementations handle the rest.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPUTargetTransformInfo.h"
18#include "AMDGPUSubtarget.h"
19#include "AMDGPUTargetMachine.h"
20#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21#include "SIModeRegisterDefaults.h"
22#include "llvm/ADT/SmallBitVector.h"
23#include "llvm/Analysis/InlineCost.h"
24#include "llvm/Analysis/LoopInfo.h"
25#include "llvm/Analysis/ValueTracking.h"
26#include "llvm/CodeGen/Analysis.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/IRBuilder.h"
29#include "llvm/IR/IntrinsicsAMDGPU.h"
30#include "llvm/IR/PatternMatch.h"
31#include "llvm/Support/KnownBits.h"
32#include <optional>
33
34using namespace llvm;
35
36#define DEBUG_TYPE "AMDGPUtti"
37
38static bool dependsOnLocalPhi(const Loop *L, const Value *Cond,
39 unsigned Depth = 0) {
40 const Instruction *I = dyn_cast<Instruction>(Val: Cond);
41 if (!I)
42 return false;
43
44 if (!L->contains(Inst: I))
45 return false;
46 for (const Value *V : I->operand_values()) {
47 if (const PHINode *PHI = dyn_cast<PHINode>(Val: V)) {
48 if (llvm::none_of(Range: L->getSubLoops(), P: [PHI](const Loop* SubLoop) {
49 return SubLoop->contains(Inst: PHI); }))
50 return true;
51 } else if (Depth < 10 && dependsOnLocalPhi(L, Cond: V, Depth: Depth+1))
52 return true;
53 }
54 return false;
55}
56
57AMDGPUTTIImpl::AMDGPUTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
58 : BaseT(TM, F.getDataLayout()), CLOpts(TM->getCLOpts()),
59 TargetTriple(F.getParent()->getTargetTriple()),
60 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
61 TLI(ST->getTargetLowering()) {}
62
63void AMDGPUTTIImpl::getUnrollingPreferences(
64 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
65 OptimizationRemarkEmitter *ORE) const {
66 const Function &F = *L->getHeader()->getParent();
67 UP.Threshold = static_cast<unsigned>(
68 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-unroll-threshold", Default: 300));
69 UP.PartialThreshold = static_cast<unsigned>(
70 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-partial-unroll-threshold", Default: 150));
71 UP.MaxCount = std::numeric_limits<unsigned>::max();
72 UP.Partial = true;
73
74 // Conditional branch in a loop back edge needs 3 additional exec
75 // manipulations in average.
76 UP.BEInsns += 3;
77
78 // We want to run unroll even for the loops which have been vectorized.
79 UP.UnrollVectorizedLoop = true;
80
81 // Enable runtime unrolling for loops whose trip count is not known at
82 // compile time.
83 UP.Runtime = true;
84
85 // Maximum alloca size than can fit registers. Reserve 16 registers.
86 const unsigned MaxAlloca = (256 - 16) * 4;
87 unsigned ThresholdPrivate = CLOpts.unroll_threshold_private;
88 unsigned ThresholdLocal = CLOpts.unroll_threshold_local;
89
90 // If this loop has the amdgpu.loop.unroll.threshold metadata we will use the
91 // provided threshold value as the default for Threshold
92 if (MDNode *LoopUnrollThreshold =
93 findOptionMDForLoop(TheLoop: L, Name: "amdgpu.loop.unroll.threshold")) {
94 if (LoopUnrollThreshold->getNumOperands() == 2) {
95 ConstantInt *MetaThresholdValue = mdconst::extract_or_null<ConstantInt>(
96 MD: LoopUnrollThreshold->getOperand(I: 1));
97 if (MetaThresholdValue) {
98 // We will also use the supplied value for PartialThreshold for now.
99 // We may introduce additional metadata if it becomes necessary in the
100 // future.
101 UP.Threshold =
102 static_cast<unsigned>(MetaThresholdValue->getSExtValue());
103 UP.PartialThreshold = UP.Threshold;
104 ThresholdPrivate = std::min(a: ThresholdPrivate, b: UP.Threshold);
105 ThresholdLocal = std::min(a: ThresholdLocal, b: UP.Threshold);
106 }
107 }
108 }
109
110 unsigned MaxBoost = std::max(a: ThresholdPrivate, b: ThresholdLocal);
111 for (const BasicBlock *BB : L->getBlocks()) {
112 const DataLayout &DL = BB->getDataLayout();
113 unsigned LocalGEPsSeen = 0;
114
115 if (llvm::any_of(Range: L->getSubLoops(), P: [BB](const Loop* SubLoop) {
116 return SubLoop->contains(BB); }))
117 continue; // Block belongs to an inner loop.
118
119 for (const Instruction &I : *BB) {
120 // Unroll a loop which contains an "if" statement whose condition
121 // defined by a PHI belonging to the loop. This may help to eliminate
122 // if region and potentially even PHI itself, saving on both divergence
123 // and registers used for the PHI.
124 // Add a small bonus for each of such "if" statements.
125 if (const CondBrInst *Br = dyn_cast<CondBrInst>(Val: &I)) {
126 if (UP.Threshold < MaxBoost) {
127 BasicBlock *Succ0 = Br->getSuccessor(i: 0);
128 BasicBlock *Succ1 = Br->getSuccessor(i: 1);
129 if ((L->contains(BB: Succ0) && L->isLoopExiting(BB: Succ0)) ||
130 (L->contains(BB: Succ1) && L->isLoopExiting(BB: Succ1)))
131 continue;
132 if (dependsOnLocalPhi(L, Cond: Br->getCondition())) {
133 UP.Threshold += CLOpts.unroll_threshold_if;
134 LLVM_DEBUG(dbgs() << "Set unroll threshold " << UP.Threshold
135 << " for loop:\n"
136 << *L << " due to " << *Br << '\n');
137 if (UP.Threshold >= MaxBoost)
138 return;
139 }
140 }
141 continue;
142 }
143
144 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: &I);
145 if (!GEP)
146 continue;
147
148 unsigned AS = GEP->getAddressSpace();
149 unsigned Threshold = 0;
150 if (AS == AMDGPUAS::PRIVATE_ADDRESS)
151 Threshold = ThresholdPrivate;
152 else if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS)
153 Threshold = ThresholdLocal;
154 else
155 continue;
156
157 if (UP.Threshold >= Threshold)
158 continue;
159
160 if (AS == AMDGPUAS::PRIVATE_ADDRESS) {
161 const Value *Ptr = GEP->getPointerOperand();
162 const AllocaInst *Alloca =
163 dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: Ptr));
164 if (!Alloca || !Alloca->isStaticAlloca())
165 continue;
166 auto AllocaSize = Alloca->getAllocationSize(DL);
167 if (!AllocaSize || AllocaSize->getFixedValue() > MaxAlloca)
168 continue;
169 } else if (AS == AMDGPUAS::LOCAL_ADDRESS ||
170 AS == AMDGPUAS::REGION_ADDRESS) {
171 LocalGEPsSeen++;
172 // Inhibit unroll for local memory if we have seen addressing not to
173 // a variable, most likely we will be unable to combine it.
174 // Do not unroll too deep inner loops for local memory to give a chance
175 // to unroll an outer loop for a more important reason.
176 if (LocalGEPsSeen > 1 || L->getLoopDepth() > 2 ||
177 (!isa<GlobalVariable>(Val: GEP->getPointerOperand()) &&
178 !isa<Argument>(Val: GEP->getPointerOperand())))
179 continue;
180 LLVM_DEBUG(dbgs() << "Allow unroll runtime for loop:\n"
181 << *L << " due to LDS use.\n");
182 UP.Runtime = CLOpts.unroll_runtime_local;
183 }
184
185 // Check if GEP depends on a value defined by this loop itself.
186 bool HasLoopDef = false;
187 for (const Value *Op : GEP->operands()) {
188 const Instruction *Inst = dyn_cast<Instruction>(Val: Op);
189 if (!Inst || L->isLoopInvariant(V: Op))
190 continue;
191
192 if (llvm::any_of(Range: L->getSubLoops(), P: [Inst](const Loop* SubLoop) {
193 return SubLoop->contains(Inst); }))
194 continue;
195 HasLoopDef = true;
196 break;
197 }
198 if (!HasLoopDef)
199 continue;
200
201 // We want to do whatever we can to limit the number of alloca
202 // instructions that make it through to the code generator. allocas
203 // require us to use indirect addressing, which is slow and prone to
204 // compiler bugs. If this loop does an address calculation on an
205 // alloca ptr, then we want to use a higher than normal loop unroll
206 // threshold. This will give SROA a better chance to eliminate these
207 // allocas.
208 //
209 // We also want to have more unrolling for local memory to let ds
210 // instructions with different offsets combine.
211 //
212 // Don't use the maximum allowed value here as it will make some
213 // programs way too big.
214 UP.Threshold = Threshold;
215 LLVM_DEBUG(dbgs() << "Set unroll threshold " << Threshold
216 << " for loop:\n"
217 << *L << " due to " << *GEP << '\n');
218 if (UP.Threshold >= MaxBoost)
219 return;
220 }
221
222 // If we got a GEP in a small BB from inner loop then increase max trip
223 // count to analyze for better estimation cost in unroll
224 if (L->isInnermost() && BB->size() < CLOpts.unroll_max_block_to_analyze)
225 UP.MaxIterationsCountToAnalyze = 32;
226 }
227}
228
229void AMDGPUTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
230 TTI::PeelingPreferences &PP) const {
231 BaseT::getPeelingPreferences(L, SE, PP);
232}
233
234uint64_t AMDGPUTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
235 return 1024;
236}
237
238GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
239 : BaseT(TM, F.getDataLayout()), CLOpts(TM->getCLOpts()),
240 ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
241 TLI(ST->getTargetLowering()), CommonTTI(TM, F),
242 IsGraphics(AMDGPU::isGraphics(CC: F.getCallingConv())) {
243 SIModeRegisterDefaults Mode(F, *ST);
244 HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
245}
246
247bool GCNTTIImpl::hasBranchDivergence(const Function *F) const {
248 return !F || !ST->isSingleLaneExecution(Kernel: *F);
249}
250
251unsigned GCNTTIImpl::getNumberOfRegisters(unsigned RCID) const {
252 // NB: RCID is not an RCID. In fact it is 0 or 1 for scalar or vector
253 // registers. See getRegisterClassForType for the implementation.
254 // In this case vector registers are not vector in terms of
255 // VGPRs, but those which can hold multiple values.
256
257 // This is really the number of registers to fill when vectorizing /
258 // interleaving loops, so we lie to avoid trying to use all registers.
259 return 4;
260}
261
262TypeSize
263GCNTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
264 switch (K) {
265 case TargetTransformInfo::RGK_Scalar:
266 return TypeSize::getFixed(ExactSize: 32);
267 case TargetTransformInfo::RGK_FixedWidthVector:
268 return TypeSize::getFixed(
269 ExactSize: (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()) ? 128
270 : ST->hasAnyPackedFP32Ops() ? 64
271 : 32);
272 case TargetTransformInfo::RGK_ScalableVector:
273 return TypeSize::getScalable(MinimumSize: 0);
274 }
275 llvm_unreachable("Unsupported register kind");
276}
277
278unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
279 return 32;
280}
281
282unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
283 if (Opcode == Instruction::Load || Opcode == Instruction::Store)
284 return 32 * 4 / ElemWidth;
285 // For a given width return the max 0number of elements that can be combined
286 // into a wider bit value:
287 return (ElemWidth == 8 && ST->has16BitInsts()) ? 4
288 : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
289 : (ElemWidth == 32 && ST->hasAnyPackedFP32Ops()) ? 2
290 : (ElemWidth == 64 &&
291 (ST->hasAnyPackedFP64Ops() || ST->hasAnyPackedU64Ops()))
292 ? 2
293 : 1;
294}
295
296bool GCNTTIImpl::preferSLPInstCountCheck() const {
297 // The integer inst-count heuristic causes regressions on gfx94x and gfx950
298 // because 2-element vector trees that pass the scalar/vector instruction
299 // count comparison still widen scalar moves (e.g. v_mov_b32 to v_mov_b64)
300 // after codegen, increasing register pressure and throughput cost without
301 // reducing the total instruction count.
302 return !ST->hasGFX940Insts() && !ST->hasGFX950Insts();
303}
304
305unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
306 unsigned ChainSizeInBytes,
307 VectorType *VecTy) const {
308 unsigned VecRegBitWidth = VF * LoadSize;
309 if (VecRegBitWidth > 128 && VecTy->getScalarSizeInBits() < 32)
310 // TODO: Support element-size less than 32bit?
311 return 128 / LoadSize;
312
313 return VF;
314}
315
316unsigned GCNTTIImpl::getStoreVectorFactor(unsigned VF, unsigned StoreSize,
317 unsigned ChainSizeInBytes,
318 VectorType *VecTy) const {
319 unsigned VecRegBitWidth = VF * StoreSize;
320 if (VecRegBitWidth > 128)
321 return 128 / StoreSize;
322
323 return VF;
324}
325
326unsigned GCNTTIImpl::getLoadStoreVecRegBitWidth(unsigned AddrSpace) const {
327 if (AddrSpace == AMDGPUAS::GLOBAL_ADDRESS ||
328 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS ||
329 AddrSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
330 AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
331 AddrSpace == AMDGPUAS::BUFFER_RESOURCE ||
332 AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) {
333 return 512;
334 }
335
336 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)
337 return 8 * ST->getMaxPrivateElementSize();
338
339 // Common to flat, global, local and region. Assume for unknown addrspace.
340 return 128;
341}
342
343bool GCNTTIImpl::isLegalToVectorizeMemChain(unsigned ChainSizeInBytes,
344 Align Alignment,
345 unsigned AddrSpace) const {
346 // We allow vectorization of flat stores, even though we may need to decompose
347 // them later if they may access private memory. We don't have enough context
348 // here, and legalization can handle it.
349 if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
350 return (Alignment >= 4 || ST->hasUnalignedScratchAccessEnabled()) &&
351 ChainSizeInBytes <= ST->getMaxPrivateElementSize();
352 }
353 return true;
354}
355
356bool GCNTTIImpl::isLegalToVectorizeLoadChain(unsigned ChainSizeInBytes,
357 Align Alignment,
358 unsigned AddrSpace) const {
359 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
360}
361
362bool GCNTTIImpl::isLegalToVectorizeStoreChain(unsigned ChainSizeInBytes,
363 Align Alignment,
364 unsigned AddrSpace) const {
365 return isLegalToVectorizeMemChain(ChainSizeInBytes, Alignment, AddrSpace);
366}
367
368uint64_t GCNTTIImpl::getMaxMemIntrinsicInlineSizeThreshold() const {
369 return 1024;
370}
371
372Type *GCNTTIImpl::getMemcpyLoopLoweringType(
373 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
374 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
375 std::optional<uint32_t> AtomicElementSize) const {
376
377 if (AtomicElementSize)
378 return Type::getIntNTy(C&: Context, N: *AtomicElementSize * 8);
379
380 // 16-byte accesses achieve the highest copy throughput.
381 // If the operation has a fixed known length that is large enough, it is
382 // worthwhile to return an even wider type and let legalization lower it into
383 // multiple accesses, effectively unrolling the memcpy loop.
384 // We also rely on legalization to decompose into smaller accesses for
385 // subtargets and address spaces where it is necessary.
386 //
387 // Don't unroll if Length is not a constant, since unrolling leads to worse
388 // performance for length values that are smaller or slightly larger than the
389 // total size of the type returned here. Mitigating that would require a more
390 // complex lowering for variable-length memcpy and memmove.
391 unsigned I32EltsInVector = 4;
392 if (CLOpts.memcpy_loop_unroll > 0 && isa<ConstantInt>(Val: Length))
393 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context),
394 NumElts: CLOpts.memcpy_loop_unroll * I32EltsInVector);
395
396 return FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: I32EltsInVector);
397}
398
399void GCNTTIImpl::getMemcpyLoopResidualLoweringType(
400 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
401 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
402 Align SrcAlign, Align DestAlign,
403 std::optional<uint32_t> AtomicCpySize) const {
404
405 if (AtomicCpySize)
406 BaseT::getMemcpyLoopResidualLoweringType(
407 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
408 DestAlign, AtomicCpySize);
409
410 Type *I32x4Ty = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: Context), NumElts: 4);
411 while (RemainingBytes >= 16) {
412 OpsOut.push_back(Elt: I32x4Ty);
413 RemainingBytes -= 16;
414 }
415
416 Type *I64Ty = Type::getInt64Ty(C&: Context);
417 while (RemainingBytes >= 8) {
418 OpsOut.push_back(Elt: I64Ty);
419 RemainingBytes -= 8;
420 }
421
422 Type *I32Ty = Type::getInt32Ty(C&: Context);
423 while (RemainingBytes >= 4) {
424 OpsOut.push_back(Elt: I32Ty);
425 RemainingBytes -= 4;
426 }
427
428 Type *I16Ty = Type::getInt16Ty(C&: Context);
429 while (RemainingBytes >= 2) {
430 OpsOut.push_back(Elt: I16Ty);
431 RemainingBytes -= 2;
432 }
433
434 Type *I8Ty = Type::getInt8Ty(C&: Context);
435 while (RemainingBytes) {
436 OpsOut.push_back(Elt: I8Ty);
437 --RemainingBytes;
438 }
439}
440
441unsigned GCNTTIImpl::getMaxInterleaveFactor(ElementCount VF,
442 bool HasUnorderedReductions) const {
443 // Disable unrolling if the loop is not vectorized.
444 // TODO: Enable this again.
445 if (VF.isScalar())
446 return 1;
447
448 return 8;
449}
450
451bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
452 MemIntrinsicInfo &Info) const {
453 switch (Inst->getIntrinsicID()) {
454 case Intrinsic::amdgcn_ds_ordered_add:
455 case Intrinsic::amdgcn_ds_ordered_swap: {
456 auto *Ordering = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 2));
457 auto *Volatile = dyn_cast<ConstantInt>(Val: Inst->getArgOperand(i: 4));
458 if (!Ordering || !Volatile)
459 return false; // Invalid.
460
461 uint64_t OrderingVal = Ordering->getZExtValue();
462 if (!isValidAtomicOrdering(I: OrderingVal))
463 return false;
464
465 Info.PtrVal = Inst->getArgOperand(i: 0);
466 Info.Ordering = static_cast<AtomicOrdering>(OrderingVal);
467 Info.ReadMem = true;
468 Info.WriteMem = true;
469 Info.IsVolatile = !Volatile->isZero();
470 return true;
471 }
472 default:
473 return false;
474 }
475}
476
477/// \returns true if \p FMul and its single fadd/fsub user \p FAddSub are
478/// expected to fuse during instruction selection. \p Ty is the type the fused
479/// operation runs on.
480static bool canFuseFMulWithFAddSub(const SITargetLowering &TLI, Type *Ty,
481 const Instruction *FMul,
482 const Instruction *FAddSub) {
483 assert((FAddSub->getOpcode() == Instruction::FAdd ||
484 FAddSub->getOpcode() == Instruction::FSub) &&
485 "Expected an fadd or an fsub");
486
487 // The mad forms fuse exactly without fast-math flags but flush denormals.
488 // An fma forms only when it is not slower than the separate operations.
489 const Function &F = *FAddSub->getFunction();
490 const bool HasFMAD = TLI.isFMADLegal(F, Ty);
491 const bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(F, Ty);
492 if (!HasFMAD && !HasFMA)
493 return false;
494
495 // Without a mad the pair fuses only when both carry contract.
496 return HasFMAD || (FAddSub->hasAllowContract() && FMul->hasAllowContract());
497}
498
499/// An fma holds one multiply, so only one fmul operand fuses with \p FAddSub.
500static const Instruction *getFusedFMul(const SITargetLowering &TLI, Type *Ty,
501 const Instruction *FAddSub) {
502 for (const Value *Op : FAddSub->operands()) {
503 const auto *FMul = dyn_cast<Instruction>(Val: Op);
504 if (FMul && FMul->getOpcode() == Instruction::FMul && FMul->hasOneUse() &&
505 canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub))
506 return FMul;
507 }
508 return nullptr;
509}
510
511static bool isFusedFMul(const SITargetLowering &TLI, Type *Ty,
512 const Instruction *FMul, const Instruction *FAddSub) {
513 const Instruction *Fused = getFusedFMul(TLI, Ty, FAddSub);
514 if (Fused == FMul)
515 return true;
516 // (a * b + c * d) + e becomes fma(a, b, fma(c, d, e)) if the outer fadd has
517 // reassoc.
518 if (!Fused || FAddSub->getOpcode() != Instruction::FAdd ||
519 !FAddSub->hasOneUse())
520 return false;
521 const auto *Outer = dyn_cast<BinaryOperator>(Val: *FAddSub->user_begin());
522 return Outer && Outer->getOpcode() == Instruction::FAdd &&
523 Outer->hasAllowReassoc() &&
524 canFuseFMulWithFAddSub(TLI, Ty, FMul, FAddSub: Outer);
525}
526
527InstructionCost GCNTTIImpl::getArithmeticInstrCost(
528 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
529 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
530 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
531
532 // Legalize the type.
533 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
534 int ISD = TLI->InstructionOpcodeToISD(Opcode);
535
536 // Because we don't have any legal vector operations, but the legal types, we
537 // need to account for split vectors.
538 unsigned NElts = LT.second.isVector() ?
539 LT.second.getVectorNumElements() : 1;
540
541 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
542
543 switch (ISD) {
544 case ISD::SHL:
545 case ISD::SRL:
546 case ISD::SRA:
547 if (SLT == MVT::i64)
548 return get64BitInstrCost(CostKind) * LT.first * NElts;
549
550 if (ST->has16BitInsts() && SLT == MVT::i16)
551 NElts = (NElts + 1) / 2;
552
553 // i32
554 return getFullRateInstrCost() * LT.first * NElts;
555 case ISD::ADD:
556 case ISD::SUB:
557 if (SLT == MVT::i64 && ST->hasAnyPackedU64Ops())
558 NElts = (NElts + 1) / 2;
559 [[fallthrough]];
560 case ISD::AND:
561 case ISD::OR:
562 case ISD::XOR:
563 if (SLT == MVT::i64) {
564 // and, or and xor are typically split into 2 VALU instructions.
565 return 2 * getFullRateInstrCost() * LT.first * NElts;
566 }
567
568 if (ST->has16BitInsts() && SLT == MVT::i16)
569 NElts = (NElts + 1) / 2;
570
571 return LT.first * NElts * getFullRateInstrCost();
572 case ISD::MUL: {
573 const int QuarterRateCost = getQuarterRateInstrCost(CostKind);
574 if (SLT == MVT::i64) {
575 const int FullRateCost = getFullRateInstrCost();
576 return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts;
577 }
578
579 if (ST->has16BitInsts() && SLT == MVT::i16)
580 NElts = (NElts + 1) / 2;
581
582 // i32
583 return QuarterRateCost * NElts * LT.first;
584 }
585 case ISD::FMUL:
586 // Check possible fuse {fadd|fsub}(a,fmul(b,c)) and return zero cost for
587 // fmul(b,c) supposing the fadd|fsub will get estimated cost for the whole
588 // fused operation. A vector fmul priced with a scalar context fuses with
589 // the scalar fadd|fsub only when it is emitted lane by lane.
590 if (CtxI && CtxI->hasOneUse() &&
591 (CtxI->getType() == Ty ||
592 getMaximumVF(ElemWidth: Ty->getScalarSizeInBits(), Opcode) == 1)) {
593 const auto *FAddSub = dyn_cast<BinaryOperator>(Val: *CtxI->user_begin());
594 if (FAddSub &&
595 (FAddSub->getOpcode() == Instruction::FAdd ||
596 FAddSub->getOpcode() == Instruction::FSub) &&
597 isFusedFMul(TLI: *TLI, Ty, FMul: CtxI, FAddSub))
598 return TargetTransformInfo::TCC_Free;
599 }
600 [[fallthrough]];
601 case ISD::FADD:
602 case ISD::FSUB:
603 if (ST->hasAnyPackedFP32Ops() && SLT == MVT::f32)
604 NElts = (NElts + 1) / 2;
605 if (ST->hasBF16PackedInsts() && SLT == MVT::bf16)
606 NElts = (NElts + 1) / 2;
607 if (SLT == MVT::f64) {
608 if (ST->hasAnyPackedFP64Ops())
609 NElts = (NElts + 1) / 2;
610 return LT.first * NElts * get64BitInstrCost(CostKind);
611 }
612
613 if (ST->has16BitInsts() && SLT == MVT::f16)
614 NElts = (NElts + 1) / 2;
615
616 if (SLT == MVT::f32 || SLT == MVT::f16 || SLT == MVT::bf16)
617 return LT.first * NElts * getFullRateInstrCost();
618 break;
619 case ISD::FDIV:
620 case ISD::FREM:
621 // FIXME: frem should be handled separately. The fdiv in it is most of it,
622 // but the current lowering is also not entirely correct.
623 if (SLT == MVT::f64) {
624 int Cost = 7 * get64BitInstrCost(CostKind) +
625 getQuarterRateInstrCost(CostKind) +
626 3 * getHalfRateInstrCost(CostKind);
627 // Add cost of workaround.
628 if (!ST->hasUsableDivScaleConditionOutput())
629 Cost += 3 * getFullRateInstrCost();
630
631 return LT.first * Cost * NElts;
632 }
633
634 if (!Args.empty() && match(V: Args[0], P: PatternMatch::m_FPOne())) {
635 // TODO: This is more complicated, unsafe flags etc.
636 if ((SLT == MVT::f32 && !HasFP32Denormals) ||
637 (SLT == MVT::f16 && ST->has16BitInsts())) {
638 return LT.first * getTransInstrCost(CostKind) * NElts;
639 }
640 }
641
642 if (SLT == MVT::f16 && ST->has16BitInsts()) {
643 // 2 x v_cvt_f32_f16
644 // f32 rcp
645 // f32 fmul
646 // v_cvt_f16_f32
647 // f16 div_fixup
648 int Cost = 4 * getFullRateInstrCost() + 2 * getTransInstrCost(CostKind);
649 return LT.first * Cost * NElts;
650 }
651
652 if (SLT == MVT::f32 && (CtxI && CtxI->hasApproxFunc())) {
653 // Fast unsafe fdiv lowering:
654 // f32 rcp
655 // f32 fmul
656 int Cost = getTransInstrCost(CostKind) + getFullRateInstrCost();
657 return LT.first * Cost * NElts;
658 }
659
660 if (SLT == MVT::f32 || SLT == MVT::f16) {
661 // 4 more v_cvt_* insts without f16 insts support
662 int Cost = (SLT == MVT::f16 ? 14 : 10) * getFullRateInstrCost() +
663 1 * getTransInstrCost(CostKind);
664
665 if (!HasFP32Denormals) {
666 // FP mode switches.
667 Cost += 2 * getFullRateInstrCost();
668 }
669
670 return LT.first * NElts * Cost;
671 }
672 break;
673 case ISD::FNEG:
674 // Use the backend' estimation. If fneg is not free each element will cost
675 // one additional instruction.
676 return TLI->isFNegFree(VT: SLT) ? 0 : NElts;
677 default:
678 break;
679 }
680
681 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
682 Args, CtxI);
683}
684
685// Return true if there's a potential benefit from using v2f16/v2i16
686// instructions for an intrinsic, even if it requires nontrivial legalization.
687static bool intrinsicHasPackedVectorBenefit(Intrinsic::ID ID) {
688 switch (ID) {
689 case Intrinsic::fma:
690 case Intrinsic::fmuladd:
691 case Intrinsic::copysign:
692 case Intrinsic::minimumnum:
693 case Intrinsic::maximumnum:
694 case Intrinsic::canonicalize:
695 // There's a small benefit to using vector ops in the legalized code.
696 case Intrinsic::round:
697 case Intrinsic::uadd_sat:
698 case Intrinsic::usub_sat:
699 case Intrinsic::sadd_sat:
700 case Intrinsic::ssub_sat:
701 case Intrinsic::abs:
702 return true;
703 default:
704 return false;
705 }
706}
707
708InstructionCost
709GCNTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
710 TTI::TargetCostKind CostKind) const {
711 switch (ICA.getID()) {
712 case Intrinsic::fabs:
713 // Free source modifier in the common case.
714 return 0;
715 case Intrinsic::amdgcn_workitem_id_x:
716 case Intrinsic::amdgcn_workitem_id_y:
717 case Intrinsic::amdgcn_workitem_id_z:
718 // TODO: If hasPackedTID, or if the calling context is not an entry point
719 // there may be a bit instruction.
720 return 0;
721 case Intrinsic::amdgcn_workgroup_id_x:
722 case Intrinsic::amdgcn_workgroup_id_y:
723 case Intrinsic::amdgcn_workgroup_id_z:
724 case Intrinsic::amdgcn_lds_kernel_id:
725 case Intrinsic::amdgcn_dispatch_ptr:
726 case Intrinsic::amdgcn_dispatch_id:
727 case Intrinsic::amdgcn_implicitarg_ptr:
728 case Intrinsic::amdgcn_queue_ptr:
729 // Read from an argument register.
730 return 0;
731 default:
732 break;
733 }
734
735 Type *RetTy = ICA.getReturnType();
736
737 Intrinsic::ID IID = ICA.getID();
738 switch (IID) {
739 case Intrinsic::exp:
740 case Intrinsic::exp2:
741 case Intrinsic::exp10: {
742 // Legalize the type.
743 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
744 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
745 unsigned NElts =
746 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
747
748 if (SLT == MVT::f64) {
749 unsigned NumOps = 20;
750 if (IID == Intrinsic::exp)
751 ++NumOps;
752 else if (IID == Intrinsic::exp10)
753 NumOps += 3;
754
755 return LT.first * NElts * NumOps * get64BitInstrCost(CostKind);
756 }
757
758 if (SLT == MVT::f32) {
759 unsigned NumFullRateOps = 0;
760 // v_exp_f32 (transcendental).
761 unsigned NumTransOps = 1;
762
763 if (!ICA.getFlags().approxFunc() && IID != Intrinsic::exp2) {
764 // Non-AFN exp/exp10: range reduction + v_exp_f32 + ldexp +
765 // overflow/underflow checks (lowerFEXP). Denorm is also handled.
766 // FMA preamble: ~13 full-rate ops; non-FMA: ~17.
767 NumFullRateOps = ST->hasFastFMAF32() ? 13 : 17;
768 } else {
769 if (IID == Intrinsic::exp) {
770 // lowerFEXPUnsafe: fmul (base conversion) + v_exp_f32.
771 NumFullRateOps = 1;
772 } else if (IID == Intrinsic::exp10) {
773 // lowerFEXP10Unsafe: 3 fmul + 2 v_exp_f32 (double-exp2).
774 NumFullRateOps = 3;
775 NumTransOps = 2;
776 }
777 // Denorm scaling adds setcc + select + fadd + select + fmul.
778 if (HasFP32Denormals)
779 NumFullRateOps += 5;
780 }
781
782 InstructionCost Cost = NumFullRateOps * getFullRateInstrCost() +
783 NumTransOps * getTransInstrCost(CostKind);
784 return LT.first * NElts * Cost;
785 }
786
787 break;
788 }
789 case Intrinsic::log:
790 case Intrinsic::log2:
791 case Intrinsic::log10: {
792 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
793 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
794 unsigned NElts =
795 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
796
797 if (SLT == MVT::f32) {
798 unsigned NumFullRateOps = 0;
799
800 if (IID == Intrinsic::log2) {
801 // LowerFLOG2: just v_log_f32.
802 } else if (ICA.getFlags().approxFunc()) {
803 // LowerFLOGUnsafe: v_log_f32 + fmul (base conversion).
804 NumFullRateOps = 1;
805 } else {
806 // LowerFLOGCommon non-AFN: v_log_f32 + extended-precision
807 // multiply + finite check.
808 NumFullRateOps = ST->hasFastFMAF32() ? 8 : 11;
809 }
810
811 if (HasFP32Denormals)
812 NumFullRateOps += 5;
813
814 InstructionCost Cost =
815 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
816 return LT.first * NElts * Cost;
817 }
818
819 break;
820 }
821 case Intrinsic::sin:
822 case Intrinsic::cos: {
823 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
824 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
825 unsigned NElts =
826 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
827
828 if (SLT == MVT::f32) {
829 // LowerTrig: fmul(1/2pi) + v_sin/v_cos.
830 unsigned NumFullRateOps = ST->hasTrigReducedRange() ? 2 : 1;
831
832 InstructionCost Cost =
833 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
834 return LT.first * NElts * Cost;
835 }
836
837 break;
838 }
839 case Intrinsic::sqrt: {
840 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
841 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
842 unsigned NElts =
843 LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
844
845 if (SLT == MVT::f32) {
846 unsigned NumFullRateOps = 0;
847
848 if (!ICA.getFlags().approxFunc()) {
849 // lowerFSQRTF32 non-AFN: v_sqrt_f32 + refinement + scale fixup.
850 NumFullRateOps = HasFP32Denormals ? 17 : 16;
851 }
852
853 InstructionCost Cost =
854 NumFullRateOps * getFullRateInstrCost() + getTransInstrCost(CostKind);
855 return LT.first * NElts * Cost;
856 }
857
858 break;
859 }
860 default:
861 break;
862 }
863
864 if (!intrinsicHasPackedVectorBenefit(ID: ICA.getID()))
865 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
866
867 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
868 MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy;
869 unsigned NElts = LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
870
871 if ((ST->hasVOP3PInsts() &&
872 (SLT == MVT::f16 || SLT == MVT::i16 ||
873 (SLT == MVT::bf16 && ST->hasBF16PackedInsts()))) ||
874 (ST->hasAnyPackedFP64Ops() && SLT == MVT::f64) ||
875 (ST->hasAnyPackedU64Ops() && SLT == MVT::i64)) {
876 NElts = (NElts + 1) / 2;
877 } else if (SLT == MVT::f32) {
878 bool HasPk2FP32Op = ST->hasAnyPackedFP32Ops() &&
879 IID != Intrinsic::minimumnum &&
880 IID != Intrinsic::maximumnum;
881 NElts = HasPk2FP32Op ? (NElts + 1) / 2 : NElts;
882 }
883
884 // TODO: Get more refined intrinsic costs?
885 unsigned InstRate = getQuarterRateInstrCost(CostKind);
886
887 switch (ICA.getID()) {
888 case Intrinsic::fma:
889 case Intrinsic::fmuladd:
890 if (SLT == MVT::f64) {
891 InstRate = get64BitInstrCost(CostKind);
892 break;
893 }
894
895 if ((SLT == MVT::f32 && ST->hasFastFMAF32()) || SLT == MVT::f16)
896 InstRate = getFullRateInstrCost();
897 else {
898 InstRate = ST->hasFastFMAF32() ? getHalfRateInstrCost(CostKind)
899 : getQuarterRateInstrCost(CostKind);
900 }
901 break;
902 case Intrinsic::copysign:
903 return NElts * getFullRateInstrCost();
904 case Intrinsic::minimumnum:
905 case Intrinsic::maximumnum: {
906 // Instruction + 2 canonicalizes. For cases that need type promotion, we the
907 // promotion takes the place of the canonicalize.
908 unsigned NumOps = 3;
909 if (const IntrinsicInst *II = ICA.getInst()) {
910 // Directly legal with ieee=0
911 // TODO: Not directly legal with strictfp
912 if (fpenvIEEEMode(I: *II) == KnownIEEEMode::Off)
913 NumOps = 1;
914 }
915
916 unsigned BaseRate =
917 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
918 InstRate = BaseRate * NumOps;
919 break;
920 }
921 case Intrinsic::canonicalize: {
922 InstRate =
923 SLT == MVT::f64 ? get64BitInstrCost(CostKind) : getFullRateInstrCost();
924 break;
925 }
926 case Intrinsic::uadd_sat:
927 case Intrinsic::usub_sat:
928 case Intrinsic::sadd_sat:
929 case Intrinsic::ssub_sat: {
930 if (SLT == MVT::i16 || SLT == MVT::i32)
931 InstRate = getFullRateInstrCost();
932
933 static const auto ValidSatTys = {MVT::v2i16, MVT::v4i16};
934 if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second)))
935 NElts = 1;
936 break;
937 }
938 case Intrinsic::abs:
939 // Expansion takes 2 instructions for VALU
940 if (SLT == MVT::i16 || SLT == MVT::i32)
941 InstRate = 2 * getFullRateInstrCost();
942 break;
943 default:
944 break;
945 }
946
947 return LT.first * NElts * InstRate;
948}
949
950InstructionCost GCNTTIImpl::getCFInstrCost(unsigned Opcode,
951 TTI::TargetCostKind CostKind,
952 const Instruction *I) const {
953 assert((I == nullptr || I->getOpcode() == Opcode) &&
954 "Opcode should reflect passed instruction.");
955 const bool SCost =
956 (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency);
957 const int CBrCost = SCost ? 5 : 7;
958 switch (Opcode) {
959 case Instruction::UncondBr:
960 // Branch instruction takes about 4 slots on gfx900.
961 return SCost ? 1 : 4;
962 case Instruction::CondBr:
963 // Suppose conditional branch takes additional 3 exec manipulations
964 // instructions in average.
965 return CBrCost;
966 case Instruction::Switch: {
967 const auto *SI = dyn_cast_or_null<SwitchInst>(Val: I);
968 // Each case (including default) takes 1 cmp + 1 cbr instructions in
969 // average.
970 return (SI ? (SI->getNumCases() + 1) : 4) * (CBrCost + 1);
971 }
972 case Instruction::Ret:
973 return SCost ? 1 : 10;
974 }
975 return BaseT::getCFInstrCost(Opcode, CostKind, I);
976}
977
978InstructionCost GCNTTIImpl::getCmpSelInstrCost(
979 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
980 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
981 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
982 // For size and latency cost kinds, return a low cost independent of vector
983 // width to enable SimplifyCFG's speculativelyExecuteBB optimization.
984 if (CostKind == TTI::TCK_SizeAndLatency)
985 return 1;
986
987 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
988 Op1Info, Op2Info, I);
989}
990
991// Measured packing cost of i1 for gfx9-12 is 4.0 to 4.8, up to 5.4 with
992// true16; unpacking is 2.6 to 2.9.
993static constexpr unsigned MaskPackCostPerElt = 4;
994static constexpr unsigned MaskUnpackCostPerElt = 3;
995
996static std::optional<unsigned> getNumberOfPackedMaskElts(Type *Ty) {
997 auto *FVT = dyn_cast<FixedVectorType>(Val: Ty);
998 if (FVT && FVT->getElementType()->isIntegerTy(BitWidth: 1) && FVT->getNumElements() > 1)
999 return FVT->getNumElements();
1000 return std::nullopt;
1001}
1002
1003InstructionCost GCNTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1004 Type *Src,
1005 TTI::CastContextHint CCH,
1006 TTI::TargetCostKind CostKind,
1007 const Instruction *I) const {
1008 // A bitcast between a vector of i1 and an integer packs or unpacks a mask.
1009 if (Opcode == Instruction::BitCast) {
1010 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Src);
1011 Elts && Dst->isIntegerTy(BitWidth: *Elts))
1012 return InstructionCost(MaskPackCostPerElt) * *Elts *
1013 getFullRateInstrCost();
1014 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty: Dst);
1015 Elts && Src->isIntegerTy(BitWidth: *Elts))
1016 return InstructionCost(MaskUnpackCostPerElt) * *Elts *
1017 getFullRateInstrCost();
1018 // The generic code scalarizes a vector cast into element casts, which for
1019 // a bitcast may pair elements of different sizes.
1020 if (DL.getTypeSizeInBits(Ty: Src) == DL.getTypeSizeInBits(Ty: Dst)) {
1021 // Integer and FP values share registers. A legalized vector may need its
1022 // lanes repacked.
1023 EVT SrcVT = TLI->getValueType(DL, Ty: Src);
1024 EVT DstVT = TLI->getValueType(DL, Ty: Dst);
1025 if (TLI->isTypeLegal(VT: SrcVT) && TLI->isTypeLegal(VT: DstVT))
1026 return 0;
1027 LLVMContext &Ctx = Src->getContext();
1028 if (!SrcVT.isVector() && !DstVT.isVector() &&
1029 TLI->getRegisterType(Context&: Ctx, VT: SrcVT).getSizeInBits() ==
1030 TLI->getRegisterType(Context&: Ctx, VT: DstVT).getSizeInBits())
1031 return 0;
1032 }
1033 }
1034
1035 // Each f32 lane is rounded and the halves are packed in pairs. A packed
1036 // conversion rounds a pair at once and true16 writes either half.
1037 if (auto *SrcVTy = dyn_cast<FixedVectorType>(Val: Src);
1038 SrcVTy && Opcode == Instruction::FPTrunc && ST->has16BitInsts() &&
1039 SrcVTy->getElementType()->isFloatTy() &&
1040 Dst->getScalarType()->isHalfTy()) {
1041 const unsigned NElts = SrcVTy->getNumElements();
1042 const unsigned Ops = ST->hasCvtPkF16F32Inst() ? divideCeil(Numerator: NElts, Denominator: 2)
1043 : ST->useRealTrue16Insts() ? NElts
1044 : NElts + NElts / 2;
1045 return InstructionCost(Ops) * getFullRateInstrCost();
1046 }
1047
1048 const int ISD = TLI->InstructionOpcodeToISD(Opcode);
1049 switch (ISD) {
1050 case ISD::SINT_TO_FP:
1051 case ISD::UINT_TO_FP:
1052 case ISD::FP_TO_SINT:
1053 case ISD::FP_TO_UINT:
1054 break;
1055 default:
1056 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1057 }
1058
1059 const bool IsIntToFP = ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP;
1060 Type *FPTy = (IsIntToFP ? Dst : Src)->getScalarType();
1061 if (!FPTy->isHalfTy() && !FPTy->isBFloatTy() && !FPTy->isFloatTy() &&
1062 !FPTy->isDoubleTy())
1063 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1064
1065 unsigned NElts = 1;
1066 if (auto *VT = dyn_cast<FixedVectorType>(Val: Src))
1067 NElts = VT->getNumElements();
1068
1069 const unsigned SrcBits = Src->getScalarSizeInBits();
1070 const unsigned DstBits = Dst->getScalarSizeInBits();
1071 const bool IsSigned = ISD == ISD::SINT_TO_FP || ISD == ISD::FP_TO_SINT;
1072 const unsigned IntBits = IsIntToFP ? SrcBits : DstBits;
1073 const bool UsesInt64 = IntBits > 32 && IntBits <= 64;
1074
1075 auto Scale = [&](unsigned FullRateOps,
1076 unsigned FP64Ops = 0) -> InstructionCost {
1077 return NElts * (InstructionCost(FullRateOps) * getFullRateInstrCost() +
1078 InstructionCost(FP64Ops) * get64BitInstrCost(CostKind));
1079 };
1080
1081 if (IsIntToFP) {
1082 // A scalar load of a whole number of bytes that is not a power of two is
1083 // split and its high part load extends the source. A constant address
1084 // space or invariant global load aligned to 4 bytes may be widened instead
1085 // and then still needs the extension. So does a buffer fat pointer load.
1086 const auto *Load =
1087 I && Src->isIntegerTy() && I->getOperand(i: 0)->getType() == Src
1088 ? dyn_cast<LoadInst>(Val: I->getOperand(i: 0))
1089 : nullptr;
1090 const unsigned AS = Load ? Load->getPointerAddressSpace() : 0;
1091 const bool LoadMayWiden =
1092 Load && Load->getAlign() >= Align(4) &&
1093 (AS == AMDGPUAS::CONSTANT_ADDRESS ||
1094 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
1095 (AS == AMDGPUAS::GLOBAL_ADDRESS &&
1096 Load->hasMetadata(KindID: LLVMContext::MD_invariant_load)));
1097 const bool LoadExtends = Load && Load->isSimple() && Load->hasOneUse() &&
1098 !LoadMayWiden &&
1099 AS != AMDGPUAS::BUFFER_FAT_POINTER &&
1100 AS != AMDGPUAS::BUFFER_STRIDED_POINTER &&
1101 SrcBits % 8 == 0 && !isPowerOf2_32(Value: SrcBits);
1102 const unsigned ExtOps =
1103 UsesInt64 && SrcBits < 64 && !LoadExtends ? (IsSigned ? 2 : 1) : 0;
1104 if (FPTy->isBFloatTy()) {
1105 if (SrcBits < 8 || (SrcBits > 32 && !UsesInt64))
1106 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1107
1108 // Each integer is converted to f32 first.
1109 InstructionCost FloatCost =
1110 Scale(UsesInt64 ? ExtOps + (IsSigned ? 12 : 8) : 1);
1111 if (SrcBits < 32)
1112 FloatCost = getCastInstrCost(
1113 Opcode, Dst: Dst->getWithNewType(EltTy: Type::getFloatTy(C&: Dst->getContext())),
1114 Src, CCH, CostKind, I);
1115
1116 // Native rounding can convert a pair. With 16 bit instructions the
1117 // expansion extracts the low significand bit, adds the rounding bias,
1118 // preserves NaNs and shifts the result. Without gfx9 instructions the two
1119 // additions cannot use v_add3_u32.
1120 InstructionCost RoundCost =
1121 ST->hasBF16ConversionInsts()
1122 ? InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) * getFullRateInstrCost()
1123 : Scale(!ST->has16BitInsts() ? 1
1124 : ST->hasGFX9Insts() ? 6
1125 : 7);
1126 return FloatCost + RoundCost;
1127 }
1128
1129 // No instruction converts from a 64 bit integer.
1130 if (UsesInt64) {
1131 if (FPTy->isDoubleTy()) {
1132 // Two conversions, ldexp and add, all using the FP64 rate.
1133 return Scale(ExtOps, 4);
1134 }
1135 if (FPTy->isFloatTy())
1136 return Scale(ExtOps + (IsSigned ? 12 : 8));
1137 return Scale(ExtOps + (IsSigned ? 13 : 9));
1138 }
1139
1140 // A narrow vector source is converted lane by lane.
1141 if (SrcBits >= 8 && SrcBits < 32 && isa<FixedVectorType>(Val: Src)) {
1142 if (FPTy->isDoubleTy())
1143 return Scale(1 + (SrcBits < 16 && IsSigned && ST->has16BitInsts()), 1);
1144 if (FPTy->isHalfTy()) {
1145 const InstructionCost PairCost =
1146 InstructionCost(NElts / 2) * getFullRateInstrCost();
1147 // Lanes wider than 16 bits, and every lane without 16 bit instructions,
1148 // are converted to f32 first. With the packed conversion each pair is
1149 // then converted at once. Otherwise each lane is converted, and without
1150 // real true16 each pair of halves is packed.
1151 if (!ST->has16BitInsts() || SrcBits > 16) {
1152 auto *FloatTy =
1153 FixedVectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()), NumElts: NElts);
1154 const InstructionCost FloatCost =
1155 getCastInstrCost(Opcode, Dst: FloatTy, Src, CCH, CostKind);
1156 if (ST->has16BitInsts() && ST->hasCvtPkF16F32Inst())
1157 return FloatCost + InstructionCost(divideCeil(Numerator: NElts, Denominator: 2)) *
1158 getFullRateInstrCost();
1159 const unsigned PackOps =
1160 !ST->has16BitInsts() ? 2 : !ST->useRealTrue16Insts();
1161 return FloatCost + Scale(1) + PackOps * PairCost;
1162 }
1163 // Each lane is converted from a 16 bit subword. Lanes narrower than 16
1164 // bits are extended lane by lane, except bytes with SDWA. Signed bytes
1165 // are sign extended a pair at a time. Real true16 reads the high
1166 // subword in place, so only signed lanes narrower than 16 bits pay per
1167 // pair. Otherwise wider lanes shift the high subword of each pair down
1168 // without SDWA, and each pair of halves is packed.
1169 const unsigned PerElt =
1170 1 + (SrcBits == 8 ? !ST->hasSDWA() : SrcBits < 16);
1171 const bool RealTrue16 = ST->useRealTrue16Insts();
1172 const unsigned PerPair =
1173 !RealTrue16 + (SrcBits == 8 || RealTrue16 ? SrcBits < 16 && IsSigned
1174 : !ST->hasSDWA());
1175 return Scale(PerElt) + PerPair * PairCost;
1176 }
1177 // An unsigned byte is converted straight out of its register.
1178 if (SrcBits == 8 && !IsSigned)
1179 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1180 if (SrcBits < 16 && !IsSigned)
1181 return Scale(2);
1182 unsigned PerElt = ST->hasSDWA() && SrcBits <= 16 ? 1 : 2;
1183 if (SrcBits < 16 && ST->has16BitInsts())
1184 ++PerElt;
1185 return Scale(PerElt);
1186 }
1187
1188 // A narrow scalar source is extended before the conversion. An unsigned
1189 // byte is converted straight out of its register, and a signed one with
1190 // SDWA. A source wider than a byte but narrower than 16 bits is masked
1191 // or sign extended first. Without 16 bit instructions a half result is
1192 // rounded from f32.
1193 if (SrcBits >= 8 && SrcBits < 32) {
1194 if (LoadExtends) {
1195 if (FPTy->isDoubleTy())
1196 return Scale(0, 1);
1197 return Scale(FPTy->isHalfTy() ? 2 : 1);
1198 }
1199 const bool Narrow = SrcBits > 8 && SrcBits < 16;
1200 const bool SignExtend16 = IsSigned && Narrow && ST->has16BitInsts();
1201 if (FPTy->isDoubleTy())
1202 return Scale(1 + 2 * SignExtend16, 1);
1203 if (SrcBits > 16)
1204 return Scale(FPTy->isHalfTy() ? 3 : 2);
1205 if (FPTy->isHalfTy() && ST->has16BitInsts())
1206 return Scale(Narrow ? (IsSigned ? 3 : 2)
1207 : SrcBits == 8 && !ST->hasSDWA() ? 2
1208 : 1);
1209 const InstructionCost FloatCost =
1210 SignExtend16 ? Scale(ST->hasSDWA() ? 3 : 4)
1211 : Narrow ? Scale(2)
1212 : SrcBits == 8 && !IsSigned ? Scale(1)
1213 : Scale(ST->hasSDWA() ? 1 : 2);
1214 return FPTy->isHalfTy() ? FloatCost + Scale(1) : FloatCost;
1215 }
1216
1217 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1218 }
1219
1220 // No instruction converts to a 64 bit integer.
1221 if (UsesInt64) {
1222 const bool IsSigned64 = IsSigned || (DstBits < 64 && NElts == 1);
1223 if (FPTy->isDoubleTy()) {
1224 // With native trunc/floor there are six FP64 operations. The expanded
1225 // rounding sequence has seven, plus integer operations and constants.
1226 return ST->haveRoundOpsF64() ? Scale(1, 6) : Scale(22, 7);
1227 }
1228 // The f32 expansion has one fma, which is quarter rate without fast FMA.
1229 const InstructionCost SlowFMACost =
1230 ST->hasFastFMAF32() ? 0
1231 : NElts * (getQuarterRateInstrCost(CostKind) -
1232 getFullRateInstrCost());
1233 if (FPTy->isFloatTy())
1234 return Scale(IsSigned64 ? 13 : 6) + SlowFMACost;
1235 if (FPTy->isBFloatTy()) {
1236 // Unlike half, bf16 does not fit in i32. Extend to f32 and use the full
1237 // i64 expansion.
1238 return Scale(1 + (IsSigned64 ? 13 : 6)) + SlowFMACost;
1239 }
1240 return Scale(3);
1241 }
1242
1243 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1244}
1245
1246InstructionCost
1247GCNTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
1248 std::optional<FastMathFlags> FMF,
1249 TTI::TargetCostKind CostKind) const {
1250 if (TTI::requiresOrderedReduction(FMF))
1251 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1252
1253 // An add or xor reduction over a vector of i1 becomes a bit count over the
1254 // packed mask; the generic model prices a shuffle tree and misses that.
1255 if (Opcode == Instruction::Add || Opcode == Instruction::Xor) {
1256 if (std::optional<unsigned> Elts = getNumberOfPackedMaskElts(Ty))
1257 return InstructionCost(MaskPackCostPerElt) * *Elts *
1258 getFullRateInstrCost();
1259 }
1260
1261 EVT OrigTy = TLI->getValueType(DL, Ty);
1262
1263 // Computes cost on targets that have packed math instructions(which support
1264 // 16-bit types only).
1265 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1266 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1267
1268 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1269 return LT.first * getFullRateInstrCost();
1270}
1271
1272InstructionCost
1273GCNTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1274 FastMathFlags FMF,
1275 TTI::TargetCostKind CostKind) const {
1276 EVT OrigTy = TLI->getValueType(DL, Ty);
1277
1278 // Computes cost on targets that have packed math instructions(which support
1279 // 16-bit types only).
1280 if (!ST->hasVOP3PInsts() || OrigTy.getScalarSizeInBits() != 16)
1281 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1282
1283 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1284 return LT.first * getHalfRateInstrCost(CostKind);
1285}
1286
1287InstructionCost GCNTTIImpl::getVectorInstrCost(
1288 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
1289 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
1290 switch (Opcode) {
1291 case Instruction::ExtractElement:
1292 case Instruction::InsertElement: {
1293 TypeSize EltSize =
1294 DL.getTypeSizeInBits(Ty: cast<VectorType>(Val: ValTy)->getElementType());
1295 // Dynamic indexing isn't free and is best avoided.
1296 if (Index == ~0u)
1297 return 2;
1298 if (EltSize < 32) {
1299 if (EltSize == 16 && Index == 0 && ST->has16BitInsts())
1300 return 0;
1301 // Inserts of booleans are free.
1302 // TODO: Extracts are free too.
1303 if (EltSize == 1 && Opcode == Instruction::InsertElement)
1304 return TargetTransformInfo::TCC_Free;
1305 // Extract element sequences of consecutive i8 values that match a
1306 // register size are free most likely. It is not possible to know
1307 // if this extract is part of a consecutive sequence so this may
1308 // apply more generally.
1309 if (Opcode == Instruction::ExtractElement && EltSize == 8) {
1310 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: ValTy)) {
1311 unsigned NumElts = FVTy->getNumElements();
1312 if (NumElts >= 4 && isPowerOf2_32(Value: NumElts))
1313 return 0;
1314 }
1315 }
1316 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1317 VIC);
1318 }
1319
1320 // Packed f32 pair formation for v_pk_*_f32. A constant-index insert writes
1321 // one subregister, so a lane holding a computed value costs at most one
1322 // v_mov_b32 at any width. A loaded lane lands in place, and arguments and
1323 // constants are already in a register, so both are free. SLP prices a
1324 // buildvector root with the insertelements themselves as Op1, and those
1325 // lanes need no packing either. Only throughput sees the move; size-based
1326 // users such as the inliner and the unroller keep inserts free.
1327 auto *VecTy = dyn_cast<FixedVectorType>(Val: ValTy);
1328 const auto *InsertedInst = dyn_cast_or_null<Instruction>(Val: Op1);
1329 if (Opcode == Instruction::InsertElement && EltSize == 32 &&
1330 CostKind == TTI::TCK_RecipThroughput && ST->hasAnyPackedFP32Ops() &&
1331 InsertedInst && VecTy && VecTy->getElementType()->isFloatTy() &&
1332 !isa<LoadInst, InsertElementInst>(Val: InsertedInst))
1333 return getFullRateInstrCost();
1334
1335 // Extracts are just reads of a subregister, so are free. Inserts are
1336 // considered free because we don't want to have any cost for scalarizing
1337 // operations, and we don't have to copy into a different register class.
1338 return 0;
1339 }
1340 default:
1341 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
1342 VIC);
1343 }
1344}
1345
1346/// Analyze if the results of inline asm are divergent. If \p Indices is empty,
1347/// this is analyzing the collective result of all output registers. Otherwise,
1348/// this is only querying a specific result index if this returns multiple
1349/// registers in a struct.
1350bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
1351 const CallBase *CB, ArrayRef<unsigned> Indices) const {
1352 // TODO: Handle complex extract indices
1353 if (Indices.size() > 1)
1354 return true;
1355
1356 const DataLayout &DL = CB->getDataLayout();
1357 const SIRegisterInfo *TRI = ST->getRegisterInfo();
1358 TargetLowering::AsmOperandInfoVector TargetConstraints =
1359 TLI->ParseConstraints(DL, TRI: ST->getRegisterInfo(), Call: *CB);
1360
1361 const int TargetOutputIdx = Indices.empty() ? -1 : Indices[0];
1362
1363 int OutputIdx = 0;
1364 for (auto &TC : TargetConstraints) {
1365 if (TC.Type != InlineAsm::isOutput)
1366 continue;
1367
1368 // Skip outputs we don't care about.
1369 if (TargetOutputIdx != -1 && TargetOutputIdx != OutputIdx++)
1370 continue;
1371
1372 TLI->ComputeConstraintToUse(OpInfo&: TC, Op: SDValue());
1373
1374 const TargetRegisterClass *RC = TLI->getRegForInlineAsmConstraint(
1375 TRI, Constraint: TC.ConstraintCode, VT: TC.ConstraintVT).second;
1376
1377 // For AGPR constraints null is returned on subtargets without AGPRs, so
1378 // assume divergent for null.
1379 if (!RC || !TRI->isSGPRClass(RC))
1380 return true;
1381 }
1382
1383 return false;
1384}
1385
1386bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
1387 const IntrinsicInst *ReadReg) const {
1388 Metadata *MD =
1389 cast<MetadataAsValue>(Val: ReadReg->getArgOperand(i: 0))->getMetadata();
1390 StringRef RegName =
1391 cast<MDString>(Val: cast<MDNode>(Val: MD)->getOperand(I: 0))->getString();
1392
1393 // Special case registers that look like VCC.
1394 MVT VT = MVT::getVT(Ty: ReadReg->getType());
1395 if (VT == MVT::i1)
1396 return true;
1397
1398 // Special case scalar registers that start with 'v'.
1399 if (RegName.starts_with(Prefix: "vcc") || RegName.empty())
1400 return false;
1401
1402 // VGPR or AGPR is divergent. There aren't any specially named vector
1403 // registers.
1404 return RegName[0] == 'v' || RegName[0] == 'a';
1405}
1406
1407/// \returns true if the result of the value could potentially be
1408/// different across workitems in a wavefront.
1409bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
1410 if (const Argument *A = dyn_cast<Argument>(Val: V))
1411 return !AMDGPU::isArgPassedInSGPR(Arg: A);
1412
1413 // Loads from the private and flat address spaces are divergent, because
1414 // threads can execute the load instruction with the same inputs and get
1415 // different results. So are loads from the VGPR address space, since each
1416 // lane reads its own registers.
1417 //
1418 // All other loads are not divergent, because if threads issue loads with the
1419 // same arguments, they will always get the same result.
1420 if (const LoadInst *Load = dyn_cast<LoadInst>(Val: V))
1421 return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
1422 Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS ||
1423 Load->getPointerAddressSpace() == AMDGPUAS::VGPR;
1424
1425 // Atomics are divergent because they are executed sequentially: when an
1426 // atomic operation refers to the same address in each thread, then each
1427 // thread after the first sees the value written by the previous thread as
1428 // original value.
1429 if (isa<AtomicRMWInst, AtomicCmpXchgInst>(Val: V))
1430 return true;
1431
1432 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
1433 Intrinsic::ID IID = Intrinsic->getIntrinsicID();
1434 switch (IID) {
1435 case Intrinsic::read_register:
1436 return isReadRegisterSourceOfDivergence(ReadReg: Intrinsic);
1437 case Intrinsic::amdgcn_workitem_id_y:
1438 case Intrinsic::amdgcn_workitem_id_z: {
1439 const Function *F = Intrinsic->getFunction();
1440 bool HasUniformYZ =
1441 ST->hasWavefrontsEvenlySplittingXDim(F: *F, /*RequiresUniformYZ=*/true);
1442 std::optional<unsigned> ThisDimSize = ST->getReqdWorkGroupSize(
1443 F: *F, Dim: IID == Intrinsic::amdgcn_workitem_id_y ? 1 : 2);
1444 return !HasUniformYZ && (!ThisDimSize || *ThisDimSize != 1);
1445 }
1446 default:
1447 return AMDGPU::isIntrinsicSourceOfDivergence(IntrID: IID);
1448 }
1449 }
1450
1451 // Assume all function calls are a source of divergence.
1452 if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) {
1453 if (CB->isInlineAsm())
1454 return isInlineAsmSourceOfDivergence(CB);
1455 return true;
1456 }
1457
1458 // Assume all function calls are a source of divergence.
1459 if (isa<InvokeInst>(Val: V))
1460 return true;
1461
1462 // If the target supports globally addressable scratch, the mapping from
1463 // scratch memory to the flat aperture changes therefore an address space cast
1464 // is no longer uniform.
1465 if (auto *CastI = dyn_cast<AddrSpaceCastInst>(Val: V)) {
1466 return CastI->getSrcAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS &&
1467 CastI->getDestAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
1468 ST->hasGloballyAddressableScratch();
1469 }
1470
1471 return false;
1472}
1473
1474bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
1475 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V))
1476 return AMDGPU::isIntrinsicAlwaysUniform(IntrID: Intrinsic->getIntrinsicID());
1477
1478 if (const CallBase *CB = dyn_cast<CallBase>(Val: V)) {
1479 if (CB->isInlineAsm())
1480 return !isInlineAsmSourceOfDivergence(CB);
1481 return false;
1482 }
1483
1484 // In most cases TID / wavefrontsize is uniform.
1485 //
1486 // However, if a kernel has uneven dimesions we can have a value of
1487 // workitem-id-x divided by the wavefrontsize non-uniform. For example
1488 // dimensions (65, 2) will have workitems with address (64, 0) and (0, 1)
1489 // packed into a same wave which gives 1 and 0 after the division by 64
1490 // respectively.
1491 //
1492 // The X dimension doesn't reset within a wave if either both the Y
1493 // and Z dimensions are of length 1, or if the X dimension's required
1494 // size is a power of 2. Note, however, if the X dimension's maximum
1495 // size is a power of 2 < the wavefront size, division by the wavefront
1496 // size is guaranteed to yield 0, so this is also a no-reset case.
1497 bool XDimDoesntResetWithinWaves = false;
1498 if (auto *I = dyn_cast<Instruction>(Val: V)) {
1499 const Function *F = I->getFunction();
1500 XDimDoesntResetWithinWaves = ST->hasWavefrontsEvenlySplittingXDim(F: *F);
1501 }
1502 using namespace llvm::PatternMatch;
1503 uint64_t C;
1504 auto MatchTidXCall = m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>();
1505 auto MaybeMaskedTidX =
1506 m_CombineOr(Ps: m_c_And(L: MatchTidXCall, R: m_Constant()), Ps: MatchTidXCall);
1507 auto MaybeCastTidX = m_CastOrSelf(Op: MaybeMaskedTidX);
1508 auto MaybeMaskedCastTidX =
1509 m_CombineOr(Ps: m_c_And(L: MaybeCastTidX, R: m_Constant()), Ps: MaybeCastTidX);
1510 if (match(V, P: m_LShr(L: MaybeMaskedCastTidX, R: m_ConstantInt(V&: C))))
1511 return C >= ST->getWavefrontSizeLog2() && XDimDoesntResetWithinWaves;
1512
1513 Constant *Mask;
1514 if (match(V, P: m_c_And(
1515 L: m_CastOrSelf(Op: m_Intrinsic<Intrinsic::amdgcn_workitem_id_x>()),
1516 R: m_Constant(C&: Mask)))) {
1517 return computeKnownBits(V: Mask, DL).countMinTrailingZeros() >=
1518 ST->getWavefrontSizeLog2() &&
1519 XDimDoesntResetWithinWaves;
1520 }
1521
1522 const ExtractValueInst *ExtValue = dyn_cast<ExtractValueInst>(Val: V);
1523 if (!ExtValue)
1524 return false;
1525
1526 const CallBase *CB = dyn_cast<CallBase>(Val: ExtValue->getOperand(i_nocapture: 0));
1527 if (!CB)
1528 return false;
1529
1530 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: CB)) {
1531 switch (Intrinsic->getIntrinsicID()) {
1532 default:
1533 return false;
1534 case Intrinsic::amdgcn_if:
1535 case Intrinsic::amdgcn_else: {
1536 ArrayRef<unsigned> Indices = ExtValue->getIndices();
1537 return Indices.size() == 1 && Indices[0] == 1;
1538 }
1539 }
1540 }
1541
1542 // If we have inline asm returning mixed SGPR and VGPR results, we inferred
1543 // divergent for the overall struct return. We need to override it in the
1544 // case we're extracting an SGPR component here.
1545 if (CB->isInlineAsm())
1546 return !isInlineAsmSourceOfDivergence(CB, Indices: ExtValue->getIndices());
1547
1548 return false;
1549}
1550
1551bool GCNTTIImpl::collectFlatAddressOperands(SmallVectorImpl<int> &OpIndexes,
1552 Intrinsic::ID IID) const {
1553 switch (IID) {
1554 case Intrinsic::amdgcn_is_shared:
1555 case Intrinsic::amdgcn_is_private:
1556 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1557 case Intrinsic::amdgcn_flat_atomic_fmin_num:
1558 case Intrinsic::amdgcn_load_to_lds:
1559 case Intrinsic::amdgcn_make_buffer_rsrc:
1560 case Intrinsic::amdgcn_flat_prefetch:
1561 OpIndexes.push_back(Elt: 0);
1562 return true;
1563 default:
1564 return false;
1565 }
1566}
1567
1568Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
1569 Value *OldV,
1570 Value *NewV) const {
1571 auto IntrID = II->getIntrinsicID();
1572 switch (IntrID) {
1573 case Intrinsic::amdgcn_is_shared:
1574 case Intrinsic::amdgcn_is_private: {
1575 unsigned TrueAS = IntrID == Intrinsic::amdgcn_is_shared ?
1576 AMDGPUAS::LOCAL_ADDRESS : AMDGPUAS::PRIVATE_ADDRESS;
1577 unsigned NewAS = NewV->getType()->getPointerAddressSpace();
1578 LLVMContext &Ctx = NewV->getType()->getContext();
1579 ConstantInt *NewVal = (TrueAS == NewAS) ?
1580 ConstantInt::getTrue(Context&: Ctx) : ConstantInt::getFalse(Context&: Ctx);
1581 return NewVal;
1582 }
1583 case Intrinsic::amdgcn_flat_atomic_fmax_num:
1584 case Intrinsic::amdgcn_flat_atomic_fmin_num: {
1585 Type *DestTy = II->getType();
1586 Type *SrcTy = NewV->getType();
1587 unsigned NewAS = SrcTy->getPointerAddressSpace();
1588 if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS))
1589 return nullptr;
1590 Module *M = II->getModule();
1591 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1592 M, id: II->getIntrinsicID(), OverloadTys: {DestTy, SrcTy, DestTy});
1593 II->setArgOperand(i: 0, v: NewV);
1594 II->setCalledFunction(NewDecl);
1595 return II;
1596 }
1597 case Intrinsic::amdgcn_load_to_lds: {
1598 Type *SrcTy = NewV->getType();
1599 Module *M = II->getModule();
1600 Function *NewDecl =
1601 Intrinsic::getOrInsertDeclaration(M, id: II->getIntrinsicID(), OverloadTys: {SrcTy});
1602 II->setArgOperand(i: 0, v: NewV);
1603 II->setCalledFunction(NewDecl);
1604 return II;
1605 }
1606 case Intrinsic::amdgcn_make_buffer_rsrc: {
1607 Type *SrcTy = NewV->getType();
1608 Type *DstTy = II->getType();
1609 Type *NumRecordsTy = II->getArgOperand(i: 2)->getType();
1610 Module *M = II->getModule();
1611 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1612 M, id: II->getIntrinsicID(), OverloadTys: {DstTy, SrcTy, NumRecordsTy});
1613 II->setArgOperand(i: 0, v: NewV);
1614 II->setCalledFunction(NewDecl);
1615 return II;
1616 }
1617 case Intrinsic::amdgcn_flat_prefetch: {
1618 unsigned NewAS = NewV->getType()->getPointerAddressSpace();
1619 if (!AMDGPU::isExtendedGlobalAddrSpace(AS: NewAS))
1620 return nullptr;
1621 // llvm.amdgcn.global.prefetch only takes a global pointer.
1622 if (NewAS != AMDGPUAS::GLOBAL_ADDRESS)
1623 NewV = IRBuilder<>(II).CreateAddrSpaceCast(
1624 V: NewV, DestTy: PointerType::get(C&: II->getContext(), AddressSpace: AMDGPUAS::GLOBAL_ADDRESS));
1625 Function *NewDecl = Intrinsic::getOrInsertDeclaration(
1626 M: II->getModule(), id: Intrinsic::amdgcn_global_prefetch);
1627 II->setArgOperand(i: 0, v: NewV);
1628 II->setCalledFunction(NewDecl);
1629 return II;
1630 }
1631 default:
1632 return nullptr;
1633 }
1634}
1635
1636InstructionCost GCNTTIImpl::getShuffleCost(
1637 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1638 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1639 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1640 TTI::VectorInstrContext VIC) const {
1641 if (!isa<FixedVectorType>(Val: SrcTy))
1642 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1643 SubTp);
1644
1645 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1646
1647 TypeSize ScalarSize = DL.getTypeSizeInBits(Ty: SrcTy->getElementType());
1648
1649 // A v_pk_*_f32 operand is an aligned register pair. A lane already in its
1650 // half of a source pair is read in place, and a splat of the low half is an
1651 // op_sel_hi. Every other lane costs a v_mov_b32 to move it into place.
1652 if (ScalarSize == 32 && CostKind == TTI::TCK_RecipThroughput &&
1653 ST->hasAnyPackedFP32Ops() && SrcTy->getElementType()->isFloatTy() &&
1654 !Mask.empty()) {
1655 const int NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements();
1656 // Source pairs of the second operand are numbered after the first.
1657 auto GetPair = [&](int Idx) {
1658 return Idx < NumSrcElts ? Idx / 2 : NumSrcElts + (Idx - NumSrcElts) / 2;
1659 };
1660 auto GetHalf = [&](int Idx) {
1661 return Idx < NumSrcElts ? Idx % 2 : (Idx - NumSrcElts) % 2;
1662 };
1663 unsigned Cost = 0;
1664 for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += 2) {
1665 const int Lo = Mask[DstIdx];
1666 const int Hi =
1667 DstIdx + 1 < Mask.size() ? Mask[DstIdx + 1] : PoisonMaskElem;
1668 const bool LoInPlace = Lo != PoisonMaskElem && GetHalf(Lo) == 0;
1669 const bool HiInPlace = Hi != PoisonMaskElem && GetHalf(Hi) == 1;
1670 if (Lo != PoisonMaskElem && Lo == Hi && LoInPlace)
1671 continue;
1672 if (LoInPlace && HiInPlace)
1673 Cost += GetPair(Lo) != GetPair(Hi);
1674 else
1675 Cost += (Lo != PoisonMaskElem && !LoInPlace) +
1676 (Hi != PoisonMaskElem && !HiInPlace);
1677 }
1678 return Cost;
1679 }
1680
1681 if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
1682 (ScalarSize == 16 || ScalarSize == 8)) {
1683 // Larger vector widths may require additional instructions, but are
1684 // typically cheaper than scalarized versions.
1685 //
1686 // We assume that shuffling at a register granularity can be done for free.
1687 // This is not true for vectors fed into memory instructions, but it is
1688 // effectively true for all other shuffling. The emphasis of the logic here
1689 // is to assist generic transform in cleaning up / canonicalizing those
1690 // shuffles.
1691
1692 // With op_sel VOP3P instructions freely can access the low half or high
1693 // half of a register, so any swizzle of two elements is free.
1694 if (auto *SrcVecTy = dyn_cast<FixedVectorType>(Val: SrcTy)) {
1695 unsigned NumSrcElts = SrcVecTy->getNumElements();
1696 if (ST->hasVOP3PInsts() && ScalarSize == 16 && NumSrcElts == 2 &&
1697 (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Reverse ||
1698 Kind == TTI::SK_PermuteSingleSrc))
1699 return 0;
1700 }
1701
1702 unsigned EltsPerReg = 32 / static_cast<unsigned>(ScalarSize);
1703 switch (Kind) {
1704 case TTI::SK_Broadcast:
1705 // A single v_perm_b32 can be re-used for all destination registers.
1706 return 1;
1707 case TTI::SK_Reverse:
1708 // One instruction per register.
1709 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1710 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1711 return InstructionCost::getInvalid();
1712 case TTI::SK_ExtractSubvector:
1713 if (Index % EltsPerReg == 0)
1714 return 0; // Shuffling at register granularity
1715 if (auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy))
1716 return divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: EltsPerReg);
1717 return InstructionCost::getInvalid();
1718 case TTI::SK_InsertSubvector: {
1719 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1720 if (!DstVecTy)
1721 return InstructionCost::getInvalid();
1722 unsigned NumDstElts = DstVecTy->getNumElements();
1723 unsigned NumInsertElts = cast<FixedVectorType>(Val: SubTp)->getNumElements();
1724 unsigned EndIndex = Index + NumInsertElts;
1725 unsigned BeginSubIdx = Index % EltsPerReg;
1726 unsigned EndSubIdx = EndIndex % EltsPerReg;
1727 InstructionCost Cost = 0;
1728
1729 if (BeginSubIdx != 0) {
1730 // Need to shift the inserted vector into place. The cost is the number
1731 // of destination registers overlapped by the inserted vector.
1732 Cost = divideCeil(Numerator: EndIndex, Denominator: EltsPerReg) - (Index / EltsPerReg);
1733 }
1734
1735 // If the last register overlap is partial, there may be three source
1736 // registers feeding into it; that takes an extra instruction.
1737 if (EndIndex < NumDstElts && BeginSubIdx < EndSubIdx)
1738 Cost += 1;
1739
1740 return Cost;
1741 }
1742 case TTI::SK_Splice: {
1743 auto *DstVecTy = dyn_cast<FixedVectorType>(Val: DstTy);
1744 if (!DstVecTy)
1745 return InstructionCost::getInvalid();
1746 unsigned NumElts = DstVecTy->getNumElements();
1747 assert(NumElts == cast<FixedVectorType>(SrcTy)->getNumElements());
1748 // Determine the sub-region of the result vector that requires
1749 // sub-register shuffles / mixing.
1750 unsigned EltsFromLHS = NumElts - Index;
1751 bool LHSIsAligned = (Index % EltsPerReg) == 0;
1752 bool RHSIsAligned = (EltsFromLHS % EltsPerReg) == 0;
1753 if (LHSIsAligned && RHSIsAligned)
1754 return 0;
1755 if (LHSIsAligned && !RHSIsAligned)
1756 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg) - (EltsFromLHS / EltsPerReg);
1757 if (!LHSIsAligned && RHSIsAligned)
1758 return divideCeil(Numerator: EltsFromLHS, Denominator: EltsPerReg);
1759 return divideCeil(Numerator: NumElts, Denominator: EltsPerReg);
1760 }
1761 default:
1762 break;
1763 }
1764
1765 if (!Mask.empty()) {
1766 unsigned NumSrcElts = cast<FixedVectorType>(Val: SrcTy)->getNumElements();
1767
1768 // Generically estimate the cost by assuming that each destination
1769 // register is derived from sources via v_perm_b32 instructions if it
1770 // can't be copied as-is.
1771 //
1772 // For each destination register, derive the cost of obtaining it based
1773 // on the number of source registers that feed into it.
1774 InstructionCost Cost = 0;
1775 for (unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx += EltsPerReg) {
1776 SmallVector<int, 4> Regs;
1777 bool Aligned = true;
1778 for (unsigned I = 0; I < EltsPerReg && DstIdx + I < Mask.size(); ++I) {
1779 int SrcIdx = Mask[DstIdx + I];
1780 if (SrcIdx == -1)
1781 continue;
1782 int Reg;
1783 if (SrcIdx < (int)NumSrcElts) {
1784 Reg = SrcIdx / EltsPerReg;
1785 if (SrcIdx % EltsPerReg != I)
1786 Aligned = false;
1787 } else {
1788 Reg = NumSrcElts + (SrcIdx - NumSrcElts) / EltsPerReg;
1789 if ((SrcIdx - NumSrcElts) % EltsPerReg != I)
1790 Aligned = false;
1791 }
1792 if (!llvm::is_contained(Range&: Regs, Element: Reg))
1793 Regs.push_back(Elt: Reg);
1794 }
1795 if (Regs.size() >= 2)
1796 Cost += Regs.size() - 1;
1797 else if (!Aligned)
1798 Cost += 1;
1799 }
1800 return Cost;
1801 }
1802 }
1803
1804 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1805 SubTp);
1806}
1807
1808/// Whether it is profitable to sink the operands of an
1809/// Instruction I to the basic block of I.
1810/// This helps using several modifiers (like abs and neg) more often.
1811bool GCNTTIImpl::isProfitableToSinkOperands(Instruction *I,
1812 SmallVectorImpl<Use *> &Ops) const {
1813 using namespace PatternMatch;
1814
1815 // The cost model prices this fmul as free assuming it fuses with its
1816 // fadd/fsub user, which needs them in one block. Sink a stranded
1817 // loop-invariant fmul back to the user when they would fuse. Single use only,
1818 // so this stays a move.
1819 if (I->getOpcode() == Instruction::FAdd ||
1820 I->getOpcode() == Instruction::FSub) {
1821 const Instruction *FMul = getFusedFMul(TLI: *TLI, Ty: I->getType(), FAddSub: I);
1822 if (FMul && FMul->getParent() != I->getParent())
1823 Ops.push_back(Elt: &I->getOperandUse(i: I->getOperand(i: 0) == FMul ? 0 : 1));
1824 }
1825
1826 for (auto &Op : I->operands()) {
1827 // Ensure we are not already sinking this operand.
1828 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op.get(); }))
1829 continue;
1830
1831 // TODO: Confirm whether m_FNegOrFSubNegZero is correct.
1832 if (match(V: &Op, P: m_FAbs(Op0: m_Value())) ||
1833 match(V: &Op, P: m_FNegOrFSubNegZero(X: m_Value()))) {
1834 Ops.push_back(Elt: &Op);
1835 continue;
1836 }
1837
1838 // Check for zero-cost multiple use InsertElement/ExtractElement
1839 // instructions
1840 if (Instruction *OpInst = dyn_cast<Instruction>(Val: Op.get())) {
1841 if (OpInst->getType()->isVectorTy() && OpInst->getNumOperands() > 1) {
1842 Instruction *VecOpInst = dyn_cast<Instruction>(Val: OpInst->getOperand(i: 0));
1843 if (VecOpInst && VecOpInst->hasOneUse())
1844 continue;
1845
1846 if (getVectorInstrCost(Opcode: OpInst->getOpcode(), ValTy: OpInst->getType(),
1847 CostKind: TTI::TCK_RecipThroughput, Index: 0,
1848 Op0: OpInst->getOperand(i: 0),
1849 Op1: OpInst->getOperand(i: 1)) == 0) {
1850 Ops.push_back(Elt: &Op);
1851 continue;
1852 }
1853 }
1854 }
1855
1856 if (auto *Shuffle = dyn_cast<ShuffleVectorInst>(Val: Op.get())) {
1857
1858 TypeSize EltSize = DL.getTypeSizeInBits(
1859 Ty: cast<VectorType>(Val: Shuffle->getType())->getElementType());
1860
1861 // For i32 (or greater) shufflevectors, these will be lowered into a
1862 // series of insert / extract elements, which will be coalesced away.
1863 if (EltSize < 16 || !ST->has16BitInsts())
1864 continue;
1865
1866 int NumSubElts, SubIndex;
1867 if (Shuffle->changesLength()) {
1868 if (Shuffle->increasesLength() && Shuffle->isIdentityWithPadding()) {
1869 Ops.push_back(Elt: &Op);
1870 continue;
1871 }
1872
1873 if ((Shuffle->isExtractSubvectorMask(Index&: SubIndex) ||
1874 Shuffle->isInsertSubvectorMask(NumSubElts, Index&: SubIndex)) &&
1875 !(SubIndex & 0x1)) {
1876 Ops.push_back(Elt: &Op);
1877 continue;
1878 }
1879 }
1880
1881 if (Shuffle->isReverse() || Shuffle->isZeroEltSplat() ||
1882 Shuffle->isSingleSource()) {
1883 Ops.push_back(Elt: &Op);
1884 continue;
1885 }
1886 }
1887 }
1888
1889 return !Ops.empty();
1890}
1891
1892bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
1893 const Function *Callee) const {
1894 const TargetMachine &TM = getTLI()->getTargetMachine();
1895 const GCNSubtarget *CallerST
1896 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller));
1897 const GCNSubtarget *CalleeST
1898 = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee));
1899
1900 if (!BaseT::areInlineCompatible(Caller, Callee))
1901 return false;
1902
1903 // FIXME: dx10_clamp can just take the caller setting, but there seems to be
1904 // no way to support merge for backend defined attributes.
1905 SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
1906 SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
1907 if (!CallerMode.isInlineCompatible(CalleeMode))
1908 return false;
1909
1910 if (Callee->hasFnAttribute(Kind: Attribute::AlwaysInline) ||
1911 Callee->hasFnAttribute(Kind: Attribute::InlineHint))
1912 return true;
1913
1914 // Hack to make compile times reasonable.
1915 if (CLOpts.inline_max_bb) {
1916 // Single BB does not increase total BB amount.
1917 if (Callee->size() == 1)
1918 return true;
1919 size_t BBSize = Caller->size() + Callee->size() - 1;
1920 if (BBSize > CLOpts.inline_max_bb) {
1921 LLVM_DEBUG(dbgs() << "AMDGPU inline max-BB rejected inlining "
1922 << Callee->getName() << " into " << Caller->getName()
1923 << ": caller BBs=" << Caller->size() << ", callee BBs="
1924 << Callee->size() << ", combined BBs=" << BBSize
1925 << ", max BBs=" << CLOpts.inline_max_bb << '\n');
1926 return false;
1927 }
1928 }
1929
1930 return true;
1931}
1932
1933static unsigned adjustInliningThresholdUsingCallee(const CallBase *CB,
1934 const SITargetLowering *TLI,
1935 const GCNTTIImpl *TTIImpl) {
1936 const int NrOfSGPRUntilSpill = 26;
1937 const int NrOfVGPRUntilSpill = 32;
1938
1939 const DataLayout &DL = TTIImpl->getDataLayout();
1940
1941 unsigned adjustThreshold = 0;
1942 int SGPRsInUse = 0;
1943 int VGPRsInUse = 0;
1944 for (const Use &A : CB->args()) {
1945 SmallVector<EVT, 4> ValueVTs;
1946 ComputeValueVTs(TLI: *TLI, DL, Ty: A.get()->getType(), ValueVTs);
1947 for (auto ArgVT : ValueVTs) {
1948 unsigned CCRegNum = TLI->getNumRegistersForCallingConv(
1949 Context&: CB->getContext(), CC: CB->getCallingConv(), VT: ArgVT);
1950 if (AMDGPU::isArgPassedInSGPR(CB, ArgNo: CB->getArgOperandNo(U: &A)))
1951 SGPRsInUse += CCRegNum;
1952 else
1953 VGPRsInUse += CCRegNum;
1954 }
1955 }
1956
1957 // The cost of passing function arguments through the stack:
1958 // 1 instruction to put a function argument on the stack in the caller.
1959 // 1 instruction to take a function argument from the stack in callee.
1960 // 1 instruction is explicitly take care of data dependencies in callee
1961 // function.
1962 InstructionCost ArgStackCost(1);
1963 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1964 Opcode: Instruction::Store, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1965 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1966 ArgStackCost += const_cast<GCNTTIImpl *>(TTIImpl)->getMemoryOpCost(
1967 Opcode: Instruction::Load, Src: Type::getInt32Ty(C&: CB->getContext()), Alignment: Align(4),
1968 AddressSpace: AMDGPUAS::PRIVATE_ADDRESS, CostKind: TTI::TCK_SizeAndLatency);
1969
1970 // The penalty cost is computed relative to the cost of instructions and does
1971 // not model any storage costs.
1972 int InstrCost = InlineConstants::getInstrCost(C: CB->getContext());
1973 adjustThreshold +=
1974 static_cast<unsigned>(std::max(a: 0, b: SGPRsInUse - NrOfSGPRUntilSpill) *
1975 ArgStackCost.getValue() * InstrCost);
1976 adjustThreshold +=
1977 static_cast<unsigned>(std::max(a: 0, b: VGPRsInUse - NrOfVGPRUntilSpill) *
1978 ArgStackCost.getValue() * InstrCost);
1979 return adjustThreshold;
1980}
1981
1982static TypeSize getCallArgsTotalAllocaSize(const CallBase *CB,
1983 const DataLayout &DL) {
1984 // If we have a pointer to a private array passed into a function
1985 // it will not be optimized out, leaving scratch usage.
1986 // This function calculates the total size in bytes of the memory that would
1987 // end in scratch if the call was not inlined.
1988 TypeSize AllocaSize = TypeSize::getFixed(ExactSize: 0);
1989 SmallPtrSet<const AllocaInst *, 8> AIVisited;
1990 for (Value *PtrArg : CB->args()) {
1991 PointerType *Ty = dyn_cast<PointerType>(Val: PtrArg->getType());
1992 if (!Ty)
1993 continue;
1994
1995 unsigned AddrSpace = Ty->getAddressSpace();
1996 if (AddrSpace != AMDGPUAS::FLAT_ADDRESS &&
1997 AddrSpace != AMDGPUAS::PRIVATE_ADDRESS)
1998 continue;
1999
2000 const AllocaInst *AI = dyn_cast<AllocaInst>(Val: getUnderlyingObject(V: PtrArg));
2001 if (!AI || !AI->isStaticAlloca() || !AIVisited.insert(Ptr: AI).second)
2002 continue;
2003
2004 if (auto Size = AI->getAllocationSize(DL))
2005 AllocaSize += *Size;
2006 }
2007 return AllocaSize;
2008}
2009
2010int GCNTTIImpl::getInliningLastCallToStaticBonus() const {
2011 return BaseT::getInliningLastCallToStaticBonus() *
2012 getInliningThresholdMultiplier();
2013}
2014
2015unsigned GCNTTIImpl::adjustInliningThreshold(const CallBase *CB) const {
2016 unsigned Threshold = adjustInliningThresholdUsingCallee(CB, TLI, TTIImpl: this);
2017
2018 // Private object passed as arguments may end up in scratch usage if the call
2019 // is not inlined. Increase the inline threshold to promote inlining.
2020 TypeSize AllocaSize = getCallArgsTotalAllocaSize(CB, DL);
2021 if (AllocaSize.getFixedValue() > 0)
2022 Threshold += CLOpts.inline_arg_alloca_cost;
2023 return Threshold;
2024}
2025
2026unsigned GCNTTIImpl::getCallerAllocaCost(const CallBase *CB,
2027 const AllocaInst *AI) const {
2028
2029 // Below the cutoff, assume that the private memory objects would be
2030 // optimized
2031 auto AllocaSize = getCallArgsTotalAllocaSize(CB, DL).getFixedValue();
2032 if (AllocaSize <= CLOpts.inline_arg_alloca_cutoff)
2033 return 0;
2034
2035 // Above the cutoff, we give a cost to each private memory object
2036 // depending its size. If the array can be optimized by SROA this cost is not
2037 // added to the total-cost in the inliner cost analysis.
2038 //
2039 // We choose the total cost of the alloca such that their sum cancels the
2040 // bonus given in the threshold (ArgAllocaCost).
2041 //
2042 // Cost_Alloca_0 + ... + Cost_Alloca_N == ArgAllocaCost
2043 //
2044 // Awkwardly, the ArgAllocaCost bonus is multiplied by threshold-multiplier,
2045 // the single-bb bonus and the vector-bonus.
2046 //
2047 // We compensate the first two multipliers, by repeating logic from the
2048 // inliner-cost in here. The vector-bonus is 0 on AMDGPU.
2049 static_assert(InlinerVectorBonusPercent == 0, "vector bonus assumed to be 0");
2050 unsigned Threshold =
2051 CLOpts.inline_arg_alloca_cost * getInliningThresholdMultiplier();
2052
2053 bool SingleBB = none_of(Range&: *CB->getCalledFunction(), P: [](const BasicBlock &BB) {
2054 return BB.getTerminator()->getNumSuccessors() > 1;
2055 });
2056 if (SingleBB) {
2057 Threshold += Threshold / 2;
2058 }
2059
2060 auto ArgAllocaSize = AI->getAllocationSize(DL);
2061 if (!ArgAllocaSize)
2062 return 0;
2063
2064 // Attribute the bonus proportionally to the alloca size
2065 unsigned AllocaThresholdBonus = static_cast<unsigned>(
2066 (Threshold * ArgAllocaSize->getFixedValue()) / AllocaSize);
2067
2068 return AllocaThresholdBonus;
2069}
2070
2071void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2072 TTI::UnrollingPreferences &UP,
2073 OptimizationRemarkEmitter *ORE) const {
2074 CommonTTI.getUnrollingPreferences(L, SE, UP, ORE);
2075}
2076
2077void GCNTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2078 TTI::PeelingPreferences &PP) const {
2079 CommonTTI.getPeelingPreferences(L, SE, PP);
2080}
2081
2082int GCNTTIImpl::getTransInstrCost(TTI::TargetCostKind CostKind) const {
2083 return getQuarterRateInstrCost(CostKind);
2084}
2085
2086int GCNTTIImpl::get64BitInstrCost(TTI::TargetCostKind CostKind) const {
2087 return ST->hasFullRate64Ops()
2088 ? getFullRateInstrCost()
2089 : ST->hasHalfRate64Ops() ? getHalfRateInstrCost(CostKind)
2090 : getQuarterRateInstrCost(CostKind);
2091}
2092
2093std::pair<InstructionCost, MVT>
2094GCNTTIImpl::getTypeLegalizationCost(Type *Ty) const {
2095 std::pair<InstructionCost, MVT> Cost = BaseT::getTypeLegalizationCost(Ty);
2096 auto Size = DL.getTypeSizeInBits(Ty);
2097 // Maximum load or store can handle 8 dwords for scalar and 4 for
2098 // vector ALU. Let's assume anything above 8 dwords is expensive
2099 // even if legal.
2100 if (Size <= 256)
2101 return Cost;
2102
2103 Cost.first += (Size + 255) / 256;
2104 return Cost;
2105}
2106
2107unsigned GCNTTIImpl::getCacheLineSize() const {
2108 if (ST->hasVmemPrefInsts() || ST->hasSmemPrefetchInsts())
2109 return ST->getDataCacheLineSize();
2110 return 0;
2111}
2112
2113unsigned GCNTTIImpl::getPrefetchDistance() const {
2114 return ST->hasPrefetch() ? 128 : 0;
2115}
2116
2117bool GCNTTIImpl::shouldPrefetchAddressSpace(unsigned AS) const {
2118 return AMDGPU::isFlatGlobalAddrSpace(AS);
2119}
2120
2121void GCNTTIImpl::collectKernelLaunchBounds(
2122 const Function &F,
2123 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
2124 SmallVector<unsigned> MaxNumWorkgroups = AMDGPU::getMaxNumWorkGroups(F);
2125 LB.push_back(Elt: {"amdgpu-max-num-workgroups[0]", MaxNumWorkgroups[0]});
2126 LB.push_back(Elt: {"amdgpu-max-num-workgroups[1]", MaxNumWorkgroups[1]});
2127 LB.push_back(Elt: {"amdgpu-max-num-workgroups[2]", MaxNumWorkgroups[2]});
2128 std::pair<unsigned, unsigned> FlatWorkGroupSize =
2129 ST->getFlatWorkGroupSizes(F);
2130 LB.push_back(Elt: {"amdgpu-flat-work-group-size[0]", FlatWorkGroupSize.first});
2131 LB.push_back(Elt: {"amdgpu-flat-work-group-size[1]", FlatWorkGroupSize.second});
2132 std::pair<unsigned, unsigned> WavesPerEU = ST->getWavesPerEU(F);
2133 LB.push_back(Elt: {"amdgpu-waves-per-eu[0]", WavesPerEU.first});
2134 LB.push_back(Elt: {"amdgpu-waves-per-eu[1]", WavesPerEU.second});
2135}
2136
2137GCNTTIImpl::KnownIEEEMode
2138GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
2139 if (!ST->hasFeature(Feature: AMDGPU::FeatureDX10ClampAndIEEEMode))
2140 return KnownIEEEMode::On; // Only mode on gfx1170+
2141
2142 const Function *F = I.getFunction();
2143 if (!F)
2144 return KnownIEEEMode::Unknown;
2145
2146 Attribute IEEEAttr = F->getFnAttribute(Kind: "amdgpu-ieee");
2147 if (IEEEAttr.isValid())
2148 return IEEEAttr.getValueAsBool() ? KnownIEEEMode::On : KnownIEEEMode::Off;
2149
2150 return AMDGPU::isShader(CC: F->getCallingConv()) ? KnownIEEEMode::Off
2151 : KnownIEEEMode::On;
2152}
2153
2154static unsigned getNumMemoryInsts(unsigned Bytes, unsigned MaxBytes,
2155 bool HasDwordx3) {
2156 unsigned NumInsts = Bytes / MaxBytes;
2157 unsigned Rem = Bytes % MaxBytes;
2158 if (HasDwordx3 && Rem % 16 >= 12) {
2159 ++NumInsts;
2160 Rem -= 12;
2161 }
2162 return NumInsts + llvm::popcount(Value: Rem);
2163}
2164
2165static bool hasUniformPointerOperand(const Instruction *I) {
2166 const auto *LI = dyn_cast_or_null<LoadInst>(Val: I);
2167 if (!LI)
2168 return false;
2169 const Value *Ptr = LI->getPointerOperand();
2170 const DataLayout &DL = LI->getDataLayout();
2171 APInt Offset(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
2172 Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,
2173 /*AllowNonInbounds=*/true);
2174 return AMDGPU::isTriviallyUniform(V: Ptr, CtxI: LI);
2175}
2176
2177static std::pair<bool, bool> getKnownBufferOobProperties(const GCNSubtarget &ST,
2178 const DataLayout &DL,
2179 const Instruction *I,
2180 unsigned Bytes) {
2181 const Value *Ptr = I ? getLoadStorePointerOperand(V: I) : nullptr;
2182 if (!Ptr ||
2183 Ptr->getType()->getPointerAddressSpace() != AMDGPUAS::BUFFER_FAT_POINTER)
2184 return {false, false};
2185 APInt Offset(DL.getIndexTypeSizeInBits(Ty: Ptr->getType()), 0);
2186 bool KnownOffset = true;
2187 while (const auto *GEP = dyn_cast<GEPOperator>(Val: Ptr)) {
2188 KnownOffset = KnownOffset && GEP->accumulateConstantOffset(DL, Offset);
2189 Ptr = GEP->getPointerOperand();
2190 }
2191 const auto *Cast = dyn_cast<AddrSpaceCastInst>(Val: Ptr);
2192 if (Cast) {
2193 if (Cast->getSrcAddressSpace() != AMDGPUAS::BUFFER_RESOURCE)
2194 return {false, false};
2195 Ptr = Cast->getPointerOperand();
2196 }
2197 const auto *Rsrc = dyn_cast<IntrinsicInst>(Val: Ptr);
2198 if (Rsrc && Rsrc->getIntrinsicID() != Intrinsic::amdgcn_make_buffer_rsrc)
2199 Rsrc = nullptr;
2200 if (!Cast && !Rsrc)
2201 return {false, false};
2202
2203 bool NoWrapFromMax =
2204 KnownOffset &&
2205 Offset.ule(RHS: APInt::getAllOnes(numBits: Offset.getBitWidth()) - Bytes);
2206 const auto *NumRecords =
2207 Rsrc ? dyn_cast<ConstantInt>(Val: Rsrc->getArgOperand(i: 2)) : nullptr;
2208 std::optional<unsigned> NumRecordsWidth =
2209 ST.getBufferResourceNumRecordsWidth();
2210 if (!NumRecords || !NumRecordsWidth)
2211 return {NoWrapFromMax, false};
2212 APInt Bound = NumRecords->getValue().zextOrTrunc(width: *NumRecordsWidth);
2213 if (Bound.isAllOnes())
2214 return {NoWrapFromMax, true};
2215 if (!KnownOffset)
2216 return {false, false};
2217 APInt BoundsDiff = Bound.zext(width: 64) - Offset.zext(width: 64);
2218 return {NoWrapFromMax, BoundsDiff.sge(RHS: Bytes) || BoundsDiff.isNonPositive()};
2219}
2220
2221InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2222 Align Alignment,
2223 unsigned AddressSpace,
2224 TTI::TargetCostKind CostKind,
2225 TTI::OperandValueInfo OpInfo,
2226 const Instruction *I) const {
2227 Type *EltTy = Src->getScalarType();
2228 if ((Opcode != Instruction::Load && Opcode != Instruction::Store) ||
2229 CostKind == TTI::TCK_Latency || isa<ScalableVectorType>(Val: Src) ||
2230 (!EltTy->isIntOrPtrTy() && !EltTy->isFloatingPointTy()))
2231 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2232 CostKind, OpInfo, I);
2233
2234 unsigned EltBits = std::max<unsigned>(a: 8, b: DL.getTypeSizeInBits(Ty: EltTy));
2235
2236 auto *VecTy = dyn_cast<FixedVectorType>(Val: Src);
2237 if (VecTy && AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER &&
2238 EltBits % 8 == 0 && !isPowerOf2_32(Value: EltBits) &&
2239 !TLI->shouldCombineMemoryType(VT: TLI->getValueType(DL, Ty: Src))) {
2240 Align EltAlign = commonAlignment(A: Alignment, Offset: EltBits / 8);
2241 return VecTy->getNumElements() * getMemoryOpCost(Opcode, Src: EltTy, Alignment: EltAlign,
2242 AddressSpace, CostKind,
2243 OpInfo);
2244 }
2245
2246 unsigned Bytes = DL.getTypeStoreSize(Ty: Src);
2247 if (Opcode == Instruction::Load &&
2248 (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS ||
2249 AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
2250 Alignment >= std::min(a: Bytes, b: 4u) &&
2251 (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
2252 hasUniformPointerOperand(I))) {
2253 unsigned Dwords = divideCeil(Numerator: Bytes, Denominator: 4);
2254 unsigned MaxBytes = getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace) / 8;
2255 bool HasDwordx3 = ST->hasScalarDwordx3Loads();
2256 // The last byte of a 7-byte remainder is not widened into the scalar load.
2257 if (!VecTy && Bytes % 4)
2258 return getNumMemoryInsts(Bytes: Dwords * 4, MaxBytes: 32, HasDwordx3) + (Bytes % 8 == 7);
2259 if (!HasDwordx3 && Alignment >= 8 && Dwords % 4 == 3)
2260 ++Dwords;
2261 return getNumMemoryInsts(Bytes: Dwords * 4, MaxBytes, HasDwordx3);
2262 }
2263
2264 unsigned MaxBytes =
2265 std::min(a: 16u, b: getLoadStoreVecRegBitWidth(AddrSpace: AddressSpace) / 8);
2266 if (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
2267 !ST->hasMultiDwordFlatScratchAddressing())
2268 MaxBytes = getLoadStoreVecRegBitWidth(AddrSpace: AMDGPUAS::PRIVATE_ADDRESS) / 8;
2269 if (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
2270 ST->hasLDSMisalignedBugInWGPMode())
2271 MaxBytes = std::clamp<uint64_t>(val: Alignment.value(), lo: 4, hi: MaxBytes);
2272 bool IsDS = AddressSpace == AMDGPUAS::LOCAL_ADDRESS ||
2273 AddressSpace == AMDGPUAS::REGION_ADDRESS;
2274 bool HasDwordx3 = ST->hasDwordx3LoadStores();
2275 bool HasUnalignedAccess = ST->hasUnalignedBufferAccessEnabled();
2276 if (AddressSpace == AMDGPUAS::PRIVATE_ADDRESS ||
2277 AddressSpace == AMDGPUAS::FLAT_ADDRESS)
2278 HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled();
2279 else if (IsDS) {
2280 HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled();
2281 if ((Alignment >= 4 || ST->hasLDSMisalignedBugInWGPMode()) && Alignment < 8)
2282 MaxBytes = 8;
2283 HasDwordx3 = HasDwordx3 &&
2284 (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess));
2285 }
2286 if (AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) {
2287 auto [NoWrapFromMax, NoPartialOOB] =
2288 getKnownBufferOobProperties(ST: *ST, DL, I, Bytes);
2289 MaxBytes = ST->getMaxBufferIntrinsicWidth(DL, T: Src, A: Alignment, NoWrapFromMax,
2290 NoPartialOOB) /
2291 8;
2292 }
2293 if (Alignment < 4 && !HasUnalignedAccess)
2294 MaxBytes = std::min<unsigned>(a: MaxBytes, b: Alignment.value());
2295 bool MergesDwords = AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER ||
2296 (ST->hasFlatInstOffsets() &&
2297 (AMDGPU::isExtendedGlobalAddrSpace(AS: AddressSpace) ||
2298 (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
2299 !ST->hasFlatSegmentOffsetBug())));
2300
2301 if (Opcode == Instruction::Load &&
2302 AddressSpace != AMDGPUAS::BUFFER_FAT_POINTER) {
2303 if (VecTy && Alignment >= PowerOf2Ceil(A: Bytes))
2304 return divideCeil(Numerator: Bytes, Denominator: MaxBytes);
2305 if (EltBits <= 16 && Alignment >= 4 && Bytes % 4) {
2306 unsigned Piece = std::min(a: EltBits == 8 ? 8u : 16u, b: MaxBytes);
2307 unsigned Granule = std::min<unsigned>(a: Alignment.value(), b: Piece);
2308 if (!IsDS && !MergesDwords)
2309 return Bytes / Piece + llvm::popcount(Value: Bytes % Piece / Granule) +
2310 (Bytes % Granule != 0);
2311 unsigned Tail = Bytes % Granule;
2312 if (Tail > EltBits / 8) {
2313 Bytes += std::max<unsigned>(a: 4, b: PowerOf2Ceil(A: Tail)) - Tail;
2314 if (IsDS)
2315 HasDwordx3 = false;
2316 }
2317 }
2318 }
2319 if (!MergesDwords && Bytes % MaxBytes > 12)
2320 HasDwordx3 = false;
2321 return getNumMemoryInsts(Bytes, MaxBytes, HasDwordx3);
2322}
2323
2324unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) const {
2325 if (VectorType *VecTy = dyn_cast<VectorType>(Val: Tp)) {
2326 if (VecTy->getElementType()->isIntegerTy(BitWidth: 8)) {
2327 unsigned ElementCount = VecTy->getElementCount().getFixedValue();
2328 return divideCeil(Numerator: ElementCount - 1, Denominator: 4);
2329 }
2330 }
2331 return BaseT::getNumberOfParts(Tp);
2332}
2333
2334ValueUniformity GCNTTIImpl::getValueUniformity(const Value *V) const {
2335 if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(Val: V)) {
2336 switch (Intrinsic->getIntrinsicID()) {
2337 case Intrinsic::amdgcn_wave_shuffle:
2338 return ValueUniformity::Custom;
2339 default:
2340 break;
2341 }
2342 }
2343
2344 if (isAlwaysUniform(V))
2345 return ValueUniformity::AlwaysUniform;
2346
2347 if (isSourceOfDivergence(V))
2348 return ValueUniformity::NeverUniform;
2349
2350 return ValueUniformity::Default;
2351}
2352
2353InstructionCost GCNTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2354 StackOffset BaseOffset,
2355 bool HasBaseReg, int64_t Scale,
2356 unsigned AddrSpace) const {
2357 if (HasBaseReg && Scale != 0) {
2358 // gfx1250+ can fold base+scale*index when scale matches the memory access
2359 // size (scale_offset bit). Supported for flat/global/constant/scratch
2360 // (VMEM, max 128 bits) and constant_32bit (SMRD, capped to 128 bits here).
2361 if (getST()->hasScaleOffset() && Ty && Ty->isSized() &&
2362 (AMDGPU::isExtendedGlobalAddrSpace(AS: AddrSpace) ||
2363 AddrSpace == AMDGPUAS::FLAT_ADDRESS ||
2364 AddrSpace == AMDGPUAS::PRIVATE_ADDRESS)) {
2365 TypeSize StoreSize = getDataLayout().getTypeStoreSize(Ty);
2366 if (TypeSize::isKnownLE(LHS: StoreSize, RHS: TypeSize::getFixed(ExactSize: 16)) &&
2367 static_cast<int64_t>(StoreSize.getFixedValue()) == Scale)
2368 return 0;
2369 }
2370 return 1;
2371 }
2372 return BaseT::getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale,
2373 AddrSpace);
2374}
2375
2376bool GCNTTIImpl::isLSRCostLess(const TTI::LSRCost &A,
2377 const TTI::LSRCost &B) const {
2378 // Favor lower per-iteration work over preheader/setup costs.
2379 // AMDGPU lacks rich addressing modes, so ScaleCost is folded into the
2380 // effective instruction count (base+scale*index requires a separate ADD).
2381 unsigned EffInsnsA = A.Insns + A.ScaleCost;
2382 unsigned EffInsnsB = B.Insns + B.ScaleCost;
2383
2384 return std::tie(args&: EffInsnsA, args: A.NumIVMuls, args: A.AddRecCost, args: A.NumBaseAdds,
2385 args: A.SetupCost, args: A.ImmCost, args: A.NumRegs) <
2386 std::tie(args&: EffInsnsB, args: B.NumIVMuls, args: B.AddRecCost, args: B.NumBaseAdds,
2387 args: B.SetupCost, args: B.ImmCost, args: B.NumRegs);
2388}
2389
2390bool GCNTTIImpl::isNumRegsMajorCostOfLSR() const {
2391 // isLSRCostLess de-prioritizes register count; keep consistent.
2392 return false;
2393}
2394
2395bool GCNTTIImpl::shouldDropLSRSolutionIfLessProfitable() const {
2396 // Prefer the baseline when LSR cannot clearly reduce per-iteration work.
2397 return true;
2398}
2399
2400bool GCNTTIImpl::isUniform(const Instruction *I,
2401 const SmallBitVector &UniformArgs) const {
2402 const IntrinsicInst *Intrinsic = cast<IntrinsicInst>(Val: I);
2403 switch (Intrinsic->getIntrinsicID()) {
2404 case Intrinsic::amdgcn_wave_shuffle:
2405 // wave_shuffle(Value, Index): result is uniform when either Value or Index
2406 // is uniform.
2407 return UniformArgs[0] || UniformArgs[1];
2408 default:
2409 llvm_unreachable("unexpected intrinsic in isUniform");
2410 }
2411}
2412