1//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/Analysis/CFG.h"
11#include "llvm/Analysis/LoopIterator.h"
12#include "llvm/Analysis/TargetLibraryInfo.h"
13#include "llvm/Analysis/TargetTransformInfoImpl.h"
14#include "llvm/IR/CFG.h"
15#include "llvm/IR/Dominators.h"
16#include "llvm/IR/Instruction.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/IntrinsicInst.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Operator.h"
21#include "llvm/InitializePasses.h"
22#include "llvm/Support/CommandLine.h"
23#include <optional>
24#include <utility>
25
26using namespace llvm;
27using namespace PatternMatch;
28
29#define DEBUG_TYPE "tti"
30
31static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(Val: false),
32 cl::Hidden,
33 cl::desc("Recognize reduction patterns."));
34
35static cl::opt<unsigned> CacheLineSize(
36 "cache-line-size", cl::init(Val: 0), cl::Hidden,
37 cl::desc("Use this to override the target cache line size when "
38 "specified by the user."));
39
40static cl::opt<unsigned> MinPageSize(
41 "min-page-size", cl::init(Val: 0), cl::Hidden,
42 cl::desc("Use this to override the target's minimum page size."));
43
44static cl::opt<unsigned> PredictableBranchThreshold(
45 "predictable-branch-threshold", cl::init(Val: 99), cl::Hidden,
46 cl::desc(
47 "Use this to override the target's predictable branch threshold (%)."));
48
49namespace {
50/// No-op implementation of the TTI interface using the utility base
51/// classes.
52///
53/// This is used when no target specific information is available.
54struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
55 explicit NoTTIImpl(const DataLayout &DL)
56 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
57};
58} // namespace
59
60TargetTransformInfo::TargetTransformInfo(
61 std::unique_ptr<const TargetTransformInfoImplBase> Impl)
62 : TTIImpl(std::move(Impl)) {}
63
64bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {
65 // If the loop has irreducible control flow, it can not be converted to
66 // Hardware loop.
67 LoopBlocksRPO RPOT(L);
68 RPOT.perform(LI: &LI);
69 if (containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI))
70 return false;
71 return true;
72}
73
74IntrinsicCostAttributes::IntrinsicCostAttributes(
75 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost,
76 bool TypeBasedOnly, const TargetLibraryInfo *LibInfo)
77 : II(dyn_cast<IntrinsicInst>(Val: &CI)), RetTy(CI.getType()), IID(Id),
78 ScalarizationCost(ScalarizationCost), LibInfo(LibInfo) {
79
80 if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: &CI))
81 FMF = FPMO->getFastMathFlags();
82
83 if (!TypeBasedOnly)
84 Arguments.insert(I: Arguments.begin(), From: CI.arg_begin(), To: CI.arg_end());
85 FunctionType *FTy = CI.getCalledFunction()->getFunctionType();
86 ParamTys.insert(I: ParamTys.begin(), From: FTy->param_begin(), To: FTy->param_end());
87}
88
89IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
90 ArrayRef<Type *> Tys,
91 FastMathFlags Flags,
92 const IntrinsicInst *I,
93 InstructionCost ScalarCost)
94 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
95 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
96}
97
98IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
99 ArrayRef<const Value *> Args)
100 : RetTy(Ty), IID(Id) {
101
102 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
103 ParamTys.reserve(N: Arguments.size());
104 for (const Value *Argument : Arguments)
105 ParamTys.push_back(Elt: Argument->getType());
106}
107
108IntrinsicCostAttributes::IntrinsicCostAttributes(
109 Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
110 ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,
111 InstructionCost ScalarCost, TargetLibraryInfo const *LibInfo)
112 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),
113 LibInfo(LibInfo) {
114 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
115 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
116}
117
118HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) {
119 // Match default options:
120 // - hardware-loop-counter-bitwidth = 32
121 // - hardware-loop-decrement = 1
122 CountType = Type::getInt32Ty(C&: L->getHeader()->getContext());
123 LoopDecrement = ConstantInt::get(Ty: CountType, V: 1);
124}
125
126bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
127 LoopInfo &LI, DominatorTree &DT,
128 bool ForceNestedLoop,
129 bool ForceHardwareLoopPHI) {
130 SmallVector<BasicBlock *, 4> ExitingBlocks;
131 L->getExitingBlocks(ExitingBlocks);
132
133 for (BasicBlock *BB : ExitingBlocks) {
134 // If we pass the updated counter back through a phi, we need to know
135 // which latch the updated value will be coming from.
136 if (!L->isLoopLatch(BB)) {
137 if (ForceHardwareLoopPHI || CounterInReg)
138 continue;
139 }
140
141 const SCEV *EC = SE.getExitCount(L, ExitingBlock: BB);
142 if (isa<SCEVCouldNotCompute>(Val: EC))
143 continue;
144 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(Val: EC)) {
145 if (ConstEC->getValue()->isZero())
146 continue;
147 } else if (!SE.isLoopInvariant(S: EC, L))
148 continue;
149
150 if (SE.getTypeSizeInBits(Ty: EC->getType()) > CountType->getBitWidth())
151 continue;
152
153 // If this exiting block is contained in a nested loop, it is not eligible
154 // for insertion of the branch-and-decrement since the inner loop would
155 // end up messing up the value in the CTR.
156 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
157 continue;
158
159 // We now have a loop-invariant count of loop iterations (which is not the
160 // constant zero) for which we know that this loop will not exit via this
161 // existing block.
162
163 // We need to make sure that this block will run on every loop iteration.
164 // For this to be true, we must dominate all blocks with backedges. Such
165 // blocks are in-loop predecessors to the header block.
166 bool NotAlways = false;
167 for (BasicBlock *Pred : predecessors(BB: L->getHeader())) {
168 if (!L->contains(BB: Pred))
169 continue;
170
171 if (!DT.dominates(A: BB, B: Pred)) {
172 NotAlways = true;
173 break;
174 }
175 }
176
177 if (NotAlways)
178 continue;
179
180 // Make sure this blocks ends with a conditional branch.
181 Instruction *TI = BB->getTerminator();
182 if (!TI)
183 continue;
184
185 if (BranchInst *BI = dyn_cast<BranchInst>(Val: TI)) {
186 if (!BI->isConditional())
187 continue;
188
189 ExitBranch = BI;
190 } else
191 continue;
192
193 // Note that this block may not be the loop latch block, even if the loop
194 // has a latch block.
195 ExitBlock = BB;
196 ExitCount = EC;
197 break;
198 }
199
200 if (!ExitBlock)
201 return false;
202 return true;
203}
204
205TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
206 : TTIImpl(std::make_unique<NoTTIImpl>(args: DL)) {}
207
208TargetTransformInfo::~TargetTransformInfo() = default;
209
210TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
211 : TTIImpl(std::move(Arg.TTIImpl)) {}
212
213TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
214 TTIImpl = std::move(RHS.TTIImpl);
215 return *this;
216}
217
218unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
219 return TTIImpl->getInliningThresholdMultiplier();
220}
221
222unsigned
223TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const {
224 return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier();
225}
226
227unsigned
228TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
229 const {
230 return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
231}
232
233int TargetTransformInfo::getInliningLastCallToStaticBonus() const {
234 return TTIImpl->getInliningLastCallToStaticBonus();
235}
236
237unsigned
238TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
239 return TTIImpl->adjustInliningThreshold(CB);
240}
241
242unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,
243 const AllocaInst *AI) const {
244 return TTIImpl->getCallerAllocaCost(CB, AI);
245}
246
247int TargetTransformInfo::getInlinerVectorBonusPercent() const {
248 return TTIImpl->getInlinerVectorBonusPercent();
249}
250
251InstructionCost TargetTransformInfo::getGEPCost(
252 Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands,
253 Type *AccessType, TTI::TargetCostKind CostKind) const {
254 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
255}
256
257InstructionCost TargetTransformInfo::getPointersChainCost(
258 ArrayRef<const Value *> Ptrs, const Value *Base,
259 const TTI::PointersChainInfo &Info, Type *AccessTy,
260 TTI::TargetCostKind CostKind) const {
261 assert((Base || !Info.isSameBase()) &&
262 "If pointers have same base address it has to be provided.");
263 return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
264}
265
266unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(
267 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,
268 BlockFrequencyInfo *BFI) const {
269 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
270}
271
272InstructionCost
273TargetTransformInfo::getInstructionCost(const User *U,
274 ArrayRef<const Value *> Operands,
275 enum TargetCostKind CostKind) const {
276 InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind);
277 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&
278 "TTI should not produce negative costs!");
279 return Cost;
280}
281
282BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
283 return PredictableBranchThreshold.getNumOccurrences() > 0
284 ? BranchProbability(PredictableBranchThreshold, 100)
285 : TTIImpl->getPredictableBranchThreshold();
286}
287
288InstructionCost TargetTransformInfo::getBranchMispredictPenalty() const {
289 return TTIImpl->getBranchMispredictPenalty();
290}
291
292bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {
293 return TTIImpl->hasBranchDivergence(F);
294}
295
296bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
297 if (const auto *Call = dyn_cast<CallBase>(Val: V)) {
298 if (Call->hasFnAttr(Kind: Attribute::NoDivergenceSource))
299 return false;
300 }
301 return TTIImpl->isSourceOfDivergence(V);
302}
303
304bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
305 return TTIImpl->isAlwaysUniform(V);
306}
307
308bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS,
309 unsigned ToAS) const {
310 return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS);
311}
312
313bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS,
314 unsigned ToAS) const {
315 return TTIImpl->addrspacesMayAlias(AS0: FromAS, AS1: ToAS);
316}
317
318unsigned TargetTransformInfo::getFlatAddressSpace() const {
319 return TTIImpl->getFlatAddressSpace();
320}
321
322bool TargetTransformInfo::collectFlatAddressOperands(
323 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {
324 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);
325}
326
327bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,
328 unsigned ToAS) const {
329 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);
330}
331
332bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace(
333 unsigned AS) const {
334 return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS);
335}
336
337unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {
338 return TTIImpl->getAssumedAddrSpace(V);
339}
340
341bool TargetTransformInfo::isSingleThreaded() const {
342 return TTIImpl->isSingleThreaded();
343}
344
345std::pair<const Value *, unsigned>
346TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const {
347 return TTIImpl->getPredicatedAddrSpace(V);
348}
349
350Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(
351 IntrinsicInst *II, Value *OldV, Value *NewV) const {
352 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
353}
354
355bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
356 return TTIImpl->isLoweredToCall(F);
357}
358
359bool TargetTransformInfo::isHardwareLoopProfitable(
360 Loop *L, ScalarEvolution &SE, AssumptionCache &AC,
361 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {
362 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
363}
364
365unsigned TargetTransformInfo::getEpilogueVectorizationMinVF() const {
366 return TTIImpl->getEpilogueVectorizationMinVF();
367}
368
369bool TargetTransformInfo::preferPredicateOverEpilogue(
370 TailFoldingInfo *TFI) const {
371 return TTIImpl->preferPredicateOverEpilogue(TFI);
372}
373
374TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(
375 bool IVUpdateMayOverflow) const {
376 return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);
377}
378
379std::optional<Instruction *>
380TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
381 IntrinsicInst &II) const {
382 return TTIImpl->instCombineIntrinsic(IC, II);
383}
384
385std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
386 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
387 bool &KnownBitsComputed) const {
388 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
389 KnownBitsComputed);
390}
391
392std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
393 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
394 APInt &UndefElts2, APInt &UndefElts3,
395 std::function<void(Instruction *, unsigned, APInt, APInt &)>
396 SimplifyAndSetOp) const {
397 return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
398 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
399 SimplifyAndSetOp);
400}
401
402void TargetTransformInfo::getUnrollingPreferences(
403 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP,
404 OptimizationRemarkEmitter *ORE) const {
405 return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE);
406}
407
408void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
409 PeelingPreferences &PP) const {
410 return TTIImpl->getPeelingPreferences(L, SE, PP);
411}
412
413bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
414 return TTIImpl->isLegalAddImmediate(Imm);
415}
416
417bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const {
418 return TTIImpl->isLegalAddScalableImmediate(Imm);
419}
420
421bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
422 return TTIImpl->isLegalICmpImmediate(Imm);
423}
424
425bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
426 int64_t BaseOffset,
427 bool HasBaseReg, int64_t Scale,
428 unsigned AddrSpace,
429 Instruction *I,
430 int64_t ScalableOffset) const {
431 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
432 Scale, AddrSpace, I, ScalableOffset);
433}
434
435bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1,
436 const LSRCost &C2) const {
437 return TTIImpl->isLSRCostLess(C1, C2);
438}
439
440bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
441 return TTIImpl->isNumRegsMajorCostOfLSR();
442}
443
444bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
445 return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
446}
447
448bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {
449 return TTIImpl->isProfitableLSRChainElement(I);
450}
451
452bool TargetTransformInfo::canMacroFuseCmp() const {
453 return TTIImpl->canMacroFuseCmp();
454}
455
456bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,
457 ScalarEvolution *SE, LoopInfo *LI,
458 DominatorTree *DT, AssumptionCache *AC,
459 TargetLibraryInfo *LibInfo) const {
460 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
461}
462
463TTI::AddressingModeKind
464TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
465 ScalarEvolution *SE) const {
466 return TTIImpl->getPreferredAddressingMode(L, SE);
467}
468
469bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, Align Alignment,
470 unsigned AddressSpace) const {
471 return TTIImpl->isLegalMaskedStore(DataType, Alignment, AddressSpace);
472}
473
474bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, Align Alignment,
475 unsigned AddressSpace) const {
476 return TTIImpl->isLegalMaskedLoad(DataType, Alignment, AddressSpace);
477}
478
479bool TargetTransformInfo::isLegalNTStore(Type *DataType,
480 Align Alignment) const {
481 return TTIImpl->isLegalNTStore(DataType, Alignment);
482}
483
484bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
485 return TTIImpl->isLegalNTLoad(DataType, Alignment);
486}
487
488bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,
489 ElementCount NumElements) const {
490 return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);
491}
492
493bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
494 Align Alignment) const {
495 return TTIImpl->isLegalMaskedGather(DataType, Alignment);
496}
497
498bool TargetTransformInfo::isLegalAltInstr(
499 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
500 const SmallBitVector &OpcodeMask) const {
501 return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
502}
503
504bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
505 Align Alignment) const {
506 return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
507}
508
509bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,
510 Align Alignment) const {
511 return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);
512}
513
514bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType,
515 Align Alignment) const {
516 return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment);
517}
518
519bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType,
520 Align Alignment) const {
521 return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment);
522}
523
524bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType,
525 Align Alignment) const {
526 return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment);
527}
528
529bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
530 Align Alignment) const {
531 return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
532}
533
534bool TargetTransformInfo::isLegalInterleavedAccessType(
535 VectorType *VTy, unsigned Factor, Align Alignment,
536 unsigned AddrSpace) const {
537 return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,
538 AddrSpace);
539}
540
541bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
542 Type *DataType) const {
543 return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
544}
545
546bool TargetTransformInfo::enableOrderedReductions() const {
547 return TTIImpl->enableOrderedReductions();
548}
549
550bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
551 return TTIImpl->hasDivRemOp(DataType, IsSigned);
552}
553
554bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
555 unsigned AddrSpace) const {
556 return TTIImpl->hasVolatileVariant(I, AddrSpace);
557}
558
559bool TargetTransformInfo::prefersVectorizedAddressing() const {
560 return TTIImpl->prefersVectorizedAddressing();
561}
562
563InstructionCost TargetTransformInfo::getScalingFactorCost(
564 Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,
565 int64_t Scale, unsigned AddrSpace) const {
566 InstructionCost Cost = TTIImpl->getScalingFactorCost(
567 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
568 assert(Cost >= 0 && "TTI should not produce negative costs!");
569 return Cost;
570}
571
572bool TargetTransformInfo::LSRWithInstrQueries() const {
573 return TTIImpl->LSRWithInstrQueries();
574}
575
576bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
577 return TTIImpl->isTruncateFree(Ty1, Ty2);
578}
579
580bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
581 return TTIImpl->isProfitableToHoist(I);
582}
583
584bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
585
586bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
587 return TTIImpl->isTypeLegal(Ty);
588}
589
590unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {
591 return TTIImpl->getRegUsageForType(Ty);
592}
593
594bool TargetTransformInfo::shouldBuildLookupTables() const {
595 return TTIImpl->shouldBuildLookupTables();
596}
597
598bool TargetTransformInfo::shouldBuildLookupTablesForConstant(
599 Constant *C) const {
600 return TTIImpl->shouldBuildLookupTablesForConstant(C);
601}
602
603bool TargetTransformInfo::shouldBuildRelLookupTables() const {
604 return TTIImpl->shouldBuildRelLookupTables();
605}
606
607bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
608 return TTIImpl->useColdCCForColdCall(F);
609}
610
611bool TargetTransformInfo::isTargetIntrinsicTriviallyScalarizable(
612 Intrinsic::ID ID) const {
613 return TTIImpl->isTargetIntrinsicTriviallyScalarizable(ID);
614}
615
616bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(
617 Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
618 return TTIImpl->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
619}
620
621bool TargetTransformInfo::isTargetIntrinsicWithOverloadTypeAtArg(
622 Intrinsic::ID ID, int OpdIdx) const {
623 return TTIImpl->isTargetIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);
624}
625
626bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField(
627 Intrinsic::ID ID, int RetIdx) const {
628 return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
629}
630
631InstructionCost TargetTransformInfo::getScalarizationOverhead(
632 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
633 TTI::TargetCostKind CostKind, bool ForPoisonSrc,
634 ArrayRef<Value *> VL) const {
635 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
636 CostKind, ForPoisonSrc, VL);
637}
638
639InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
640 ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
641 TTI::TargetCostKind CostKind) const {
642 return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind);
643}
644
645bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
646 return TTIImpl->supportsEfficientVectorElementLoadStore();
647}
648
649bool TargetTransformInfo::supportsTailCalls() const {
650 return TTIImpl->supportsTailCalls();
651}
652
653bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const {
654 return TTIImpl->supportsTailCallFor(CB);
655}
656
657bool TargetTransformInfo::enableAggressiveInterleaving(
658 bool LoopHasReductions) const {
659 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
660}
661
662TargetTransformInfo::MemCmpExpansionOptions
663TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
664 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);
665}
666
667bool TargetTransformInfo::enableSelectOptimize() const {
668 return TTIImpl->enableSelectOptimize();
669}
670
671bool TargetTransformInfo::shouldTreatInstructionLikeSelect(
672 const Instruction *I) const {
673 return TTIImpl->shouldTreatInstructionLikeSelect(I);
674}
675
676bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
677 return TTIImpl->enableInterleavedAccessVectorization();
678}
679
680bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
681 return TTIImpl->enableMaskedInterleavedAccessVectorization();
682}
683
684bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
685 return TTIImpl->isFPVectorizationPotentiallyUnsafe();
686}
687
688bool
689TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
690 unsigned BitWidth,
691 unsigned AddressSpace,
692 Align Alignment,
693 unsigned *Fast) const {
694 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,
695 AddressSpace, Alignment, Fast);
696}
697
698TargetTransformInfo::PopcntSupportKind
699TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
700 return TTIImpl->getPopcntSupport(IntTyWidthInBit);
701}
702
703bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
704 return TTIImpl->haveFastSqrt(Ty);
705}
706
707bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
708 const Instruction *I) const {
709 return TTIImpl->isExpensiveToSpeculativelyExecute(I);
710}
711
712bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
713 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
714}
715
716InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {
717 InstructionCost Cost = TTIImpl->getFPOpCost(Ty);
718 assert(Cost >= 0 && "TTI should not produce negative costs!");
719 return Cost;
720}
721
722InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,
723 unsigned Idx,
724 const APInt &Imm,
725 Type *Ty) const {
726 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
727 assert(Cost >= 0 && "TTI should not produce negative costs!");
728 return Cost;
729}
730
731InstructionCost
732TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,
733 TTI::TargetCostKind CostKind) const {
734 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);
735 assert(Cost >= 0 && "TTI should not produce negative costs!");
736 return Cost;
737}
738
739InstructionCost TargetTransformInfo::getIntImmCostInst(
740 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,
741 TTI::TargetCostKind CostKind, Instruction *Inst) const {
742 InstructionCost Cost =
743 TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
744 assert(Cost >= 0 && "TTI should not produce negative costs!");
745 return Cost;
746}
747
748InstructionCost
749TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
750 const APInt &Imm, Type *Ty,
751 TTI::TargetCostKind CostKind) const {
752 InstructionCost Cost =
753 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
754 assert(Cost >= 0 && "TTI should not produce negative costs!");
755 return Cost;
756}
757
758bool TargetTransformInfo::preferToKeepConstantsAttached(
759 const Instruction &Inst, const Function &Fn) const {
760 return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);
761}
762
763unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
764 return TTIImpl->getNumberOfRegisters(ClassID);
765}
766
767bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty,
768 bool IsStore) const {
769 return TTIImpl->hasConditionalLoadStoreForType(Ty, IsStore);
770}
771
772unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,
773 Type *Ty) const {
774 return TTIImpl->getRegisterClassForType(Vector, Ty);
775}
776
777const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
778 return TTIImpl->getRegisterClassName(ClassID);
779}
780
781TypeSize TargetTransformInfo::getRegisterBitWidth(
782 TargetTransformInfo::RegisterKind K) const {
783 return TTIImpl->getRegisterBitWidth(K);
784}
785
786unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
787 return TTIImpl->getMinVectorRegisterBitWidth();
788}
789
790std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {
791 return TTIImpl->getMaxVScale();
792}
793
794std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {
795 return TTIImpl->getVScaleForTuning();
796}
797
798bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const {
799 return TTIImpl->isVScaleKnownToBeAPowerOfTwo();
800}
801
802bool TargetTransformInfo::shouldMaximizeVectorBandwidth(
803 TargetTransformInfo::RegisterKind K) const {
804 return TTIImpl->shouldMaximizeVectorBandwidth(K);
805}
806
807ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,
808 bool IsScalable) const {
809 return TTIImpl->getMinimumVF(ElemWidth, IsScalable);
810}
811
812unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,
813 unsigned Opcode) const {
814 return TTIImpl->getMaximumVF(ElemWidth, Opcode);
815}
816
817unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
818 Type *ScalarValTy) const {
819 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
820}
821
822bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
823 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
824 return TTIImpl->shouldConsiderAddressTypePromotion(
825 I, AllowPromotionWithoutCommonHeader);
826}
827
828unsigned TargetTransformInfo::getCacheLineSize() const {
829 return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize
830 : TTIImpl->getCacheLineSize();
831}
832
833std::optional<unsigned>
834TargetTransformInfo::getCacheSize(CacheLevel Level) const {
835 return TTIImpl->getCacheSize(Level);
836}
837
838std::optional<unsigned>
839TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {
840 return TTIImpl->getCacheAssociativity(Level);
841}
842
843std::optional<unsigned> TargetTransformInfo::getMinPageSize() const {
844 return MinPageSize.getNumOccurrences() > 0 ? MinPageSize
845 : TTIImpl->getMinPageSize();
846}
847
848unsigned TargetTransformInfo::getPrefetchDistance() const {
849 return TTIImpl->getPrefetchDistance();
850}
851
852unsigned TargetTransformInfo::getMinPrefetchStride(
853 unsigned NumMemAccesses, unsigned NumStridedMemAccesses,
854 unsigned NumPrefetches, bool HasCall) const {
855 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
856 NumPrefetches, HasCall);
857}
858
859unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
860 return TTIImpl->getMaxPrefetchIterationsAhead();
861}
862
863bool TargetTransformInfo::enableWritePrefetching() const {
864 return TTIImpl->enableWritePrefetching();
865}
866
867bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
868 return TTIImpl->shouldPrefetchAddressSpace(AS);
869}
870
871InstructionCost TargetTransformInfo::getPartialReductionCost(
872 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
873 ElementCount VF, PartialReductionExtendKind OpAExtend,
874 PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
875 TTI::TargetCostKind CostKind) const {
876 return TTIImpl->getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
877 AccumType, VF, OpAExtend, OpBExtend,
878 BinOp, CostKind);
879}
880
881unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
882 return TTIImpl->getMaxInterleaveFactor(VF);
883}
884
885TargetTransformInfo::OperandValueInfo
886TargetTransformInfo::getOperandInfo(const Value *V) {
887 OperandValueKind OpInfo = OK_AnyValue;
888 OperandValueProperties OpProps = OP_None;
889
890 // undef/poison don't materialize constants.
891 if (isa<UndefValue>(Val: V))
892 return {.Kind: OK_AnyValue, .Properties: OP_None};
893
894 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V)) {
895 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
896 if (CI->getValue().isPowerOf2())
897 OpProps = OP_PowerOf2;
898 else if (CI->getValue().isNegatedPowerOf2())
899 OpProps = OP_NegatedPowerOf2;
900 }
901 return {.Kind: OK_UniformConstantValue, .Properties: OpProps};
902 }
903
904 // A broadcast shuffle creates a uniform value.
905 // TODO: Add support for non-zero index broadcasts.
906 // TODO: Add support for different source vector width.
907 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(Val: V))
908 if (ShuffleInst->isZeroEltSplat())
909 OpInfo = OK_UniformValue;
910
911 const Value *Splat = getSplatValue(V);
912
913 // Check for a splat of a constant or for a non uniform vector of constants
914 // and check if the constant(s) are all powers of two.
915 if (Splat) {
916 // Check for a splat of a uniform value. This is not loop aware, so return
917 // true only for the obviously uniform cases (argument, globalvalue)
918 if (isa<Argument>(Val: Splat) || isa<GlobalValue>(Val: Splat)) {
919 OpInfo = OK_UniformValue;
920 } else if (isa<Constant>(Val: Splat)) {
921 OpInfo = OK_UniformConstantValue;
922 if (auto *CI = dyn_cast<ConstantInt>(Val: Splat)) {
923 if (CI->getValue().isPowerOf2())
924 OpProps = OP_PowerOf2;
925 else if (CI->getValue().isNegatedPowerOf2())
926 OpProps = OP_NegatedPowerOf2;
927 }
928 }
929 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
930 OpInfo = OK_NonUniformConstantValue;
931 bool AllPow2 = true, AllNegPow2 = true;
932 for (uint64_t I = 0, E = CDS->getNumElements(); I != E; ++I) {
933 if (auto *CI = dyn_cast<ConstantInt>(Val: CDS->getElementAsConstant(i: I))) {
934 AllPow2 &= CI->getValue().isPowerOf2();
935 AllNegPow2 &= CI->getValue().isNegatedPowerOf2();
936 if (AllPow2 || AllNegPow2)
937 continue;
938 }
939 AllPow2 = AllNegPow2 = false;
940 break;
941 }
942 OpProps = AllPow2 ? OP_PowerOf2 : OpProps;
943 OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps;
944 } else if (isa<ConstantVector>(Val: V) || isa<ConstantDataVector>(Val: V)) {
945 OpInfo = OK_NonUniformConstantValue;
946 }
947
948 return {.Kind: OpInfo, .Properties: OpProps};
949}
950
951InstructionCost TargetTransformInfo::getArithmeticInstrCost(
952 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
953 OperandValueInfo Op1Info, OperandValueInfo Op2Info,
954 ArrayRef<const Value *> Args, const Instruction *CxtI,
955 const TargetLibraryInfo *TLibInfo) const {
956
957 // Use call cost for frem intructions that have platform specific vector math
958 // functions, as those will be replaced with calls later by SelectionDAG or
959 // ReplaceWithVecLib pass.
960 if (TLibInfo && Opcode == Instruction::FRem) {
961 VectorType *VecTy = dyn_cast<VectorType>(Val: Ty);
962 LibFunc Func;
963 if (VecTy &&
964 TLibInfo->getLibFunc(Opcode: Instruction::FRem, Ty: Ty->getScalarType(), F&: Func) &&
965 TLibInfo->isFunctionVectorizable(F: TLibInfo->getName(F: Func),
966 VF: VecTy->getElementCount()))
967 return getCallInstrCost(F: nullptr, RetTy: VecTy, Tys: {VecTy, VecTy}, CostKind);
968 }
969
970 InstructionCost Cost =
971 TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind,
972 Opd1Info: Op1Info, Opd2Info: Op2Info,
973 Args, CxtI);
974 assert(Cost >= 0 && "TTI should not produce negative costs!");
975 return Cost;
976}
977
978InstructionCost TargetTransformInfo::getAltInstrCost(
979 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
980 const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
981 InstructionCost Cost =
982 TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
983 assert(Cost >= 0 && "TTI should not produce negative costs!");
984 return Cost;
985}
986
987InstructionCost TargetTransformInfo::getShuffleCost(
988 ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef<int> Mask,
989 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
990 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
991 assert((Mask.empty() || DstTy->isScalableTy() ||
992 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
993 "Expected the Mask to match the return size if given");
994 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
995 "Expected the same scalar types");
996 InstructionCost Cost = TTIImpl->getShuffleCost(
997 Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI);
998 assert(Cost >= 0 && "TTI should not produce negative costs!");
999 return Cost;
1000}
1001
1002TargetTransformInfo::PartialReductionExtendKind
1003TargetTransformInfo::getPartialReductionExtendKind(Instruction *I) {
1004 if (isa<SExtInst>(Val: I))
1005 return PR_SignExtend;
1006 if (isa<ZExtInst>(Val: I))
1007 return PR_ZeroExtend;
1008 return PR_None;
1009}
1010
1011TTI::CastContextHint
1012TargetTransformInfo::getCastContextHint(const Instruction *I) {
1013 if (!I)
1014 return CastContextHint::None;
1015
1016 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
1017 unsigned GatScatOp) {
1018 const Instruction *I = dyn_cast<Instruction>(Val: V);
1019 if (!I)
1020 return CastContextHint::None;
1021
1022 if (I->getOpcode() == LdStOp)
1023 return CastContextHint::Normal;
1024
1025 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1026 if (II->getIntrinsicID() == MaskedOp)
1027 return TTI::CastContextHint::Masked;
1028 if (II->getIntrinsicID() == GatScatOp)
1029 return TTI::CastContextHint::GatherScatter;
1030 }
1031
1032 return TTI::CastContextHint::None;
1033 };
1034
1035 switch (I->getOpcode()) {
1036 case Instruction::ZExt:
1037 case Instruction::SExt:
1038 case Instruction::FPExt:
1039 return getLoadStoreKind(I->getOperand(i: 0), Instruction::Load,
1040 Intrinsic::masked_load, Intrinsic::masked_gather);
1041 case Instruction::Trunc:
1042 case Instruction::FPTrunc:
1043 if (I->hasOneUse())
1044 return getLoadStoreKind(*I->user_begin(), Instruction::Store,
1045 Intrinsic::masked_store,
1046 Intrinsic::masked_scatter);
1047 break;
1048 default:
1049 return CastContextHint::None;
1050 }
1051
1052 return TTI::CastContextHint::None;
1053}
1054
1055InstructionCost TargetTransformInfo::getCastInstrCost(
1056 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,
1057 TTI::TargetCostKind CostKind, const Instruction *I) const {
1058 assert((I == nullptr || I->getOpcode() == Opcode) &&
1059 "Opcode should reflect passed instruction.");
1060 InstructionCost Cost =
1061 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1062 assert(Cost >= 0 && "TTI should not produce negative costs!");
1063 return Cost;
1064}
1065
1066InstructionCost TargetTransformInfo::getExtractWithExtendCost(
1067 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index,
1068 TTI::TargetCostKind CostKind) const {
1069 InstructionCost Cost =
1070 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index, CostKind);
1071 assert(Cost >= 0 && "TTI should not produce negative costs!");
1072 return Cost;
1073}
1074
1075InstructionCost TargetTransformInfo::getCFInstrCost(
1076 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {
1077 assert((I == nullptr || I->getOpcode() == Opcode) &&
1078 "Opcode should reflect passed instruction.");
1079 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);
1080 assert(Cost >= 0 && "TTI should not produce negative costs!");
1081 return Cost;
1082}
1083
1084InstructionCost TargetTransformInfo::getCmpSelInstrCost(
1085 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
1086 TTI::TargetCostKind CostKind, OperandValueInfo Op1Info,
1087 OperandValueInfo Op2Info, const Instruction *I) const {
1088 assert((I == nullptr || I->getOpcode() == Opcode) &&
1089 "Opcode should reflect passed instruction.");
1090 InstructionCost Cost = TTIImpl->getCmpSelInstrCost(
1091 Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
1092 assert(Cost >= 0 && "TTI should not produce negative costs!");
1093 return Cost;
1094}
1095
1096InstructionCost TargetTransformInfo::getVectorInstrCost(
1097 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1098 const Value *Op0, const Value *Op1) const {
1099 assert((Opcode == Instruction::InsertElement ||
1100 Opcode == Instruction::ExtractElement) &&
1101 "Expecting Opcode to be insertelement/extractelement.");
1102 InstructionCost Cost =
1103 TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
1104 assert(Cost >= 0 && "TTI should not produce negative costs!");
1105 return Cost;
1106}
1107
1108InstructionCost TargetTransformInfo::getVectorInstrCost(
1109 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1110 Value *Scalar,
1111 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const {
1112 assert((Opcode == Instruction::InsertElement ||
1113 Opcode == Instruction::ExtractElement) &&
1114 "Expecting Opcode to be insertelement/extractelement.");
1115 InstructionCost Cost = TTIImpl->getVectorInstrCost(
1116 Opcode, Val, CostKind, Index, Scalar, ScalarUserAndIdx);
1117 assert(Cost >= 0 && "TTI should not produce negative costs!");
1118 return Cost;
1119}
1120
1121InstructionCost
1122TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
1123 TTI::TargetCostKind CostKind,
1124 unsigned Index) const {
1125 // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1126 // This is mentioned in the interface description and respected by all
1127 // callers, but never asserted upon.
1128 InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index);
1129 assert(Cost >= 0 && "TTI should not produce negative costs!");
1130 return Cost;
1131}
1132
1133InstructionCost TargetTransformInfo::getInsertExtractValueCost(
1134 unsigned Opcode, TTI::TargetCostKind CostKind) const {
1135 assert((Opcode == Instruction::InsertValue ||
1136 Opcode == Instruction::ExtractValue) &&
1137 "Expecting Opcode to be insertvalue/extractvalue.");
1138 InstructionCost Cost = TTIImpl->getInsertExtractValueCost(Opcode, CostKind);
1139 assert(Cost >= 0 && "TTI should not produce negative costs!");
1140 return Cost;
1141}
1142
1143InstructionCost TargetTransformInfo::getReplicationShuffleCost(
1144 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
1145 TTI::TargetCostKind CostKind) const {
1146 InstructionCost Cost = TTIImpl->getReplicationShuffleCost(
1147 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind);
1148 assert(Cost >= 0 && "TTI should not produce negative costs!");
1149 return Cost;
1150}
1151
1152InstructionCost TargetTransformInfo::getMemoryOpCost(
1153 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1154 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
1155 const Instruction *I) const {
1156 assert((I == nullptr || I->getOpcode() == Opcode) &&
1157 "Opcode should reflect passed instruction.");
1158 InstructionCost Cost = TTIImpl->getMemoryOpCost(
1159 Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);
1160 assert(Cost >= 0 && "TTI should not produce negative costs!");
1161 return Cost;
1162}
1163
1164InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
1165 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1166 TTI::TargetCostKind CostKind) const {
1167 InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
1168 AddressSpace, CostKind);
1169 assert(Cost >= 0 && "TTI should not produce negative costs!");
1170 return Cost;
1171}
1172
1173InstructionCost TargetTransformInfo::getGatherScatterOpCost(
1174 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1175 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1176 InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
1177 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1178 assert((!Cost.isValid() || Cost >= 0) &&
1179 "TTI should not produce negative costs!");
1180 return Cost;
1181}
1182
1183InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
1184 unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
1185 TTI::TargetCostKind CostKind, const Instruction *I) const {
1186 InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(
1187 Opcode, DataTy, VariableMask, Alignment, CostKind, I);
1188 assert(Cost >= 0 && "TTI should not produce negative costs!");
1189 return Cost;
1190}
1191
1192InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
1193 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1194 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1195 InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
1196 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1197 assert(Cost >= 0 && "TTI should not produce negative costs!");
1198 return Cost;
1199}
1200
1201InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
1202 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1203 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1204 bool UseMaskForCond, bool UseMaskForGaps) const {
1205 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(
1206 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
1207 UseMaskForCond, UseMaskForGaps);
1208 assert(Cost >= 0 && "TTI should not produce negative costs!");
1209 return Cost;
1210}
1211
1212InstructionCost
1213TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1214 TTI::TargetCostKind CostKind) const {
1215 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);
1216 assert(Cost >= 0 && "TTI should not produce negative costs!");
1217 return Cost;
1218}
1219
1220InstructionCost
1221TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
1222 ArrayRef<Type *> Tys,
1223 TTI::TargetCostKind CostKind) const {
1224 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);
1225 assert(Cost >= 0 && "TTI should not produce negative costs!");
1226 return Cost;
1227}
1228
1229unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
1230 return TTIImpl->getNumberOfParts(Tp);
1231}
1232
1233InstructionCost
1234TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
1235 const SCEV *Ptr) const {
1236 InstructionCost Cost = TTIImpl->getAddressComputationCost(Tp, SE, Ptr);
1237 assert(Cost >= 0 && "TTI should not produce negative costs!");
1238 return Cost;
1239}
1240
1241InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {
1242 InstructionCost Cost = TTIImpl->getMemcpyCost(I);
1243 assert(Cost >= 0 && "TTI should not produce negative costs!");
1244 return Cost;
1245}
1246
1247uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const {
1248 return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold();
1249}
1250
1251InstructionCost TargetTransformInfo::getArithmeticReductionCost(
1252 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
1253 TTI::TargetCostKind CostKind) const {
1254 InstructionCost Cost =
1255 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1256 assert(Cost >= 0 && "TTI should not produce negative costs!");
1257 return Cost;
1258}
1259
1260InstructionCost TargetTransformInfo::getMinMaxReductionCost(
1261 Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
1262 TTI::TargetCostKind CostKind) const {
1263 InstructionCost Cost =
1264 TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1265 assert(Cost >= 0 && "TTI should not produce negative costs!");
1266 return Cost;
1267}
1268
1269InstructionCost TargetTransformInfo::getExtendedReductionCost(
1270 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
1271 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1272 return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,
1273 CostKind);
1274}
1275
1276InstructionCost TargetTransformInfo::getMulAccReductionCost(
1277 bool IsUnsigned, Type *ResTy, VectorType *Ty,
1278 TTI::TargetCostKind CostKind) const {
1279 return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind);
1280}
1281
1282InstructionCost
1283TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1284 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
1285}
1286
1287bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
1288 MemIntrinsicInfo &Info) const {
1289 return TTIImpl->getTgtMemIntrinsic(Inst, Info);
1290}
1291
1292unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
1293 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
1294}
1295
1296Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
1297 IntrinsicInst *Inst, Type *ExpectedType) const {
1298 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
1299}
1300
1301Type *TargetTransformInfo::getMemcpyLoopLoweringType(
1302 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1303 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
1304 std::optional<uint32_t> AtomicElementSize) const {
1305 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
1306 DestAddrSpace, SrcAlign, DestAlign,
1307 AtomicElementSize);
1308}
1309
1310void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
1311 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
1312 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1313 Align SrcAlign, Align DestAlign,
1314 std::optional<uint32_t> AtomicCpySize) const {
1315 TTIImpl->getMemcpyLoopResidualLoweringType(
1316 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
1317 DestAlign, AtomicCpySize);
1318}
1319
1320bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
1321 const Function *Callee) const {
1322 return TTIImpl->areInlineCompatible(Caller, Callee);
1323}
1324
1325unsigned
1326TargetTransformInfo::getInlineCallPenalty(const Function *F,
1327 const CallBase &Call,
1328 unsigned DefaultCallPenalty) const {
1329 return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty);
1330}
1331
1332bool TargetTransformInfo::areTypesABICompatible(
1333 const Function *Caller, const Function *Callee,
1334 const ArrayRef<Type *> &Types) const {
1335 return TTIImpl->areTypesABICompatible(Caller, Callee, Types);
1336}
1337
1338bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
1339 Type *Ty) const {
1340 return TTIImpl->isIndexedLoadLegal(Mode, Ty);
1341}
1342
1343bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
1344 Type *Ty) const {
1345 return TTIImpl->isIndexedStoreLegal(Mode, Ty);
1346}
1347
1348unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
1349 return TTIImpl->getLoadStoreVecRegBitWidth(AddrSpace: AS);
1350}
1351
1352bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
1353 return TTIImpl->isLegalToVectorizeLoad(LI);
1354}
1355
1356bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
1357 return TTIImpl->isLegalToVectorizeStore(SI);
1358}
1359
1360bool TargetTransformInfo::isLegalToVectorizeLoadChain(
1361 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1362 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
1363 AddrSpace);
1364}
1365
1366bool TargetTransformInfo::isLegalToVectorizeStoreChain(
1367 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1368 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
1369 AddrSpace);
1370}
1371
1372bool TargetTransformInfo::isLegalToVectorizeReduction(
1373 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
1374 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);
1375}
1376
1377bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {
1378 return TTIImpl->isElementTypeLegalForScalableVector(Ty);
1379}
1380
1381unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
1382 unsigned LoadSize,
1383 unsigned ChainSizeInBytes,
1384 VectorType *VecTy) const {
1385 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
1386}
1387
1388unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
1389 unsigned StoreSize,
1390 unsigned ChainSizeInBytes,
1391 VectorType *VecTy) const {
1392 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
1393}
1394
1395bool TargetTransformInfo::preferFixedOverScalableIfEqualCost() const {
1396 return TTIImpl->preferFixedOverScalableIfEqualCost();
1397}
1398
1399bool TargetTransformInfo::preferInLoopReduction(RecurKind Kind,
1400 Type *Ty) const {
1401 return TTIImpl->preferInLoopReduction(Kind, Ty);
1402}
1403
1404bool TargetTransformInfo::preferAlternateOpcodeVectorization() const {
1405 return TTIImpl->preferAlternateOpcodeVectorization();
1406}
1407
1408bool TargetTransformInfo::preferPredicatedReductionSelect() const {
1409 return TTIImpl->preferPredicatedReductionSelect();
1410}
1411
1412bool TargetTransformInfo::preferEpilogueVectorization() const {
1413 return TTIImpl->preferEpilogueVectorization();
1414}
1415
1416TargetTransformInfo::VPLegalization
1417TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
1418 return TTIImpl->getVPLegalizationStrategy(PI: VPI);
1419}
1420
1421bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const {
1422 return TTIImpl->hasArmWideBranch(Thumb);
1423}
1424
1425uint64_t TargetTransformInfo::getFeatureMask(const Function &F) const {
1426 return TTIImpl->getFeatureMask(F);
1427}
1428
1429bool TargetTransformInfo::isMultiversionedFunction(const Function &F) const {
1430 return TTIImpl->isMultiversionedFunction(F);
1431}
1432
1433unsigned TargetTransformInfo::getMaxNumArgs() const {
1434 return TTIImpl->getMaxNumArgs();
1435}
1436
1437bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
1438 return TTIImpl->shouldExpandReduction(II);
1439}
1440
1441TargetTransformInfo::ReductionShuffle
1442TargetTransformInfo::getPreferredExpandedReductionShuffle(
1443 const IntrinsicInst *II) const {
1444 return TTIImpl->getPreferredExpandedReductionShuffle(II);
1445}
1446
1447unsigned TargetTransformInfo::getGISelRematGlobalCost() const {
1448 return TTIImpl->getGISelRematGlobalCost();
1449}
1450
1451unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const {
1452 return TTIImpl->getMinTripCountTailFoldingThreshold();
1453}
1454
1455bool TargetTransformInfo::supportsScalableVectors() const {
1456 return TTIImpl->supportsScalableVectors();
1457}
1458
1459bool TargetTransformInfo::enableScalableVectorization() const {
1460 return TTIImpl->enableScalableVectorization();
1461}
1462
1463bool TargetTransformInfo::hasActiveVectorLength() const {
1464 return TTIImpl->hasActiveVectorLength();
1465}
1466
1467bool TargetTransformInfo::isProfitableToSinkOperands(
1468 Instruction *I, SmallVectorImpl<Use *> &OpsToSink) const {
1469 return TTIImpl->isProfitableToSinkOperands(I, Ops&: OpsToSink);
1470}
1471
1472bool TargetTransformInfo::isVectorShiftByScalarCheap(Type *Ty) const {
1473 return TTIImpl->isVectorShiftByScalarCheap(Ty);
1474}
1475
1476unsigned
1477TargetTransformInfo::getNumBytesToPadGlobalArray(unsigned Size,
1478 Type *ArrayType) const {
1479 return TTIImpl->getNumBytesToPadGlobalArray(Size, ArrayType);
1480}
1481
1482void TargetTransformInfo::collectKernelLaunchBounds(
1483 const Function &F,
1484 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
1485 return TTIImpl->collectKernelLaunchBounds(F, LB);
1486}
1487
1488TargetTransformInfoImplBase::~TargetTransformInfoImplBase() = default;
1489
1490TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1491
1492TargetIRAnalysis::TargetIRAnalysis(
1493 std::function<Result(const Function &)> TTICallback)
1494 : TTICallback(std::move(TTICallback)) {}
1495
1496TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1497 FunctionAnalysisManager &) {
1498 assert(!F.isIntrinsic() && "Should not request TTI for intrinsics");
1499 return TTICallback(F);
1500}
1501
1502AnalysisKey TargetIRAnalysis::Key;
1503
1504TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1505 return Result(F.getDataLayout());
1506}
1507
1508// Register the basic pass.
1509INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1510 "Target Transform Information", false, true)
1511char TargetTransformInfoWrapperPass::ID = 0;
1512
1513void TargetTransformInfoWrapperPass::anchor() {}
1514
1515TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1516 : ImmutablePass(ID) {}
1517
1518TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1519 TargetIRAnalysis TIRA)
1520 : ImmutablePass(ID), TIRA(std::move(TIRA)) {}
1521
1522TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1523 FunctionAnalysisManager DummyFAM;
1524 TTI = TIRA.run(F, DummyFAM);
1525 return *TTI;
1526}
1527
1528ImmutablePass *
1529llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1530 return new TargetTransformInfoWrapperPass(std::move(TIRA));
1531}
1532