1//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/Analysis/CFG.h"
11#include "llvm/Analysis/LoopIterator.h"
12#include "llvm/Analysis/TargetLibraryInfo.h"
13#include "llvm/Analysis/TargetTransformInfoImpl.h"
14#include "llvm/IR/CFG.h"
15#include "llvm/IR/Dominators.h"
16#include "llvm/IR/Instruction.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/IntrinsicInst.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Operator.h"
21#include "llvm/IR/PatternMatch.h"
22#include "llvm/InitializePasses.h"
23#include "llvm/Support/CommandLine.h"
24#include <optional>
25#include <utility>
26
27using namespace llvm;
28using namespace PatternMatch;
29
30#define DEBUG_TYPE "tti"
31
32static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(Val: false),
33 cl::Hidden,
34 cl::desc("Recognize reduction patterns."));
35
36static cl::opt<unsigned> CacheLineSize(
37 "cache-line-size", cl::init(Val: 0), cl::Hidden,
38 cl::desc("Use this to override the target cache line size when "
39 "specified by the user."));
40
41static cl::opt<unsigned> MinPageSize(
42 "min-page-size", cl::init(Val: 0), cl::Hidden,
43 cl::desc("Use this to override the target's minimum page size."));
44
45static cl::opt<unsigned> PredictableBranchThreshold(
46 "predictable-branch-threshold", cl::init(Val: 99), cl::Hidden,
47 cl::desc(
48 "Use this to override the target's predictable branch threshold (%)."));
49
50namespace {
51/// No-op implementation of the TTI interface using the utility base
52/// classes.
53///
54/// This is used when no target specific information is available.
55struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
56 explicit NoTTIImpl(const DataLayout &DL)
57 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
58};
59} // namespace
60
61bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {
62 // If the loop has irreducible control flow, it can not be converted to
63 // Hardware loop.
64 LoopBlocksRPO RPOT(L);
65 RPOT.perform(LI: &LI);
66 if (containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI))
67 return false;
68 return true;
69}
70
71IntrinsicCostAttributes::IntrinsicCostAttributes(
72 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost,
73 bool TypeBasedOnly)
74 : II(dyn_cast<IntrinsicInst>(Val: &CI)), RetTy(CI.getType()), IID(Id),
75 ScalarizationCost(ScalarizationCost) {
76
77 if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: &CI))
78 FMF = FPMO->getFastMathFlags();
79
80 if (!TypeBasedOnly)
81 Arguments.insert(I: Arguments.begin(), From: CI.arg_begin(), To: CI.arg_end());
82 FunctionType *FTy = CI.getCalledFunction()->getFunctionType();
83 ParamTys.insert(I: ParamTys.begin(), From: FTy->param_begin(), To: FTy->param_end());
84}
85
86IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
87 ArrayRef<Type *> Tys,
88 FastMathFlags Flags,
89 const IntrinsicInst *I,
90 InstructionCost ScalarCost)
91 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
92 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
93}
94
95IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
96 ArrayRef<const Value *> Args)
97 : RetTy(Ty), IID(Id) {
98
99 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
100 ParamTys.reserve(N: Arguments.size());
101 for (const Value *Argument : Arguments)
102 ParamTys.push_back(Elt: Argument->getType());
103}
104
105IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
106 ArrayRef<const Value *> Args,
107 ArrayRef<Type *> Tys,
108 FastMathFlags Flags,
109 const IntrinsicInst *I,
110 InstructionCost ScalarCost)
111 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
112 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
113 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
114}
115
116HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) {
117 // Match default options:
118 // - hardware-loop-counter-bitwidth = 32
119 // - hardware-loop-decrement = 1
120 CountType = Type::getInt32Ty(C&: L->getHeader()->getContext());
121 LoopDecrement = ConstantInt::get(Ty: CountType, V: 1);
122}
123
124bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
125 LoopInfo &LI, DominatorTree &DT,
126 bool ForceNestedLoop,
127 bool ForceHardwareLoopPHI) {
128 SmallVector<BasicBlock *, 4> ExitingBlocks;
129 L->getExitingBlocks(ExitingBlocks);
130
131 for (BasicBlock *BB : ExitingBlocks) {
132 // If we pass the updated counter back through a phi, we need to know
133 // which latch the updated value will be coming from.
134 if (!L->isLoopLatch(BB)) {
135 if (ForceHardwareLoopPHI || CounterInReg)
136 continue;
137 }
138
139 const SCEV *EC = SE.getExitCount(L, ExitingBlock: BB);
140 if (isa<SCEVCouldNotCompute>(Val: EC))
141 continue;
142 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(Val: EC)) {
143 if (ConstEC->getValue()->isZero())
144 continue;
145 } else if (!SE.isLoopInvariant(S: EC, L))
146 continue;
147
148 if (SE.getTypeSizeInBits(Ty: EC->getType()) > CountType->getBitWidth())
149 continue;
150
151 // If this exiting block is contained in a nested loop, it is not eligible
152 // for insertion of the branch-and-decrement since the inner loop would
153 // end up messing up the value in the CTR.
154 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
155 continue;
156
157 // We now have a loop-invariant count of loop iterations (which is not the
158 // constant zero) for which we know that this loop will not exit via this
159 // existing block.
160
161 // We need to make sure that this block will run on every loop iteration.
162 // For this to be true, we must dominate all blocks with backedges. Such
163 // blocks are in-loop predecessors to the header block.
164 bool NotAlways = false;
165 for (BasicBlock *Pred : predecessors(BB: L->getHeader())) {
166 if (!L->contains(BB: Pred))
167 continue;
168
169 if (!DT.dominates(A: BB, B: Pred)) {
170 NotAlways = true;
171 break;
172 }
173 }
174
175 if (NotAlways)
176 continue;
177
178 // Make sure this blocks ends with a conditional branch.
179 Instruction *TI = BB->getTerminator();
180 if (!TI)
181 continue;
182
183 if (BranchInst *BI = dyn_cast<BranchInst>(Val: TI)) {
184 if (!BI->isConditional())
185 continue;
186
187 ExitBranch = BI;
188 } else
189 continue;
190
191 // Note that this block may not be the loop latch block, even if the loop
192 // has a latch block.
193 ExitBlock = BB;
194 ExitCount = EC;
195 break;
196 }
197
198 if (!ExitBlock)
199 return false;
200 return true;
201}
202
203TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
204 : TTIImpl(new Model<NoTTIImpl>(NoTTIImpl(DL))) {}
205
206TargetTransformInfo::~TargetTransformInfo() = default;
207
208TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
209 : TTIImpl(std::move(Arg.TTIImpl)) {}
210
211TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
212 TTIImpl = std::move(RHS.TTIImpl);
213 return *this;
214}
215
216unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
217 return TTIImpl->getInliningThresholdMultiplier();
218}
219
220unsigned
221TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const {
222 return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier();
223}
224
225unsigned
226TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
227 const {
228 return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
229}
230
231unsigned
232TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
233 return TTIImpl->adjustInliningThreshold(CB);
234}
235
236unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,
237 const AllocaInst *AI) const {
238 return TTIImpl->getCallerAllocaCost(CB, AI);
239}
240
241int TargetTransformInfo::getInlinerVectorBonusPercent() const {
242 return TTIImpl->getInlinerVectorBonusPercent();
243}
244
245InstructionCost TargetTransformInfo::getGEPCost(
246 Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands,
247 Type *AccessType, TTI::TargetCostKind CostKind) const {
248 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
249}
250
251InstructionCost TargetTransformInfo::getPointersChainCost(
252 ArrayRef<const Value *> Ptrs, const Value *Base,
253 const TTI::PointersChainInfo &Info, Type *AccessTy,
254 TTI::TargetCostKind CostKind) const {
255 assert((Base || !Info.isSameBase()) &&
256 "If pointers have same base address it has to be provided.");
257 return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
258}
259
260unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(
261 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,
262 BlockFrequencyInfo *BFI) const {
263 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
264}
265
266InstructionCost
267TargetTransformInfo::getInstructionCost(const User *U,
268 ArrayRef<const Value *> Operands,
269 enum TargetCostKind CostKind) const {
270 InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind);
271 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&
272 "TTI should not produce negative costs!");
273 return Cost;
274}
275
276BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
277 return PredictableBranchThreshold.getNumOccurrences() > 0
278 ? BranchProbability(PredictableBranchThreshold, 100)
279 : TTIImpl->getPredictableBranchThreshold();
280}
281
282InstructionCost TargetTransformInfo::getBranchMispredictPenalty() const {
283 return TTIImpl->getBranchMispredictPenalty();
284}
285
286bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {
287 return TTIImpl->hasBranchDivergence(F);
288}
289
290bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const {
291 return TTIImpl->isSourceOfDivergence(V);
292}
293
294bool llvm::TargetTransformInfo::isAlwaysUniform(const Value *V) const {
295 return TTIImpl->isAlwaysUniform(V);
296}
297
298bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS,
299 unsigned ToAS) const {
300 return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS);
301}
302
303bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS,
304 unsigned ToAS) const {
305 return TTIImpl->addrspacesMayAlias(AS0: FromAS, AS1: ToAS);
306}
307
308unsigned TargetTransformInfo::getFlatAddressSpace() const {
309 return TTIImpl->getFlatAddressSpace();
310}
311
312bool TargetTransformInfo::collectFlatAddressOperands(
313 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {
314 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);
315}
316
317bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,
318 unsigned ToAS) const {
319 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);
320}
321
322bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace(
323 unsigned AS) const {
324 return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS);
325}
326
327unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {
328 return TTIImpl->getAssumedAddrSpace(V);
329}
330
331bool TargetTransformInfo::isSingleThreaded() const {
332 return TTIImpl->isSingleThreaded();
333}
334
335std::pair<const Value *, unsigned>
336TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const {
337 return TTIImpl->getPredicatedAddrSpace(V);
338}
339
340Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(
341 IntrinsicInst *II, Value *OldV, Value *NewV) const {
342 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
343}
344
345bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
346 return TTIImpl->isLoweredToCall(F);
347}
348
349bool TargetTransformInfo::isHardwareLoopProfitable(
350 Loop *L, ScalarEvolution &SE, AssumptionCache &AC,
351 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {
352 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
353}
354
355bool TargetTransformInfo::preferPredicateOverEpilogue(
356 TailFoldingInfo *TFI) const {
357 return TTIImpl->preferPredicateOverEpilogue(TFI);
358}
359
360TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(
361 bool IVUpdateMayOverflow) const {
362 return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);
363}
364
365std::optional<Instruction *>
366TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
367 IntrinsicInst &II) const {
368 return TTIImpl->instCombineIntrinsic(IC, II);
369}
370
371std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
372 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
373 bool &KnownBitsComputed) const {
374 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
375 KnownBitsComputed);
376}
377
378std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
379 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
380 APInt &UndefElts2, APInt &UndefElts3,
381 std::function<void(Instruction *, unsigned, APInt, APInt &)>
382 SimplifyAndSetOp) const {
383 return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
384 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
385 SimplifyAndSetOp);
386}
387
388void TargetTransformInfo::getUnrollingPreferences(
389 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP,
390 OptimizationRemarkEmitter *ORE) const {
391 return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE);
392}
393
394void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
395 PeelingPreferences &PP) const {
396 return TTIImpl->getPeelingPreferences(L, SE, PP);
397}
398
399bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
400 return TTIImpl->isLegalAddImmediate(Imm);
401}
402
403bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const {
404 return TTIImpl->isLegalAddScalableImmediate(Imm);
405}
406
407bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
408 return TTIImpl->isLegalICmpImmediate(Imm);
409}
410
411bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
412 int64_t BaseOffset,
413 bool HasBaseReg, int64_t Scale,
414 unsigned AddrSpace,
415 Instruction *I,
416 int64_t ScalableOffset) const {
417 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
418 Scale, AddrSpace, I, ScalableOffset);
419}
420
421bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1,
422 const LSRCost &C2) const {
423 return TTIImpl->isLSRCostLess(C1, C2);
424}
425
426bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
427 return TTIImpl->isNumRegsMajorCostOfLSR();
428}
429
430bool TargetTransformInfo::shouldFoldTerminatingConditionAfterLSR() const {
431 return TTIImpl->shouldFoldTerminatingConditionAfterLSR();
432}
433
434bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
435 return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
436}
437
438bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {
439 return TTIImpl->isProfitableLSRChainElement(I);
440}
441
442bool TargetTransformInfo::canMacroFuseCmp() const {
443 return TTIImpl->canMacroFuseCmp();
444}
445
446bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,
447 ScalarEvolution *SE, LoopInfo *LI,
448 DominatorTree *DT, AssumptionCache *AC,
449 TargetLibraryInfo *LibInfo) const {
450 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
451}
452
453TTI::AddressingModeKind
454TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
455 ScalarEvolution *SE) const {
456 return TTIImpl->getPreferredAddressingMode(L, SE);
457}
458
459bool TargetTransformInfo::isLegalMaskedStore(Type *DataType,
460 Align Alignment) const {
461 return TTIImpl->isLegalMaskedStore(DataType, Alignment);
462}
463
464bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType,
465 Align Alignment) const {
466 return TTIImpl->isLegalMaskedLoad(DataType, Alignment);
467}
468
469bool TargetTransformInfo::isLegalNTStore(Type *DataType,
470 Align Alignment) const {
471 return TTIImpl->isLegalNTStore(DataType, Alignment);
472}
473
474bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
475 return TTIImpl->isLegalNTLoad(DataType, Alignment);
476}
477
478bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,
479 ElementCount NumElements) const {
480 return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);
481}
482
483bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
484 Align Alignment) const {
485 return TTIImpl->isLegalMaskedGather(DataType, Alignment);
486}
487
488bool TargetTransformInfo::isLegalAltInstr(
489 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
490 const SmallBitVector &OpcodeMask) const {
491 return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
492}
493
494bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
495 Align Alignment) const {
496 return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
497}
498
499bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,
500 Align Alignment) const {
501 return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);
502}
503
504bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType,
505 Align Alignment) const {
506 return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment);
507}
508
509bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType,
510 Align Alignment) const {
511 return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment);
512}
513
514bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType,
515 Align Alignment) const {
516 return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment);
517}
518
519bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
520 Align Alignment) const {
521 return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
522}
523
524bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
525 Type *DataType) const {
526 return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
527}
528
529bool TargetTransformInfo::enableOrderedReductions() const {
530 return TTIImpl->enableOrderedReductions();
531}
532
533bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
534 return TTIImpl->hasDivRemOp(DataType, IsSigned);
535}
536
537bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
538 unsigned AddrSpace) const {
539 return TTIImpl->hasVolatileVariant(I, AddrSpace);
540}
541
542bool TargetTransformInfo::prefersVectorizedAddressing() const {
543 return TTIImpl->prefersVectorizedAddressing();
544}
545
546InstructionCost TargetTransformInfo::getScalingFactorCost(
547 Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,
548 int64_t Scale, unsigned AddrSpace) const {
549 InstructionCost Cost = TTIImpl->getScalingFactorCost(
550 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
551 assert(Cost >= 0 && "TTI should not produce negative costs!");
552 return Cost;
553}
554
555bool TargetTransformInfo::LSRWithInstrQueries() const {
556 return TTIImpl->LSRWithInstrQueries();
557}
558
559bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
560 return TTIImpl->isTruncateFree(Ty1, Ty2);
561}
562
563bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
564 return TTIImpl->isProfitableToHoist(I);
565}
566
567bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
568
569bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
570 return TTIImpl->isTypeLegal(Ty);
571}
572
573unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {
574 return TTIImpl->getRegUsageForType(Ty);
575}
576
577bool TargetTransformInfo::shouldBuildLookupTables() const {
578 return TTIImpl->shouldBuildLookupTables();
579}
580
581bool TargetTransformInfo::shouldBuildLookupTablesForConstant(
582 Constant *C) const {
583 return TTIImpl->shouldBuildLookupTablesForConstant(C);
584}
585
586bool TargetTransformInfo::shouldBuildRelLookupTables() const {
587 return TTIImpl->shouldBuildRelLookupTables();
588}
589
590bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
591 return TTIImpl->useColdCCForColdCall(F);
592}
593
594InstructionCost TargetTransformInfo::getScalarizationOverhead(
595 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
596 TTI::TargetCostKind CostKind) const {
597 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
598 CostKind);
599}
600
601InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
602 ArrayRef<const Value *> Args, ArrayRef<Type *> Tys,
603 TTI::TargetCostKind CostKind) const {
604 return TTIImpl->getOperandsScalarizationOverhead(Args, Tys, CostKind);
605}
606
607bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
608 return TTIImpl->supportsEfficientVectorElementLoadStore();
609}
610
611bool TargetTransformInfo::supportsTailCalls() const {
612 return TTIImpl->supportsTailCalls();
613}
614
615bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const {
616 return TTIImpl->supportsTailCallFor(CB);
617}
618
619bool TargetTransformInfo::enableAggressiveInterleaving(
620 bool LoopHasReductions) const {
621 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
622}
623
624TargetTransformInfo::MemCmpExpansionOptions
625TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
626 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);
627}
628
629bool TargetTransformInfo::enableSelectOptimize() const {
630 return TTIImpl->enableSelectOptimize();
631}
632
633bool TargetTransformInfo::shouldTreatInstructionLikeSelect(
634 const Instruction *I) const {
635 return TTIImpl->shouldTreatInstructionLikeSelect(I);
636}
637
638bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
639 return TTIImpl->enableInterleavedAccessVectorization();
640}
641
642bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
643 return TTIImpl->enableMaskedInterleavedAccessVectorization();
644}
645
646bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
647 return TTIImpl->isFPVectorizationPotentiallyUnsafe();
648}
649
650bool
651TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
652 unsigned BitWidth,
653 unsigned AddressSpace,
654 Align Alignment,
655 unsigned *Fast) const {
656 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,
657 AddressSpace, Alignment, Fast);
658}
659
660TargetTransformInfo::PopcntSupportKind
661TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
662 return TTIImpl->getPopcntSupport(IntTyWidthInBit);
663}
664
665bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
666 return TTIImpl->haveFastSqrt(Ty);
667}
668
669bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
670 const Instruction *I) const {
671 return TTIImpl->isExpensiveToSpeculativelyExecute(I);
672}
673
674bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
675 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
676}
677
678InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {
679 InstructionCost Cost = TTIImpl->getFPOpCost(Ty);
680 assert(Cost >= 0 && "TTI should not produce negative costs!");
681 return Cost;
682}
683
684InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,
685 unsigned Idx,
686 const APInt &Imm,
687 Type *Ty) const {
688 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opc: Opcode, Idx, Imm, Ty);
689 assert(Cost >= 0 && "TTI should not produce negative costs!");
690 return Cost;
691}
692
693InstructionCost
694TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,
695 TTI::TargetCostKind CostKind) const {
696 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);
697 assert(Cost >= 0 && "TTI should not produce negative costs!");
698 return Cost;
699}
700
701InstructionCost TargetTransformInfo::getIntImmCostInst(
702 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,
703 TTI::TargetCostKind CostKind, Instruction *Inst) const {
704 InstructionCost Cost =
705 TTIImpl->getIntImmCostInst(Opc: Opcode, Idx, Imm, Ty, CostKind, Inst);
706 assert(Cost >= 0 && "TTI should not produce negative costs!");
707 return Cost;
708}
709
710InstructionCost
711TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
712 const APInt &Imm, Type *Ty,
713 TTI::TargetCostKind CostKind) const {
714 InstructionCost Cost =
715 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
716 assert(Cost >= 0 && "TTI should not produce negative costs!");
717 return Cost;
718}
719
720bool TargetTransformInfo::preferToKeepConstantsAttached(
721 const Instruction &Inst, const Function &Fn) const {
722 return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);
723}
724
725unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
726 return TTIImpl->getNumberOfRegisters(ClassID);
727}
728
729bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty) const {
730 return TTIImpl->hasConditionalLoadStoreForType(Ty);
731}
732
733unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,
734 Type *Ty) const {
735 return TTIImpl->getRegisterClassForType(Vector, Ty);
736}
737
738const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
739 return TTIImpl->getRegisterClassName(ClassID);
740}
741
742TypeSize TargetTransformInfo::getRegisterBitWidth(
743 TargetTransformInfo::RegisterKind K) const {
744 return TTIImpl->getRegisterBitWidth(K);
745}
746
747unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
748 return TTIImpl->getMinVectorRegisterBitWidth();
749}
750
751std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {
752 return TTIImpl->getMaxVScale();
753}
754
755std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {
756 return TTIImpl->getVScaleForTuning();
757}
758
759bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const {
760 return TTIImpl->isVScaleKnownToBeAPowerOfTwo();
761}
762
763bool TargetTransformInfo::shouldMaximizeVectorBandwidth(
764 TargetTransformInfo::RegisterKind K) const {
765 return TTIImpl->shouldMaximizeVectorBandwidth(K);
766}
767
768ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,
769 bool IsScalable) const {
770 return TTIImpl->getMinimumVF(ElemWidth, IsScalable);
771}
772
773unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,
774 unsigned Opcode) const {
775 return TTIImpl->getMaximumVF(ElemWidth, Opcode);
776}
777
778unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
779 Type *ScalarValTy) const {
780 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
781}
782
783bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
784 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
785 return TTIImpl->shouldConsiderAddressTypePromotion(
786 I, AllowPromotionWithoutCommonHeader);
787}
788
789unsigned TargetTransformInfo::getCacheLineSize() const {
790 return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize
791 : TTIImpl->getCacheLineSize();
792}
793
794std::optional<unsigned>
795TargetTransformInfo::getCacheSize(CacheLevel Level) const {
796 return TTIImpl->getCacheSize(Level);
797}
798
799std::optional<unsigned>
800TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {
801 return TTIImpl->getCacheAssociativity(Level);
802}
803
804std::optional<unsigned> TargetTransformInfo::getMinPageSize() const {
805 return MinPageSize.getNumOccurrences() > 0 ? MinPageSize
806 : TTIImpl->getMinPageSize();
807}
808
809unsigned TargetTransformInfo::getPrefetchDistance() const {
810 return TTIImpl->getPrefetchDistance();
811}
812
813unsigned TargetTransformInfo::getMinPrefetchStride(
814 unsigned NumMemAccesses, unsigned NumStridedMemAccesses,
815 unsigned NumPrefetches, bool HasCall) const {
816 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
817 NumPrefetches, HasCall);
818}
819
820unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
821 return TTIImpl->getMaxPrefetchIterationsAhead();
822}
823
824bool TargetTransformInfo::enableWritePrefetching() const {
825 return TTIImpl->enableWritePrefetching();
826}
827
828bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
829 return TTIImpl->shouldPrefetchAddressSpace(AS);
830}
831
832unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
833 return TTIImpl->getMaxInterleaveFactor(VF);
834}
835
836TargetTransformInfo::OperandValueInfo
837TargetTransformInfo::getOperandInfo(const Value *V) {
838 OperandValueKind OpInfo = OK_AnyValue;
839 OperandValueProperties OpProps = OP_None;
840
841 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V)) {
842 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
843 if (CI->getValue().isPowerOf2())
844 OpProps = OP_PowerOf2;
845 else if (CI->getValue().isNegatedPowerOf2())
846 OpProps = OP_NegatedPowerOf2;
847 }
848 return {.Kind: OK_UniformConstantValue, .Properties: OpProps};
849 }
850
851 // A broadcast shuffle creates a uniform value.
852 // TODO: Add support for non-zero index broadcasts.
853 // TODO: Add support for different source vector width.
854 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(Val: V))
855 if (ShuffleInst->isZeroEltSplat())
856 OpInfo = OK_UniformValue;
857
858 const Value *Splat = getSplatValue(V);
859
860 // Check for a splat of a constant or for a non uniform vector of constants
861 // and check if the constant(s) are all powers of two.
862 if (isa<ConstantVector>(Val: V) || isa<ConstantDataVector>(Val: V)) {
863 OpInfo = OK_NonUniformConstantValue;
864 if (Splat) {
865 OpInfo = OK_UniformConstantValue;
866 if (auto *CI = dyn_cast<ConstantInt>(Val: Splat)) {
867 if (CI->getValue().isPowerOf2())
868 OpProps = OP_PowerOf2;
869 else if (CI->getValue().isNegatedPowerOf2())
870 OpProps = OP_NegatedPowerOf2;
871 }
872 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
873 bool AllPow2 = true, AllNegPow2 = true;
874 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
875 if (auto *CI = dyn_cast<ConstantInt>(Val: CDS->getElementAsConstant(i: I))) {
876 AllPow2 &= CI->getValue().isPowerOf2();
877 AllNegPow2 &= CI->getValue().isNegatedPowerOf2();
878 if (AllPow2 || AllNegPow2)
879 continue;
880 }
881 AllPow2 = AllNegPow2 = false;
882 break;
883 }
884 OpProps = AllPow2 ? OP_PowerOf2 : OpProps;
885 OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps;
886 }
887 }
888
889 // Check for a splat of a uniform value. This is not loop aware, so return
890 // true only for the obviously uniform cases (argument, globalvalue)
891 if (Splat && (isa<Argument>(Val: Splat) || isa<GlobalValue>(Val: Splat)))
892 OpInfo = OK_UniformValue;
893
894 return {.Kind: OpInfo, .Properties: OpProps};
895}
896
897InstructionCost TargetTransformInfo::getArithmeticInstrCost(
898 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
899 OperandValueInfo Op1Info, OperandValueInfo Op2Info,
900 ArrayRef<const Value *> Args, const Instruction *CxtI,
901 const TargetLibraryInfo *TLibInfo) const {
902
903 // Use call cost for frem intructions that have platform specific vector math
904 // functions, as those will be replaced with calls later by SelectionDAG or
905 // ReplaceWithVecLib pass.
906 if (TLibInfo && Opcode == Instruction::FRem) {
907 VectorType *VecTy = dyn_cast<VectorType>(Val: Ty);
908 LibFunc Func;
909 if (VecTy &&
910 TLibInfo->getLibFunc(Opcode: Instruction::FRem, Ty: Ty->getScalarType(), F&: Func) &&
911 TLibInfo->isFunctionVectorizable(F: TLibInfo->getName(F: Func),
912 VF: VecTy->getElementCount()))
913 return getCallInstrCost(F: nullptr, RetTy: VecTy, Tys: {VecTy, VecTy}, CostKind);
914 }
915
916 InstructionCost Cost =
917 TTIImpl->getArithmeticInstrCost(Opcode, Ty, CostKind,
918 Opd1Info: Op1Info, Opd2Info: Op2Info,
919 Args, CxtI);
920 assert(Cost >= 0 && "TTI should not produce negative costs!");
921 return Cost;
922}
923
924InstructionCost TargetTransformInfo::getAltInstrCost(
925 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
926 const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
927 InstructionCost Cost =
928 TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
929 assert(Cost >= 0 && "TTI should not produce negative costs!");
930 return Cost;
931}
932
933InstructionCost TargetTransformInfo::getShuffleCost(
934 ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask,
935 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
936 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
937 InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Tp: Ty, Mask, CostKind,
938 Index, SubTp, Args, CxtI);
939 assert(Cost >= 0 && "TTI should not produce negative costs!");
940 return Cost;
941}
942
943TTI::CastContextHint
944TargetTransformInfo::getCastContextHint(const Instruction *I) {
945 if (!I)
946 return CastContextHint::None;
947
948 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
949 unsigned GatScatOp) {
950 const Instruction *I = dyn_cast<Instruction>(Val: V);
951 if (!I)
952 return CastContextHint::None;
953
954 if (I->getOpcode() == LdStOp)
955 return CastContextHint::Normal;
956
957 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
958 if (II->getIntrinsicID() == MaskedOp)
959 return TTI::CastContextHint::Masked;
960 if (II->getIntrinsicID() == GatScatOp)
961 return TTI::CastContextHint::GatherScatter;
962 }
963
964 return TTI::CastContextHint::None;
965 };
966
967 switch (I->getOpcode()) {
968 case Instruction::ZExt:
969 case Instruction::SExt:
970 case Instruction::FPExt:
971 return getLoadStoreKind(I->getOperand(i: 0), Instruction::Load,
972 Intrinsic::masked_load, Intrinsic::masked_gather);
973 case Instruction::Trunc:
974 case Instruction::FPTrunc:
975 if (I->hasOneUse())
976 return getLoadStoreKind(*I->user_begin(), Instruction::Store,
977 Intrinsic::masked_store,
978 Intrinsic::masked_scatter);
979 break;
980 default:
981 return CastContextHint::None;
982 }
983
984 return TTI::CastContextHint::None;
985}
986
987InstructionCost TargetTransformInfo::getCastInstrCost(
988 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,
989 TTI::TargetCostKind CostKind, const Instruction *I) const {
990 assert((I == nullptr || I->getOpcode() == Opcode) &&
991 "Opcode should reflect passed instruction.");
992 InstructionCost Cost =
993 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
994 assert(Cost >= 0 && "TTI should not produce negative costs!");
995 return Cost;
996}
997
998InstructionCost TargetTransformInfo::getExtractWithExtendCost(
999 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index) const {
1000 InstructionCost Cost =
1001 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index);
1002 assert(Cost >= 0 && "TTI should not produce negative costs!");
1003 return Cost;
1004}
1005
1006InstructionCost TargetTransformInfo::getCFInstrCost(
1007 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {
1008 assert((I == nullptr || I->getOpcode() == Opcode) &&
1009 "Opcode should reflect passed instruction.");
1010 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);
1011 assert(Cost >= 0 && "TTI should not produce negative costs!");
1012 return Cost;
1013}
1014
1015InstructionCost TargetTransformInfo::getCmpSelInstrCost(
1016 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
1017 TTI::TargetCostKind CostKind, const Instruction *I) const {
1018 assert((I == nullptr || I->getOpcode() == Opcode) &&
1019 "Opcode should reflect passed instruction.");
1020 InstructionCost Cost =
1021 TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
1022 assert(Cost >= 0 && "TTI should not produce negative costs!");
1023 return Cost;
1024}
1025
1026InstructionCost TargetTransformInfo::getVectorInstrCost(
1027 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1028 Value *Op0, Value *Op1) const {
1029 // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1030 // This is mentioned in the interface description and respected by all
1031 // callers, but never asserted upon.
1032 InstructionCost Cost =
1033 TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1);
1034 assert(Cost >= 0 && "TTI should not produce negative costs!");
1035 return Cost;
1036}
1037
1038InstructionCost
1039TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
1040 TTI::TargetCostKind CostKind,
1041 unsigned Index) const {
1042 // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1043 // This is mentioned in the interface description and respected by all
1044 // callers, but never asserted upon.
1045 InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, CostKind, Index);
1046 assert(Cost >= 0 && "TTI should not produce negative costs!");
1047 return Cost;
1048}
1049
1050InstructionCost TargetTransformInfo::getReplicationShuffleCost(
1051 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
1052 TTI::TargetCostKind CostKind) const {
1053 InstructionCost Cost = TTIImpl->getReplicationShuffleCost(
1054 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind);
1055 assert(Cost >= 0 && "TTI should not produce negative costs!");
1056 return Cost;
1057}
1058
1059InstructionCost TargetTransformInfo::getMemoryOpCost(
1060 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1061 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
1062 const Instruction *I) const {
1063 assert((I == nullptr || I->getOpcode() == Opcode) &&
1064 "Opcode should reflect passed instruction.");
1065 InstructionCost Cost = TTIImpl->getMemoryOpCost(
1066 Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);
1067 assert(Cost >= 0 && "TTI should not produce negative costs!");
1068 return Cost;
1069}
1070
1071InstructionCost TargetTransformInfo::getMaskedMemoryOpCost(
1072 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1073 TTI::TargetCostKind CostKind) const {
1074 InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment,
1075 AddressSpace, CostKind);
1076 assert(Cost >= 0 && "TTI should not produce negative costs!");
1077 return Cost;
1078}
1079
1080InstructionCost TargetTransformInfo::getGatherScatterOpCost(
1081 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1082 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1083 InstructionCost Cost = TTIImpl->getGatherScatterOpCost(
1084 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1085 assert((!Cost.isValid() || Cost >= 0) &&
1086 "TTI should not produce negative costs!");
1087 return Cost;
1088}
1089
1090InstructionCost TargetTransformInfo::getStridedMemoryOpCost(
1091 unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
1092 Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const {
1093 InstructionCost Cost = TTIImpl->getStridedMemoryOpCost(
1094 Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I);
1095 assert(Cost >= 0 && "TTI should not produce negative costs!");
1096 return Cost;
1097}
1098
1099InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
1100 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1101 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1102 bool UseMaskForCond, bool UseMaskForGaps) const {
1103 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(
1104 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
1105 UseMaskForCond, UseMaskForGaps);
1106 assert(Cost >= 0 && "TTI should not produce negative costs!");
1107 return Cost;
1108}
1109
1110InstructionCost
1111TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1112 TTI::TargetCostKind CostKind) const {
1113 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);
1114 assert(Cost >= 0 && "TTI should not produce negative costs!");
1115 return Cost;
1116}
1117
1118InstructionCost
1119TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
1120 ArrayRef<Type *> Tys,
1121 TTI::TargetCostKind CostKind) const {
1122 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);
1123 assert(Cost >= 0 && "TTI should not produce negative costs!");
1124 return Cost;
1125}
1126
1127unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
1128 return TTIImpl->getNumberOfParts(Tp);
1129}
1130
1131InstructionCost
1132TargetTransformInfo::getAddressComputationCost(Type *Tp, ScalarEvolution *SE,
1133 const SCEV *Ptr) const {
1134 InstructionCost Cost = TTIImpl->getAddressComputationCost(Ty: Tp, SE, Ptr);
1135 assert(Cost >= 0 && "TTI should not produce negative costs!");
1136 return Cost;
1137}
1138
1139InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {
1140 InstructionCost Cost = TTIImpl->getMemcpyCost(I);
1141 assert(Cost >= 0 && "TTI should not produce negative costs!");
1142 return Cost;
1143}
1144
1145uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const {
1146 return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold();
1147}
1148
1149InstructionCost TargetTransformInfo::getArithmeticReductionCost(
1150 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
1151 TTI::TargetCostKind CostKind) const {
1152 InstructionCost Cost =
1153 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1154 assert(Cost >= 0 && "TTI should not produce negative costs!");
1155 return Cost;
1156}
1157
1158InstructionCost TargetTransformInfo::getMinMaxReductionCost(
1159 Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
1160 TTI::TargetCostKind CostKind) const {
1161 InstructionCost Cost =
1162 TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1163 assert(Cost >= 0 && "TTI should not produce negative costs!");
1164 return Cost;
1165}
1166
1167InstructionCost TargetTransformInfo::getExtendedReductionCost(
1168 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
1169 FastMathFlags FMF, TTI::TargetCostKind CostKind) const {
1170 return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,
1171 CostKind);
1172}
1173
1174InstructionCost TargetTransformInfo::getMulAccReductionCost(
1175 bool IsUnsigned, Type *ResTy, VectorType *Ty,
1176 TTI::TargetCostKind CostKind) const {
1177 return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind);
1178}
1179
1180InstructionCost
1181TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1182 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
1183}
1184
1185bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
1186 MemIntrinsicInfo &Info) const {
1187 return TTIImpl->getTgtMemIntrinsic(Inst, Info);
1188}
1189
1190unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
1191 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
1192}
1193
1194Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
1195 IntrinsicInst *Inst, Type *ExpectedType) const {
1196 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType);
1197}
1198
1199Type *TargetTransformInfo::getMemcpyLoopLoweringType(
1200 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1201 unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign,
1202 std::optional<uint32_t> AtomicElementSize) const {
1203 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
1204 DestAddrSpace, SrcAlign, DestAlign,
1205 AtomicElementSize);
1206}
1207
1208void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
1209 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
1210 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1211 unsigned SrcAlign, unsigned DestAlign,
1212 std::optional<uint32_t> AtomicCpySize) const {
1213 TTIImpl->getMemcpyLoopResidualLoweringType(
1214 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
1215 DestAlign, AtomicCpySize);
1216}
1217
1218bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
1219 const Function *Callee) const {
1220 return TTIImpl->areInlineCompatible(Caller, Callee);
1221}
1222
1223unsigned
1224TargetTransformInfo::getInlineCallPenalty(const Function *F,
1225 const CallBase &Call,
1226 unsigned DefaultCallPenalty) const {
1227 return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty);
1228}
1229
1230bool TargetTransformInfo::areTypesABICompatible(
1231 const Function *Caller, const Function *Callee,
1232 const ArrayRef<Type *> &Types) const {
1233 return TTIImpl->areTypesABICompatible(Caller, Callee, Types);
1234}
1235
1236bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
1237 Type *Ty) const {
1238 return TTIImpl->isIndexedLoadLegal(Mode, Ty);
1239}
1240
1241bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
1242 Type *Ty) const {
1243 return TTIImpl->isIndexedStoreLegal(Mode, Ty);
1244}
1245
1246unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
1247 return TTIImpl->getLoadStoreVecRegBitWidth(AddrSpace: AS);
1248}
1249
1250bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
1251 return TTIImpl->isLegalToVectorizeLoad(LI);
1252}
1253
1254bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
1255 return TTIImpl->isLegalToVectorizeStore(SI);
1256}
1257
1258bool TargetTransformInfo::isLegalToVectorizeLoadChain(
1259 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1260 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
1261 AddrSpace);
1262}
1263
1264bool TargetTransformInfo::isLegalToVectorizeStoreChain(
1265 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1266 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
1267 AddrSpace);
1268}
1269
1270bool TargetTransformInfo::isLegalToVectorizeReduction(
1271 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
1272 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);
1273}
1274
1275bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {
1276 return TTIImpl->isElementTypeLegalForScalableVector(Ty);
1277}
1278
1279unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
1280 unsigned LoadSize,
1281 unsigned ChainSizeInBytes,
1282 VectorType *VecTy) const {
1283 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
1284}
1285
1286unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
1287 unsigned StoreSize,
1288 unsigned ChainSizeInBytes,
1289 VectorType *VecTy) const {
1290 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
1291}
1292
1293bool TargetTransformInfo::preferFixedOverScalableIfEqualCost() const {
1294 return TTIImpl->preferFixedOverScalableIfEqualCost();
1295}
1296
1297bool TargetTransformInfo::preferInLoopReduction(unsigned Opcode, Type *Ty,
1298 ReductionFlags Flags) const {
1299 return TTIImpl->preferInLoopReduction(Opcode, Ty, Flags);
1300}
1301
1302bool TargetTransformInfo::preferPredicatedReductionSelect(
1303 unsigned Opcode, Type *Ty, ReductionFlags Flags) const {
1304 return TTIImpl->preferPredicatedReductionSelect(Opcode, Ty, Flags);
1305}
1306
1307bool TargetTransformInfo::preferEpilogueVectorization() const {
1308 return TTIImpl->preferEpilogueVectorization();
1309}
1310
1311TargetTransformInfo::VPLegalization
1312TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
1313 return TTIImpl->getVPLegalizationStrategy(PI: VPI);
1314}
1315
1316bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const {
1317 return TTIImpl->hasArmWideBranch(Thumb);
1318}
1319
1320unsigned TargetTransformInfo::getMaxNumArgs() const {
1321 return TTIImpl->getMaxNumArgs();
1322}
1323
1324bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
1325 return TTIImpl->shouldExpandReduction(II);
1326}
1327
1328TargetTransformInfo::ReductionShuffle
1329TargetTransformInfo::getPreferredExpandedReductionShuffle(
1330 const IntrinsicInst *II) const {
1331 return TTIImpl->getPreferredExpandedReductionShuffle(II);
1332}
1333
1334unsigned TargetTransformInfo::getGISelRematGlobalCost() const {
1335 return TTIImpl->getGISelRematGlobalCost();
1336}
1337
1338unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const {
1339 return TTIImpl->getMinTripCountTailFoldingThreshold();
1340}
1341
1342bool TargetTransformInfo::supportsScalableVectors() const {
1343 return TTIImpl->supportsScalableVectors();
1344}
1345
1346bool TargetTransformInfo::enableScalableVectorization() const {
1347 return TTIImpl->enableScalableVectorization();
1348}
1349
1350bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
1351 Align Alignment) const {
1352 return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
1353}
1354
1355TargetTransformInfo::Concept::~Concept() = default;
1356
1357TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1358
1359TargetIRAnalysis::TargetIRAnalysis(
1360 std::function<Result(const Function &)> TTICallback)
1361 : TTICallback(std::move(TTICallback)) {}
1362
1363TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1364 FunctionAnalysisManager &) {
1365 return TTICallback(F);
1366}
1367
1368AnalysisKey TargetIRAnalysis::Key;
1369
1370TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1371 return Result(F.getDataLayout());
1372}
1373
1374// Register the basic pass.
1375INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1376 "Target Transform Information", false, true)
1377char TargetTransformInfoWrapperPass::ID = 0;
1378
1379void TargetTransformInfoWrapperPass::anchor() {}
1380
1381TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1382 : ImmutablePass(ID) {
1383 initializeTargetTransformInfoWrapperPassPass(
1384 Registry&: *PassRegistry::getPassRegistry());
1385}
1386
1387TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1388 TargetIRAnalysis TIRA)
1389 : ImmutablePass(ID), TIRA(std::move(TIRA)) {
1390 initializeTargetTransformInfoWrapperPassPass(
1391 Registry&: *PassRegistry::getPassRegistry());
1392}
1393
1394TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1395 FunctionAnalysisManager DummyFAM;
1396 TTI = TIRA.run(F, DummyFAM);
1397 return *TTI;
1398}
1399
1400ImmutablePass *
1401llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1402 return new TargetTransformInfoWrapperPass(std::move(TIRA));
1403}
1404