1//===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "llvm/Analysis/TargetTransformInfo.h"
10#include "llvm/ADT/SmallVector.h"
11#include "llvm/Analysis/CFG.h"
12#include "llvm/Analysis/LoopIterator.h"
13#include "llvm/Analysis/TargetLibraryInfo.h"
14#include "llvm/Analysis/TargetTransformInfoImpl.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Dominators.h"
17#include "llvm/IR/Instruction.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/Module.h"
21#include "llvm/IR/Operator.h"
22#include "llvm/InitializePasses.h"
23#include "llvm/Support/CommandLine.h"
24#include <optional>
25#include <utility>
26
27using namespace llvm;
28using namespace PatternMatch;
29
30#define DEBUG_TYPE "tti"
31
32static cl::opt<bool> EnableReduxCost("costmodel-reduxcost", cl::init(Val: false),
33 cl::Hidden,
34 cl::desc("Recognize reduction patterns."));
35
36static cl::opt<unsigned> CacheLineSize(
37 "cache-line-size", cl::init(Val: 0), cl::Hidden,
38 cl::desc("Use this to override the target cache line size when "
39 "specified by the user."));
40
41static cl::opt<unsigned> MinPageSize(
42 "min-page-size", cl::init(Val: 0), cl::Hidden,
43 cl::desc("Use this to override the target's minimum page size."));
44
45static cl::opt<unsigned> PredictableBranchThreshold(
46 "predictable-branch-threshold", cl::init(Val: 99), cl::Hidden,
47 cl::desc(
48 "Use this to override the target's predictable branch threshold (%)."));
49
50namespace {
51/// No-op implementation of the TTI interface using the utility base
52/// classes.
53///
54/// This is used when no target specific information is available.
55struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> {
56 explicit NoTTIImpl(const DataLayout &DL)
57 : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {}
58};
59} // namespace
60
61TargetTransformInfo::TargetTransformInfo(
62 std::unique_ptr<const TargetTransformInfoImplBase> Impl)
63 : TTIImpl(std::move(Impl)) {}
64
65bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) {
66 // If the loop has irreducible control flow, it can not be converted to
67 // Hardware loop.
68 LoopBlocksRPO RPOT(L);
69 RPOT.perform(LI: &LI);
70 if (containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI))
71 return false;
72 return true;
73}
74
75IntrinsicCostAttributes::IntrinsicCostAttributes(
76 Intrinsic::ID Id, const CallBase &CI, InstructionCost ScalarizationCost,
77 bool TypeBasedOnly, const TargetLibraryInfo *LibInfo)
78 : II(dyn_cast<IntrinsicInst>(Val: &CI)), RetTy(CI.getType()), IID(Id),
79 ScalarizationCost(ScalarizationCost), LibInfo(LibInfo) {
80
81 if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: &CI))
82 FMF = FPMO->getFastMathFlags();
83
84 if (!TypeBasedOnly)
85 Arguments.insert(I: Arguments.begin(), From: CI.arg_begin(), To: CI.arg_end());
86 FunctionType *FTy = CI.getCalledFunction()->getFunctionType();
87 ParamTys.insert(I: ParamTys.begin(), From: FTy->param_begin(), To: FTy->param_end());
88}
89
90IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy,
91 ArrayRef<Type *> Tys,
92 FastMathFlags Flags,
93 const IntrinsicInst *I,
94 InstructionCost ScalarCost)
95 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) {
96 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
97}
98
99IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty,
100 ArrayRef<const Value *> Args)
101 : RetTy(Ty), IID(Id) {
102
103 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
104 ParamTys.reserve(N: Arguments.size());
105 for (const Value *Argument : Arguments)
106 ParamTys.push_back(Elt: Argument->getType());
107}
108
109IntrinsicCostAttributes::IntrinsicCostAttributes(
110 Intrinsic::ID Id, Type *RTy, ArrayRef<const Value *> Args,
111 ArrayRef<Type *> Tys, FastMathFlags Flags, const IntrinsicInst *I,
112 InstructionCost ScalarCost, TargetLibraryInfo const *LibInfo)
113 : II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost),
114 LibInfo(LibInfo) {
115 ParamTys.insert(I: ParamTys.begin(), From: Tys.begin(), To: Tys.end());
116 Arguments.insert(I: Arguments.begin(), From: Args.begin(), To: Args.end());
117}
118
119HardwareLoopInfo::HardwareLoopInfo(Loop *L) : L(L) {
120 // Match default options:
121 // - hardware-loop-counter-bitwidth = 32
122 // - hardware-loop-decrement = 1
123 CountType = Type::getInt32Ty(C&: L->getHeader()->getContext());
124 LoopDecrement = ConstantInt::get(Ty: CountType, V: 1);
125}
126
127bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
128 LoopInfo &LI, DominatorTree &DT,
129 bool ForceNestedLoop,
130 bool ForceHardwareLoopPHI) {
131 SmallVector<BasicBlock *, 4> ExitingBlocks;
132 L->getExitingBlocks(ExitingBlocks);
133
134 for (BasicBlock *BB : ExitingBlocks) {
135 // If we pass the updated counter back through a phi, we need to know
136 // which latch the updated value will be coming from.
137 if (!L->isLoopLatch(BB)) {
138 if (ForceHardwareLoopPHI || CounterInReg)
139 continue;
140 }
141
142 const SCEV *EC = SE.getExitCount(L, ExitingBlock: BB);
143 if (isa<SCEVCouldNotCompute>(Val: EC))
144 continue;
145 if (const SCEVConstant *ConstEC = dyn_cast<SCEVConstant>(Val: EC)) {
146 if (ConstEC->getValue()->isZero())
147 continue;
148 } else if (!SE.isLoopInvariant(S: EC, L))
149 continue;
150
151 if (SE.getTypeSizeInBits(Ty: EC->getType()) > CountType->getBitWidth())
152 continue;
153
154 // If this exiting block is contained in a nested loop, it is not eligible
155 // for insertion of the branch-and-decrement since the inner loop would
156 // end up messing up the value in the CTR.
157 if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop)
158 continue;
159
160 // We now have a loop-invariant count of loop iterations (which is not the
161 // constant zero) for which we know that this loop will not exit via this
162 // existing block.
163
164 // We need to make sure that this block will run on every loop iteration.
165 // For this to be true, we must dominate all blocks with backedges. Such
166 // blocks are in-loop predecessors to the header block.
167 bool NotAlways = false;
168 for (BasicBlock *Pred : predecessors(BB: L->getHeader())) {
169 if (!L->contains(BB: Pred))
170 continue;
171
172 if (!DT.dominates(A: BB, B: Pred)) {
173 NotAlways = true;
174 break;
175 }
176 }
177
178 if (NotAlways)
179 continue;
180
181 // Make sure this blocks ends with a conditional branch.
182 Instruction *TI = BB->getTerminator();
183 if (!TI)
184 continue;
185
186 if (BranchInst *BI = dyn_cast<BranchInst>(Val: TI)) {
187 if (!BI->isConditional())
188 continue;
189
190 ExitBranch = BI;
191 } else
192 continue;
193
194 // Note that this block may not be the loop latch block, even if the loop
195 // has a latch block.
196 ExitBlock = BB;
197 ExitCount = EC;
198 break;
199 }
200
201 if (!ExitBlock)
202 return false;
203 return true;
204}
205
206TargetTransformInfo::TargetTransformInfo(const DataLayout &DL)
207 : TTIImpl(std::make_unique<NoTTIImpl>(args: DL)) {}
208
209TargetTransformInfo::~TargetTransformInfo() = default;
210
211TargetTransformInfo::TargetTransformInfo(TargetTransformInfo &&Arg)
212 : TTIImpl(std::move(Arg.TTIImpl)) {}
213
214TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) {
215 TTIImpl = std::move(RHS.TTIImpl);
216 return *this;
217}
218
219unsigned TargetTransformInfo::getInliningThresholdMultiplier() const {
220 return TTIImpl->getInliningThresholdMultiplier();
221}
222
223unsigned
224TargetTransformInfo::getInliningCostBenefitAnalysisSavingsMultiplier() const {
225 return TTIImpl->getInliningCostBenefitAnalysisSavingsMultiplier();
226}
227
228unsigned
229TargetTransformInfo::getInliningCostBenefitAnalysisProfitableMultiplier()
230 const {
231 return TTIImpl->getInliningCostBenefitAnalysisProfitableMultiplier();
232}
233
234int TargetTransformInfo::getInliningLastCallToStaticBonus() const {
235 return TTIImpl->getInliningLastCallToStaticBonus();
236}
237
238unsigned
239TargetTransformInfo::adjustInliningThreshold(const CallBase *CB) const {
240 return TTIImpl->adjustInliningThreshold(CB);
241}
242
243unsigned TargetTransformInfo::getCallerAllocaCost(const CallBase *CB,
244 const AllocaInst *AI) const {
245 return TTIImpl->getCallerAllocaCost(CB, AI);
246}
247
248int TargetTransformInfo::getInlinerVectorBonusPercent() const {
249 return TTIImpl->getInlinerVectorBonusPercent();
250}
251
252InstructionCost TargetTransformInfo::getGEPCost(
253 Type *PointeeType, const Value *Ptr, ArrayRef<const Value *> Operands,
254 Type *AccessType, TTI::TargetCostKind CostKind) const {
255 return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, AccessType, CostKind);
256}
257
258InstructionCost TargetTransformInfo::getPointersChainCost(
259 ArrayRef<const Value *> Ptrs, const Value *Base,
260 const TTI::PointersChainInfo &Info, Type *AccessTy,
261 TTI::TargetCostKind CostKind) const {
262 assert((Base || !Info.isSameBase()) &&
263 "If pointers have same base address it has to be provided.");
264 return TTIImpl->getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
265}
266
267unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters(
268 const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI,
269 BlockFrequencyInfo *BFI) const {
270 return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI);
271}
272
273InstructionCost
274TargetTransformInfo::getInstructionCost(const User *U,
275 ArrayRef<const Value *> Operands,
276 enum TargetCostKind CostKind) const {
277 InstructionCost Cost = TTIImpl->getInstructionCost(U, Operands, CostKind);
278 assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) &&
279 "TTI should not produce negative costs!");
280 return Cost;
281}
282
283BranchProbability TargetTransformInfo::getPredictableBranchThreshold() const {
284 return PredictableBranchThreshold.getNumOccurrences() > 0
285 ? BranchProbability(PredictableBranchThreshold, 100)
286 : TTIImpl->getPredictableBranchThreshold();
287}
288
289InstructionCost TargetTransformInfo::getBranchMispredictPenalty() const {
290 return TTIImpl->getBranchMispredictPenalty();
291}
292
293bool TargetTransformInfo::hasBranchDivergence(const Function *F) const {
294 return TTIImpl->hasBranchDivergence(F);
295}
296
297InstructionUniformity
298llvm::TargetTransformInfo::getInstructionUniformity(const Value *V) const {
299 // Calls with the NoDivergenceSource attribute are always uniform.
300 if (const auto *Call = dyn_cast<CallBase>(Val: V)) {
301 if (Call->hasFnAttr(Kind: Attribute::NoDivergenceSource))
302 return InstructionUniformity::AlwaysUniform;
303 }
304 return TTIImpl->getInstructionUniformity(V);
305}
306
307bool llvm::TargetTransformInfo::isValidAddrSpaceCast(unsigned FromAS,
308 unsigned ToAS) const {
309 return TTIImpl->isValidAddrSpaceCast(FromAS, ToAS);
310}
311
312bool llvm::TargetTransformInfo::addrspacesMayAlias(unsigned FromAS,
313 unsigned ToAS) const {
314 return TTIImpl->addrspacesMayAlias(AS0: FromAS, AS1: ToAS);
315}
316
317unsigned TargetTransformInfo::getFlatAddressSpace() const {
318 return TTIImpl->getFlatAddressSpace();
319}
320
321bool TargetTransformInfo::collectFlatAddressOperands(
322 SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const {
323 return TTIImpl->collectFlatAddressOperands(OpIndexes, IID);
324}
325
326bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS,
327 unsigned ToAS) const {
328 return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS);
329}
330
331std::pair<KnownBits, KnownBits>
332TargetTransformInfo::computeKnownBitsAddrSpaceCast(unsigned ToAS,
333 const Value &PtrOp) const {
334 return TTIImpl->computeKnownBitsAddrSpaceCast(ToAS, PtrOp);
335}
336
337KnownBits TargetTransformInfo::computeKnownBitsAddrSpaceCast(
338 unsigned FromAS, unsigned ToAS, const KnownBits &FromPtrBits) const {
339 return TTIImpl->computeKnownBitsAddrSpaceCast(FromAS, ToAS, FromPtrBits);
340}
341
342APInt TargetTransformInfo::getAddrSpaceCastPreservedPtrMask(
343 unsigned SrcAS, unsigned DstAS) const {
344 return TTIImpl->getAddrSpaceCastPreservedPtrMask(SrcAS, DstAS);
345}
346
347bool TargetTransformInfo::canHaveNonUndefGlobalInitializerInAddressSpace(
348 unsigned AS) const {
349 return TTIImpl->canHaveNonUndefGlobalInitializerInAddressSpace(AS);
350}
351
352unsigned TargetTransformInfo::getAssumedAddrSpace(const Value *V) const {
353 return TTIImpl->getAssumedAddrSpace(V);
354}
355
356bool TargetTransformInfo::isSingleThreaded() const {
357 return TTIImpl->isSingleThreaded();
358}
359
360std::pair<const Value *, unsigned>
361TargetTransformInfo::getPredicatedAddrSpace(const Value *V) const {
362 return TTIImpl->getPredicatedAddrSpace(V);
363}
364
365Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace(
366 IntrinsicInst *II, Value *OldV, Value *NewV) const {
367 return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV);
368}
369
370bool TargetTransformInfo::isLoweredToCall(const Function *F) const {
371 return TTIImpl->isLoweredToCall(F);
372}
373
374bool TargetTransformInfo::isHardwareLoopProfitable(
375 Loop *L, ScalarEvolution &SE, AssumptionCache &AC,
376 TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const {
377 return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
378}
379
380unsigned TargetTransformInfo::getEpilogueVectorizationMinVF() const {
381 return TTIImpl->getEpilogueVectorizationMinVF();
382}
383
384bool TargetTransformInfo::preferPredicateOverEpilogue(
385 TailFoldingInfo *TFI) const {
386 return TTIImpl->preferPredicateOverEpilogue(TFI);
387}
388
389TailFoldingStyle TargetTransformInfo::getPreferredTailFoldingStyle(
390 bool IVUpdateMayOverflow) const {
391 return TTIImpl->getPreferredTailFoldingStyle(IVUpdateMayOverflow);
392}
393
394std::optional<Instruction *>
395TargetTransformInfo::instCombineIntrinsic(InstCombiner &IC,
396 IntrinsicInst &II) const {
397 return TTIImpl->instCombineIntrinsic(IC, II);
398}
399
400std::optional<Value *> TargetTransformInfo::simplifyDemandedUseBitsIntrinsic(
401 InstCombiner &IC, IntrinsicInst &II, APInt DemandedMask, KnownBits &Known,
402 bool &KnownBitsComputed) const {
403 return TTIImpl->simplifyDemandedUseBitsIntrinsic(IC, II, DemandedMask, Known,
404 KnownBitsComputed);
405}
406
407std::optional<Value *> TargetTransformInfo::simplifyDemandedVectorEltsIntrinsic(
408 InstCombiner &IC, IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts,
409 APInt &UndefElts2, APInt &UndefElts3,
410 std::function<void(Instruction *, unsigned, APInt, APInt &)>
411 SimplifyAndSetOp) const {
412 return TTIImpl->simplifyDemandedVectorEltsIntrinsic(
413 IC, II, DemandedElts, UndefElts, UndefElts2, UndefElts3,
414 SimplifyAndSetOp);
415}
416
417void TargetTransformInfo::getUnrollingPreferences(
418 Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP,
419 OptimizationRemarkEmitter *ORE) const {
420 return TTIImpl->getUnrollingPreferences(L, SE, UP, ORE);
421}
422
423void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
424 PeelingPreferences &PP) const {
425 return TTIImpl->getPeelingPreferences(L, SE, PP);
426}
427
428bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const {
429 return TTIImpl->isLegalAddImmediate(Imm);
430}
431
432bool TargetTransformInfo::isLegalAddScalableImmediate(int64_t Imm) const {
433 return TTIImpl->isLegalAddScalableImmediate(Imm);
434}
435
436bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const {
437 return TTIImpl->isLegalICmpImmediate(Imm);
438}
439
440bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV,
441 int64_t BaseOffset,
442 bool HasBaseReg, int64_t Scale,
443 unsigned AddrSpace,
444 Instruction *I,
445 int64_t ScalableOffset) const {
446 return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg,
447 Scale, AddrSpace, I, ScalableOffset);
448}
449
450bool TargetTransformInfo::isLSRCostLess(const LSRCost &C1,
451 const LSRCost &C2) const {
452 return TTIImpl->isLSRCostLess(C1, C2);
453}
454
455bool TargetTransformInfo::isNumRegsMajorCostOfLSR() const {
456 return TTIImpl->isNumRegsMajorCostOfLSR();
457}
458
459bool TargetTransformInfo::shouldDropLSRSolutionIfLessProfitable() const {
460 return TTIImpl->shouldDropLSRSolutionIfLessProfitable();
461}
462
463bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const {
464 return TTIImpl->isProfitableLSRChainElement(I);
465}
466
467bool TargetTransformInfo::canMacroFuseCmp() const {
468 return TTIImpl->canMacroFuseCmp();
469}
470
471bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI,
472 ScalarEvolution *SE, LoopInfo *LI,
473 DominatorTree *DT, AssumptionCache *AC,
474 TargetLibraryInfo *LibInfo) const {
475 return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo);
476}
477
478TTI::AddressingModeKind
479TargetTransformInfo::getPreferredAddressingMode(const Loop *L,
480 ScalarEvolution *SE) const {
481 return TTIImpl->getPreferredAddressingMode(L, SE);
482}
483
484bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, Align Alignment,
485 unsigned AddressSpace,
486 TTI::MaskKind MaskKind) const {
487 return TTIImpl->isLegalMaskedStore(DataType, Alignment, AddressSpace,
488 MaskKind);
489}
490
491bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, Align Alignment,
492 unsigned AddressSpace,
493 TTI::MaskKind MaskKind) const {
494 return TTIImpl->isLegalMaskedLoad(DataType, Alignment, AddressSpace,
495 MaskKind);
496}
497
498bool TargetTransformInfo::isLegalNTStore(Type *DataType,
499 Align Alignment) const {
500 return TTIImpl->isLegalNTStore(DataType, Alignment);
501}
502
503bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
504 return TTIImpl->isLegalNTLoad(DataType, Alignment);
505}
506
507bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,
508 ElementCount NumElements) const {
509 return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);
510}
511
512bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
513 Align Alignment) const {
514 return TTIImpl->isLegalMaskedGather(DataType, Alignment);
515}
516
517bool TargetTransformInfo::isLegalAltInstr(
518 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
519 const SmallBitVector &OpcodeMask) const {
520 return TTIImpl->isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask);
521}
522
523bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType,
524 Align Alignment) const {
525 return TTIImpl->isLegalMaskedScatter(DataType, Alignment);
526}
527
528bool TargetTransformInfo::forceScalarizeMaskedGather(VectorType *DataType,
529 Align Alignment) const {
530 return TTIImpl->forceScalarizeMaskedGather(DataType, Alignment);
531}
532
533bool TargetTransformInfo::forceScalarizeMaskedScatter(VectorType *DataType,
534 Align Alignment) const {
535 return TTIImpl->forceScalarizeMaskedScatter(DataType, Alignment);
536}
537
538bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType,
539 Align Alignment) const {
540 return TTIImpl->isLegalMaskedCompressStore(DataType, Alignment);
541}
542
543bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType,
544 Align Alignment) const {
545 return TTIImpl->isLegalMaskedExpandLoad(DataType, Alignment);
546}
547
548bool TargetTransformInfo::isLegalStridedLoadStore(Type *DataType,
549 Align Alignment) const {
550 return TTIImpl->isLegalStridedLoadStore(DataType, Alignment);
551}
552
553bool TargetTransformInfo::isLegalInterleavedAccessType(
554 VectorType *VTy, unsigned Factor, Align Alignment,
555 unsigned AddrSpace) const {
556 return TTIImpl->isLegalInterleavedAccessType(VTy, Factor, Alignment,
557 AddrSpace);
558}
559
560bool TargetTransformInfo::isLegalMaskedVectorHistogram(Type *AddrType,
561 Type *DataType) const {
562 return TTIImpl->isLegalMaskedVectorHistogram(AddrType, DataType);
563}
564
565bool TargetTransformInfo::enableOrderedReductions() const {
566 return TTIImpl->enableOrderedReductions();
567}
568
569bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const {
570 return TTIImpl->hasDivRemOp(DataType, IsSigned);
571}
572
573bool TargetTransformInfo::hasVolatileVariant(Instruction *I,
574 unsigned AddrSpace) const {
575 return TTIImpl->hasVolatileVariant(I, AddrSpace);
576}
577
578bool TargetTransformInfo::prefersVectorizedAddressing() const {
579 return TTIImpl->prefersVectorizedAddressing();
580}
581
582InstructionCost TargetTransformInfo::getScalingFactorCost(
583 Type *Ty, GlobalValue *BaseGV, StackOffset BaseOffset, bool HasBaseReg,
584 int64_t Scale, unsigned AddrSpace) const {
585 InstructionCost Cost = TTIImpl->getScalingFactorCost(
586 Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace);
587 assert(Cost >= 0 && "TTI should not produce negative costs!");
588 return Cost;
589}
590
591bool TargetTransformInfo::LSRWithInstrQueries() const {
592 return TTIImpl->LSRWithInstrQueries();
593}
594
595bool TargetTransformInfo::isTruncateFree(Type *Ty1, Type *Ty2) const {
596 return TTIImpl->isTruncateFree(Ty1, Ty2);
597}
598
599bool TargetTransformInfo::isProfitableToHoist(Instruction *I) const {
600 return TTIImpl->isProfitableToHoist(I);
601}
602
603bool TargetTransformInfo::useAA() const { return TTIImpl->useAA(); }
604
605bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
606 return TTIImpl->isTypeLegal(Ty);
607}
608
609unsigned TargetTransformInfo::getRegUsageForType(Type *Ty) const {
610 return TTIImpl->getRegUsageForType(Ty);
611}
612
613bool TargetTransformInfo::shouldBuildLookupTables() const {
614 return TTIImpl->shouldBuildLookupTables();
615}
616
617bool TargetTransformInfo::shouldBuildLookupTablesForConstant(
618 Constant *C) const {
619 return TTIImpl->shouldBuildLookupTablesForConstant(C);
620}
621
622bool TargetTransformInfo::shouldBuildRelLookupTables() const {
623 return TTIImpl->shouldBuildRelLookupTables();
624}
625
626bool TargetTransformInfo::useColdCCForColdCall(Function &F) const {
627 return TTIImpl->useColdCCForColdCall(F);
628}
629
630bool TargetTransformInfo::useFastCCForInternalCall(Function &F) const {
631 return TTIImpl->useFastCCForInternalCall(F);
632}
633
634bool TargetTransformInfo::isTargetIntrinsicTriviallyScalarizable(
635 Intrinsic::ID ID) const {
636 return TTIImpl->isTargetIntrinsicTriviallyScalarizable(ID);
637}
638
639bool TargetTransformInfo::isTargetIntrinsicWithScalarOpAtArg(
640 Intrinsic::ID ID, unsigned ScalarOpdIdx) const {
641 return TTIImpl->isTargetIntrinsicWithScalarOpAtArg(ID, ScalarOpdIdx);
642}
643
644bool TargetTransformInfo::isTargetIntrinsicWithOverloadTypeAtArg(
645 Intrinsic::ID ID, int OpdIdx) const {
646 return TTIImpl->isTargetIntrinsicWithOverloadTypeAtArg(ID, OpdIdx);
647}
648
649bool TargetTransformInfo::isTargetIntrinsicWithStructReturnOverloadAtField(
650 Intrinsic::ID ID, int RetIdx) const {
651 return TTIImpl->isTargetIntrinsicWithStructReturnOverloadAtField(ID, RetIdx);
652}
653
654TargetTransformInfo::VectorInstrContext
655TargetTransformInfo::getVectorInstrContextHint(const Instruction *I) {
656 if (!I)
657 return VectorInstrContext::None;
658
659 // For inserts, check if the value being inserted comes from a single-use
660 // load.
661 if (isa<InsertElementInst>(Val: I) && isa<LoadInst>(Val: I->getOperand(i: 1)) &&
662 I->getOperand(i: 1)->hasOneUse())
663 return VectorInstrContext::Load;
664
665 // For extracts, check if it has a single use that is a store.
666 if (isa<ExtractElementInst>(Val: I) && I->hasOneUse() &&
667 isa<StoreInst>(Val: *I->user_begin()))
668 return VectorInstrContext::Store;
669
670 return VectorInstrContext::None;
671}
672
673InstructionCost TargetTransformInfo::getScalarizationOverhead(
674 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
675 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
676 TTI::VectorInstrContext VIC) const {
677 return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract,
678 CostKind, ForPoisonSrc, VL, VIC);
679}
680
681InstructionCost TargetTransformInfo::getOperandsScalarizationOverhead(
682 ArrayRef<Type *> Tys, TTI::TargetCostKind CostKind,
683 TTI::VectorInstrContext VIC) const {
684 return TTIImpl->getOperandsScalarizationOverhead(Tys, CostKind, VIC);
685}
686
687bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const {
688 return TTIImpl->supportsEfficientVectorElementLoadStore();
689}
690
691bool TargetTransformInfo::supportsTailCalls() const {
692 return TTIImpl->supportsTailCalls();
693}
694
695bool TargetTransformInfo::supportsTailCallFor(const CallBase *CB) const {
696 return TTIImpl->supportsTailCallFor(CB);
697}
698
699bool TargetTransformInfo::enableAggressiveInterleaving(
700 bool LoopHasReductions) const {
701 return TTIImpl->enableAggressiveInterleaving(LoopHasReductions);
702}
703
704TargetTransformInfo::MemCmpExpansionOptions
705TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
706 return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp);
707}
708
709bool TargetTransformInfo::enableSelectOptimize() const {
710 return TTIImpl->enableSelectOptimize();
711}
712
713bool TargetTransformInfo::shouldTreatInstructionLikeSelect(
714 const Instruction *I) const {
715 return TTIImpl->shouldTreatInstructionLikeSelect(I);
716}
717
718bool TargetTransformInfo::enableInterleavedAccessVectorization() const {
719 return TTIImpl->enableInterleavedAccessVectorization();
720}
721
722bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
723 return TTIImpl->enableMaskedInterleavedAccessVectorization();
724}
725
726bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
727 return TTIImpl->isFPVectorizationPotentiallyUnsafe();
728}
729
730bool
731TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context,
732 unsigned BitWidth,
733 unsigned AddressSpace,
734 Align Alignment,
735 unsigned *Fast) const {
736 return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth,
737 AddressSpace, Alignment, Fast);
738}
739
740TargetTransformInfo::PopcntSupportKind
741TargetTransformInfo::getPopcntSupport(unsigned IntTyWidthInBit) const {
742 return TTIImpl->getPopcntSupport(IntTyWidthInBit);
743}
744
745bool TargetTransformInfo::haveFastSqrt(Type *Ty) const {
746 return TTIImpl->haveFastSqrt(Ty);
747}
748
749bool TargetTransformInfo::isExpensiveToSpeculativelyExecute(
750 const Instruction *I) const {
751 return TTIImpl->isExpensiveToSpeculativelyExecute(I);
752}
753
754bool TargetTransformInfo::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const {
755 return TTIImpl->isFCmpOrdCheaperThanFCmpZero(Ty);
756}
757
758InstructionCost TargetTransformInfo::getFPOpCost(Type *Ty) const {
759 InstructionCost Cost = TTIImpl->getFPOpCost(Ty);
760 assert(Cost >= 0 && "TTI should not produce negative costs!");
761 return Cost;
762}
763
764InstructionCost TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode,
765 unsigned Idx,
766 const APInt &Imm,
767 Type *Ty) const {
768 InstructionCost Cost = TTIImpl->getIntImmCodeSizeCost(Opcode, Idx, Imm, Ty);
769 assert(Cost >= 0 && "TTI should not produce negative costs!");
770 return Cost;
771}
772
773InstructionCost
774TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty,
775 TTI::TargetCostKind CostKind) const {
776 InstructionCost Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind);
777 assert(Cost >= 0 && "TTI should not produce negative costs!");
778 return Cost;
779}
780
781InstructionCost TargetTransformInfo::getIntImmCostInst(
782 unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty,
783 TTI::TargetCostKind CostKind, Instruction *Inst) const {
784 InstructionCost Cost =
785 TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
786 assert(Cost >= 0 && "TTI should not produce negative costs!");
787 return Cost;
788}
789
790InstructionCost
791TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
792 const APInt &Imm, Type *Ty,
793 TTI::TargetCostKind CostKind) const {
794 InstructionCost Cost =
795 TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
796 assert(Cost >= 0 && "TTI should not produce negative costs!");
797 return Cost;
798}
799
800bool TargetTransformInfo::preferToKeepConstantsAttached(
801 const Instruction &Inst, const Function &Fn) const {
802 return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);
803}
804
805unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
806 return TTIImpl->getNumberOfRegisters(ClassID);
807}
808
809bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty,
810 bool IsStore) const {
811 return TTIImpl->hasConditionalLoadStoreForType(Ty, IsStore);
812}
813
814unsigned TargetTransformInfo::getRegisterClassForType(bool Vector,
815 Type *Ty) const {
816 return TTIImpl->getRegisterClassForType(Vector, Ty);
817}
818
819const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const {
820 return TTIImpl->getRegisterClassName(ClassID);
821}
822
823TypeSize TargetTransformInfo::getRegisterBitWidth(
824 TargetTransformInfo::RegisterKind K) const {
825 return TTIImpl->getRegisterBitWidth(K);
826}
827
828unsigned TargetTransformInfo::getMinVectorRegisterBitWidth() const {
829 return TTIImpl->getMinVectorRegisterBitWidth();
830}
831
832std::optional<unsigned> TargetTransformInfo::getMaxVScale() const {
833 return TTIImpl->getMaxVScale();
834}
835
836std::optional<unsigned> TargetTransformInfo::getVScaleForTuning() const {
837 return TTIImpl->getVScaleForTuning();
838}
839
840bool TargetTransformInfo::isVScaleKnownToBeAPowerOfTwo() const {
841 return TTIImpl->isVScaleKnownToBeAPowerOfTwo();
842}
843
844bool TargetTransformInfo::shouldMaximizeVectorBandwidth(
845 TargetTransformInfo::RegisterKind K) const {
846 return TTIImpl->shouldMaximizeVectorBandwidth(K);
847}
848
849ElementCount TargetTransformInfo::getMinimumVF(unsigned ElemWidth,
850 bool IsScalable) const {
851 return TTIImpl->getMinimumVF(ElemWidth, IsScalable);
852}
853
854unsigned TargetTransformInfo::getMaximumVF(unsigned ElemWidth,
855 unsigned Opcode) const {
856 return TTIImpl->getMaximumVF(ElemWidth, Opcode);
857}
858
859unsigned TargetTransformInfo::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
860 Type *ScalarValTy) const {
861 return TTIImpl->getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy);
862}
863
864bool TargetTransformInfo::shouldConsiderAddressTypePromotion(
865 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
866 return TTIImpl->shouldConsiderAddressTypePromotion(
867 I, AllowPromotionWithoutCommonHeader);
868}
869
870unsigned TargetTransformInfo::getCacheLineSize() const {
871 return CacheLineSize.getNumOccurrences() > 0 ? CacheLineSize
872 : TTIImpl->getCacheLineSize();
873}
874
875std::optional<unsigned>
876TargetTransformInfo::getCacheSize(CacheLevel Level) const {
877 return TTIImpl->getCacheSize(Level);
878}
879
880std::optional<unsigned>
881TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const {
882 return TTIImpl->getCacheAssociativity(Level);
883}
884
885std::optional<unsigned> TargetTransformInfo::getMinPageSize() const {
886 return MinPageSize.getNumOccurrences() > 0 ? MinPageSize
887 : TTIImpl->getMinPageSize();
888}
889
890unsigned TargetTransformInfo::getPrefetchDistance() const {
891 return TTIImpl->getPrefetchDistance();
892}
893
894unsigned TargetTransformInfo::getMinPrefetchStride(
895 unsigned NumMemAccesses, unsigned NumStridedMemAccesses,
896 unsigned NumPrefetches, bool HasCall) const {
897 return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses,
898 NumPrefetches, HasCall);
899}
900
901unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const {
902 return TTIImpl->getMaxPrefetchIterationsAhead();
903}
904
905bool TargetTransformInfo::enableWritePrefetching() const {
906 return TTIImpl->enableWritePrefetching();
907}
908
909bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
910 return TTIImpl->shouldPrefetchAddressSpace(AS);
911}
912
913InstructionCost TargetTransformInfo::getPartialReductionCost(
914 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
915 ElementCount VF, PartialReductionExtendKind OpAExtend,
916 PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
917 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
918 return TTIImpl->getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
919 AccumType, VF, OpAExtend, OpBExtend,
920 BinOp, CostKind, FMF);
921}
922
923unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
924 return TTIImpl->getMaxInterleaveFactor(VF);
925}
926
927TargetTransformInfo::OperandValueInfo
928TargetTransformInfo::getOperandInfo(const Value *V) {
929 OperandValueKind OpInfo = OK_AnyValue;
930 OperandValueProperties OpProps = OP_None;
931
932 // undef/poison don't materialize constants.
933 if (isa<UndefValue>(Val: V))
934 return {.Kind: OK_AnyValue, .Properties: OP_None};
935
936 if (isa<ConstantInt>(Val: V) || isa<ConstantFP>(Val: V)) {
937 if (const auto *CI = dyn_cast<ConstantInt>(Val: V)) {
938 if (CI->getValue().isPowerOf2())
939 OpProps = OP_PowerOf2;
940 else if (CI->getValue().isNegatedPowerOf2())
941 OpProps = OP_NegatedPowerOf2;
942 }
943 return {.Kind: OK_UniformConstantValue, .Properties: OpProps};
944 }
945
946 // A broadcast shuffle creates a uniform value.
947 // TODO: Add support for non-zero index broadcasts.
948 // TODO: Add support for different source vector width.
949 if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(Val: V))
950 if (ShuffleInst->isZeroEltSplat())
951 OpInfo = OK_UniformValue;
952
953 const Value *Splat = getSplatValue(V);
954
955 // Check for a splat of a constant or for a non uniform vector of constants
956 // and check if the constant(s) are all powers of two.
957 if (Splat) {
958 // Check for a splat of a uniform value. This is not loop aware, so return
959 // true only for the obviously uniform cases (argument, globalvalue)
960 if (isa<Argument>(Val: Splat) || isa<GlobalValue>(Val: Splat)) {
961 OpInfo = OK_UniformValue;
962 } else if (isa<Constant>(Val: Splat)) {
963 OpInfo = OK_UniformConstantValue;
964 if (auto *CI = dyn_cast<ConstantInt>(Val: Splat)) {
965 if (CI->getValue().isPowerOf2())
966 OpProps = OP_PowerOf2;
967 else if (CI->getValue().isNegatedPowerOf2())
968 OpProps = OP_NegatedPowerOf2;
969 }
970 }
971 } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(Val: V)) {
972 OpInfo = OK_NonUniformConstantValue;
973 bool AllPow2 = true, AllNegPow2 = true;
974 for (uint64_t I = 0, E = CDS->getNumElements(); I != E; ++I) {
975 if (auto *CI = dyn_cast<ConstantInt>(Val: CDS->getElementAsConstant(i: I))) {
976 AllPow2 &= CI->getValue().isPowerOf2();
977 AllNegPow2 &= CI->getValue().isNegatedPowerOf2();
978 if (AllPow2 || AllNegPow2)
979 continue;
980 }
981 AllPow2 = AllNegPow2 = false;
982 break;
983 }
984 OpProps = AllPow2 ? OP_PowerOf2 : OpProps;
985 OpProps = AllNegPow2 ? OP_NegatedPowerOf2 : OpProps;
986 } else if (isa<ConstantVector>(Val: V) || isa<ConstantDataVector>(Val: V)) {
987 OpInfo = OK_NonUniformConstantValue;
988 }
989
990 return {.Kind: OpInfo, .Properties: OpProps};
991}
992
993TargetTransformInfo::OperandValueInfo
994TargetTransformInfo::commonOperandInfo(const Value *X, const Value *Y) {
995 OperandValueInfo OpInfoX = getOperandInfo(V: X);
996 if (X == Y)
997 return OpInfoX;
998 return OpInfoX.mergeWith(OpInfoY: getOperandInfo(V: Y));
999}
1000
1001InstructionCost TargetTransformInfo::getArithmeticInstrCost(
1002 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1003 OperandValueInfo Op1Info, OperandValueInfo Op2Info,
1004 ArrayRef<const Value *> Args, const Instruction *CxtI,
1005 const TargetLibraryInfo *TLibInfo) const {
1006
1007 // Use call cost for frem intructions that have platform specific vector math
1008 // functions, as those will be replaced with calls later by SelectionDAG or
1009 // ReplaceWithVecLib pass.
1010 if (TLibInfo && Opcode == Instruction::FRem) {
1011 VectorType *VecTy = dyn_cast<VectorType>(Val: Ty);
1012 LibFunc Func;
1013 if (VecTy &&
1014 TLibInfo->getLibFunc(Opcode: Instruction::FRem, Ty: Ty->getScalarType(), F&: Func) &&
1015 TLibInfo->isFunctionVectorizable(F: TLibInfo->getName(F: Func),
1016 VF: VecTy->getElementCount()))
1017 return getCallInstrCost(F: nullptr, RetTy: VecTy, Tys: {VecTy, VecTy}, CostKind);
1018 }
1019
1020 InstructionCost Cost = TTIImpl->getArithmeticInstrCost(
1021 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info, Args, CxtI);
1022 assert(Cost >= 0 && "TTI should not produce negative costs!");
1023 return Cost;
1024}
1025
1026InstructionCost TargetTransformInfo::getAltInstrCost(
1027 VectorType *VecTy, unsigned Opcode0, unsigned Opcode1,
1028 const SmallBitVector &OpcodeMask, TTI::TargetCostKind CostKind) const {
1029 InstructionCost Cost =
1030 TTIImpl->getAltInstrCost(VecTy, Opcode0, Opcode1, OpcodeMask, CostKind);
1031 assert(Cost >= 0 && "TTI should not produce negative costs!");
1032 return Cost;
1033}
1034
1035InstructionCost TargetTransformInfo::getShuffleCost(
1036 ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef<int> Mask,
1037 TTI::TargetCostKind CostKind, int Index, VectorType *SubTp,
1038 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
1039 assert((Mask.empty() || DstTy->isScalableTy() ||
1040 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1041 "Expected the Mask to match the return size if given");
1042 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1043 "Expected the same scalar types");
1044 InstructionCost Cost = TTIImpl->getShuffleCost(
1045 Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, Args, CxtI);
1046 assert(Cost >= 0 && "TTI should not produce negative costs!");
1047 return Cost;
1048}
1049
1050TargetTransformInfo::PartialReductionExtendKind
1051TargetTransformInfo::getPartialReductionExtendKind(Instruction *I) {
1052 if (auto *Cast = dyn_cast<CastInst>(Val: I))
1053 return getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
1054 return PR_None;
1055}
1056
1057TargetTransformInfo::PartialReductionExtendKind
1058TargetTransformInfo::getPartialReductionExtendKind(
1059 Instruction::CastOps CastOpc) {
1060 switch (CastOpc) {
1061 case Instruction::CastOps::ZExt:
1062 return PR_ZeroExtend;
1063 case Instruction::CastOps::SExt:
1064 return PR_SignExtend;
1065 case Instruction::CastOps::FPExt:
1066 return PR_FPExtend;
1067 default:
1068 return PR_None;
1069 }
1070 llvm_unreachable("Unhandled cast opcode");
1071}
1072
1073TTI::CastContextHint
1074TargetTransformInfo::getCastContextHint(const Instruction *I) {
1075 if (!I)
1076 return CastContextHint::None;
1077
1078 auto getLoadStoreKind = [](const Value *V, unsigned LdStOp, unsigned MaskedOp,
1079 unsigned GatScatOp) {
1080 const Instruction *I = dyn_cast<Instruction>(Val: V);
1081 if (!I)
1082 return CastContextHint::None;
1083
1084 if (I->getOpcode() == LdStOp)
1085 return CastContextHint::Normal;
1086
1087 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1088 if (II->getIntrinsicID() == MaskedOp)
1089 return TTI::CastContextHint::Masked;
1090 if (II->getIntrinsicID() == GatScatOp)
1091 return TTI::CastContextHint::GatherScatter;
1092 }
1093
1094 return TTI::CastContextHint::None;
1095 };
1096
1097 switch (I->getOpcode()) {
1098 case Instruction::ZExt:
1099 case Instruction::SExt:
1100 case Instruction::FPExt:
1101 return getLoadStoreKind(I->getOperand(i: 0), Instruction::Load,
1102 Intrinsic::masked_load, Intrinsic::masked_gather);
1103 case Instruction::Trunc:
1104 case Instruction::FPTrunc:
1105 if (I->hasOneUse())
1106 return getLoadStoreKind(*I->user_begin(), Instruction::Store,
1107 Intrinsic::masked_store,
1108 Intrinsic::masked_scatter);
1109 break;
1110 default:
1111 return CastContextHint::None;
1112 }
1113
1114 return TTI::CastContextHint::None;
1115}
1116
1117InstructionCost TargetTransformInfo::getCastInstrCost(
1118 unsigned Opcode, Type *Dst, Type *Src, CastContextHint CCH,
1119 TTI::TargetCostKind CostKind, const Instruction *I) const {
1120 assert((I == nullptr || I->getOpcode() == Opcode) &&
1121 "Opcode should reflect passed instruction.");
1122 InstructionCost Cost =
1123 TTIImpl->getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1124 assert(Cost >= 0 && "TTI should not produce negative costs!");
1125 return Cost;
1126}
1127
1128InstructionCost TargetTransformInfo::getExtractWithExtendCost(
1129 unsigned Opcode, Type *Dst, VectorType *VecTy, unsigned Index,
1130 TTI::TargetCostKind CostKind) const {
1131 InstructionCost Cost =
1132 TTIImpl->getExtractWithExtendCost(Opcode, Dst, VecTy, Index, CostKind);
1133 assert(Cost >= 0 && "TTI should not produce negative costs!");
1134 return Cost;
1135}
1136
1137InstructionCost TargetTransformInfo::getCFInstrCost(
1138 unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I) const {
1139 assert((I == nullptr || I->getOpcode() == Opcode) &&
1140 "Opcode should reflect passed instruction.");
1141 InstructionCost Cost = TTIImpl->getCFInstrCost(Opcode, CostKind, I);
1142 assert(Cost >= 0 && "TTI should not produce negative costs!");
1143 return Cost;
1144}
1145
1146InstructionCost TargetTransformInfo::getCmpSelInstrCost(
1147 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
1148 TTI::TargetCostKind CostKind, OperandValueInfo Op1Info,
1149 OperandValueInfo Op2Info, const Instruction *I) const {
1150 assert((I == nullptr || I->getOpcode() == Opcode) &&
1151 "Opcode should reflect passed instruction.");
1152 InstructionCost Cost = TTIImpl->getCmpSelInstrCost(
1153 Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
1154 assert(Cost >= 0 && "TTI should not produce negative costs!");
1155 return Cost;
1156}
1157
1158InstructionCost TargetTransformInfo::getVectorInstrCost(
1159 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1160 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
1161 assert((Opcode == Instruction::InsertElement ||
1162 Opcode == Instruction::ExtractElement) &&
1163 "Expecting Opcode to be insertelement/extractelement.");
1164 InstructionCost Cost =
1165 TTIImpl->getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC);
1166 assert(Cost >= 0 && "TTI should not produce negative costs!");
1167 return Cost;
1168}
1169
1170InstructionCost TargetTransformInfo::getVectorInstrCost(
1171 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
1172 Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
1173 TTI::VectorInstrContext VIC) const {
1174 assert((Opcode == Instruction::InsertElement ||
1175 Opcode == Instruction::ExtractElement) &&
1176 "Expecting Opcode to be insertelement/extractelement.");
1177 InstructionCost Cost = TTIImpl->getVectorInstrCost(
1178 Opcode, Val, CostKind, Index, Scalar, ScalarUserAndIdx, VIC);
1179 assert(Cost >= 0 && "TTI should not produce negative costs!");
1180 return Cost;
1181}
1182
1183InstructionCost TargetTransformInfo::getVectorInstrCost(
1184 const Instruction &I, Type *Val, TTI::TargetCostKind CostKind,
1185 unsigned Index, TTI::VectorInstrContext VIC) const {
1186 // FIXME: Assert that Opcode is either InsertElement or ExtractElement.
1187 // This is mentioned in the interface description and respected by all
1188 // callers, but never asserted upon.
1189 InstructionCost Cost =
1190 TTIImpl->getVectorInstrCost(I, Val, CostKind, Index, VIC);
1191 assert(Cost >= 0 && "TTI should not produce negative costs!");
1192 return Cost;
1193}
1194
1195InstructionCost TargetTransformInfo::getIndexedVectorInstrCostFromEnd(
1196 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind,
1197 unsigned Index) const {
1198 InstructionCost Cost =
1199 TTIImpl->getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind, Index);
1200 assert(Cost >= 0 && "TTI should not produce negative costs!");
1201 return Cost;
1202}
1203
1204InstructionCost TargetTransformInfo::getInsertExtractValueCost(
1205 unsigned Opcode, TTI::TargetCostKind CostKind) const {
1206 assert((Opcode == Instruction::InsertValue ||
1207 Opcode == Instruction::ExtractValue) &&
1208 "Expecting Opcode to be insertvalue/extractvalue.");
1209 InstructionCost Cost = TTIImpl->getInsertExtractValueCost(Opcode, CostKind);
1210 assert(Cost >= 0 && "TTI should not produce negative costs!");
1211 return Cost;
1212}
1213
1214InstructionCost TargetTransformInfo::getReplicationShuffleCost(
1215 Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
1216 TTI::TargetCostKind CostKind) const {
1217 InstructionCost Cost = TTIImpl->getReplicationShuffleCost(
1218 EltTy, ReplicationFactor, VF, DemandedDstElts, CostKind);
1219 assert(Cost >= 0 && "TTI should not produce negative costs!");
1220 return Cost;
1221}
1222
1223InstructionCost TargetTransformInfo::getMemoryOpCost(
1224 unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
1225 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
1226 const Instruction *I) const {
1227 assert((I == nullptr || I->getOpcode() == Opcode) &&
1228 "Opcode should reflect passed instruction.");
1229 InstructionCost Cost = TTIImpl->getMemoryOpCost(
1230 Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);
1231 assert(Cost >= 0 && "TTI should not produce negative costs!");
1232 return Cost;
1233}
1234
1235InstructionCost TargetTransformInfo::getInterleavedMemoryOpCost(
1236 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1237 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1238 bool UseMaskForCond, bool UseMaskForGaps) const {
1239 InstructionCost Cost = TTIImpl->getInterleavedMemoryOpCost(
1240 Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind,
1241 UseMaskForCond, UseMaskForGaps);
1242 assert(Cost >= 0 && "TTI should not produce negative costs!");
1243 return Cost;
1244}
1245
1246InstructionCost
1247TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1248 TTI::TargetCostKind CostKind) const {
1249 InstructionCost Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind);
1250 assert(Cost >= 0 && "TTI should not produce negative costs!");
1251 return Cost;
1252}
1253
1254InstructionCost TargetTransformInfo::getMemIntrinsicInstrCost(
1255 const MemIntrinsicCostAttributes &MICA,
1256 TTI::TargetCostKind CostKind) const {
1257 InstructionCost Cost = TTIImpl->getMemIntrinsicInstrCost(MICA, CostKind);
1258 assert(Cost >= 0 && "TTI should not produce negative costs!");
1259 return Cost;
1260}
1261
1262InstructionCost
1263TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy,
1264 ArrayRef<Type *> Tys,
1265 TTI::TargetCostKind CostKind) const {
1266 InstructionCost Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind);
1267 assert(Cost >= 0 && "TTI should not produce negative costs!");
1268 return Cost;
1269}
1270
1271unsigned TargetTransformInfo::getNumberOfParts(Type *Tp) const {
1272 return TTIImpl->getNumberOfParts(Tp);
1273}
1274
1275InstructionCost TargetTransformInfo::getAddressComputationCost(
1276 Type *PtrTy, ScalarEvolution *SE, const SCEV *Ptr,
1277 TTI::TargetCostKind CostKind) const {
1278 InstructionCost Cost =
1279 TTIImpl->getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1280 assert(Cost >= 0 && "TTI should not produce negative costs!");
1281 return Cost;
1282}
1283
1284InstructionCost TargetTransformInfo::getMemcpyCost(const Instruction *I) const {
1285 InstructionCost Cost = TTIImpl->getMemcpyCost(I);
1286 assert(Cost >= 0 && "TTI should not produce negative costs!");
1287 return Cost;
1288}
1289
1290uint64_t TargetTransformInfo::getMaxMemIntrinsicInlineSizeThreshold() const {
1291 return TTIImpl->getMaxMemIntrinsicInlineSizeThreshold();
1292}
1293
1294InstructionCost TargetTransformInfo::getArithmeticReductionCost(
1295 unsigned Opcode, VectorType *Ty, std::optional<FastMathFlags> FMF,
1296 TTI::TargetCostKind CostKind) const {
1297 InstructionCost Cost =
1298 TTIImpl->getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
1299 assert(Cost >= 0 && "TTI should not produce negative costs!");
1300 return Cost;
1301}
1302
1303InstructionCost TargetTransformInfo::getMinMaxReductionCost(
1304 Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF,
1305 TTI::TargetCostKind CostKind) const {
1306 InstructionCost Cost =
1307 TTIImpl->getMinMaxReductionCost(IID, Ty, FMF, CostKind);
1308 assert(Cost >= 0 && "TTI should not produce negative costs!");
1309 return Cost;
1310}
1311
1312InstructionCost TargetTransformInfo::getExtendedReductionCost(
1313 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *Ty,
1314 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1315 return TTIImpl->getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty, FMF,
1316 CostKind);
1317}
1318
1319InstructionCost TargetTransformInfo::getMulAccReductionCost(
1320 bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,
1321 TTI::TargetCostKind CostKind) const {
1322 return TTIImpl->getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty,
1323 CostKind);
1324}
1325
1326InstructionCost
1327TargetTransformInfo::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1328 return TTIImpl->getCostOfKeepingLiveOverCall(Tys);
1329}
1330
1331bool TargetTransformInfo::getTgtMemIntrinsic(IntrinsicInst *Inst,
1332 MemIntrinsicInfo &Info) const {
1333 return TTIImpl->getTgtMemIntrinsic(Inst, Info);
1334}
1335
1336unsigned TargetTransformInfo::getAtomicMemIntrinsicMaxElementSize() const {
1337 return TTIImpl->getAtomicMemIntrinsicMaxElementSize();
1338}
1339
1340Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic(
1341 IntrinsicInst *Inst, Type *ExpectedType, bool CanCreate) const {
1342 return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType,
1343 CanCreate);
1344}
1345
1346Type *TargetTransformInfo::getMemcpyLoopLoweringType(
1347 LLVMContext &Context, Value *Length, unsigned SrcAddrSpace,
1348 unsigned DestAddrSpace, Align SrcAlign, Align DestAlign,
1349 std::optional<uint32_t> AtomicElementSize) const {
1350 return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace,
1351 DestAddrSpace, SrcAlign, DestAlign,
1352 AtomicElementSize);
1353}
1354
1355void TargetTransformInfo::getMemcpyLoopResidualLoweringType(
1356 SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context,
1357 unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace,
1358 Align SrcAlign, Align DestAlign,
1359 std::optional<uint32_t> AtomicCpySize) const {
1360 TTIImpl->getMemcpyLoopResidualLoweringType(
1361 OpsOut, Context, RemainingBytes, SrcAddrSpace, DestAddrSpace, SrcAlign,
1362 DestAlign, AtomicCpySize);
1363}
1364
1365bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
1366 const Function *Callee) const {
1367 return TTIImpl->areInlineCompatible(Caller, Callee);
1368}
1369
1370unsigned
1371TargetTransformInfo::getInlineCallPenalty(const Function *F,
1372 const CallBase &Call,
1373 unsigned DefaultCallPenalty) const {
1374 return TTIImpl->getInlineCallPenalty(F, Call, DefaultCallPenalty);
1375}
1376
1377bool TargetTransformInfo::shouldCopyAttributeWhenOutliningFrom(
1378 const Function *Caller, const Attribute &Attr) const {
1379 return TTIImpl->shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
1380}
1381bool TargetTransformInfo::areTypesABICompatible(const Function *Caller,
1382 const Function *Callee,
1383 ArrayRef<Type *> Types) const {
1384 return TTIImpl->areTypesABICompatible(Caller, Callee, Types);
1385}
1386
1387bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
1388 Type *Ty) const {
1389 return TTIImpl->isIndexedLoadLegal(Mode, Ty);
1390}
1391
1392bool TargetTransformInfo::isIndexedStoreLegal(MemIndexedMode Mode,
1393 Type *Ty) const {
1394 return TTIImpl->isIndexedStoreLegal(Mode, Ty);
1395}
1396
1397unsigned TargetTransformInfo::getLoadStoreVecRegBitWidth(unsigned AS) const {
1398 return TTIImpl->getLoadStoreVecRegBitWidth(AddrSpace: AS);
1399}
1400
1401bool TargetTransformInfo::isLegalToVectorizeLoad(LoadInst *LI) const {
1402 return TTIImpl->isLegalToVectorizeLoad(LI);
1403}
1404
1405bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const {
1406 return TTIImpl->isLegalToVectorizeStore(SI);
1407}
1408
1409bool TargetTransformInfo::isLegalToVectorizeLoadChain(
1410 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1411 return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment,
1412 AddrSpace);
1413}
1414
1415bool TargetTransformInfo::isLegalToVectorizeStoreChain(
1416 unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const {
1417 return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment,
1418 AddrSpace);
1419}
1420
1421bool TargetTransformInfo::isLegalToVectorizeReduction(
1422 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
1423 return TTIImpl->isLegalToVectorizeReduction(RdxDesc, VF);
1424}
1425
1426bool TargetTransformInfo::isElementTypeLegalForScalableVector(Type *Ty) const {
1427 return TTIImpl->isElementTypeLegalForScalableVector(Ty);
1428}
1429
1430unsigned TargetTransformInfo::getLoadVectorFactor(unsigned VF,
1431 unsigned LoadSize,
1432 unsigned ChainSizeInBytes,
1433 VectorType *VecTy) const {
1434 return TTIImpl->getLoadVectorFactor(VF, LoadSize, ChainSizeInBytes, VecTy);
1435}
1436
1437unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF,
1438 unsigned StoreSize,
1439 unsigned ChainSizeInBytes,
1440 VectorType *VecTy) const {
1441 return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy);
1442}
1443
1444bool TargetTransformInfo::preferFixedOverScalableIfEqualCost(
1445 bool IsEpilogue) const {
1446 return TTIImpl->preferFixedOverScalableIfEqualCost(IsEpilogue);
1447}
1448
1449bool TargetTransformInfo::preferInLoopReduction(RecurKind Kind,
1450 Type *Ty) const {
1451 return TTIImpl->preferInLoopReduction(Kind, Ty);
1452}
1453
1454bool TargetTransformInfo::preferAlternateOpcodeVectorization() const {
1455 return TTIImpl->preferAlternateOpcodeVectorization();
1456}
1457
1458bool TargetTransformInfo::preferPredicatedReductionSelect() const {
1459 return TTIImpl->preferPredicatedReductionSelect();
1460}
1461
1462bool TargetTransformInfo::preferEpilogueVectorization(
1463 ElementCount Iters) const {
1464 return TTIImpl->preferEpilogueVectorization(Iters);
1465}
1466
1467bool TargetTransformInfo::shouldConsiderVectorizationRegPressure() const {
1468 return TTIImpl->shouldConsiderVectorizationRegPressure();
1469}
1470
1471TargetTransformInfo::VPLegalization
1472TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
1473 return TTIImpl->getVPLegalizationStrategy(PI: VPI);
1474}
1475
1476bool TargetTransformInfo::hasArmWideBranch(bool Thumb) const {
1477 return TTIImpl->hasArmWideBranch(Thumb);
1478}
1479
1480APInt TargetTransformInfo::getFeatureMask(const Function &F) const {
1481 return TTIImpl->getFeatureMask(F);
1482}
1483
1484APInt TargetTransformInfo::getPriorityMask(const Function &F) const {
1485 return TTIImpl->getPriorityMask(F);
1486}
1487
1488bool TargetTransformInfo::isMultiversionedFunction(const Function &F) const {
1489 return TTIImpl->isMultiversionedFunction(F);
1490}
1491
1492unsigned TargetTransformInfo::getMaxNumArgs() const {
1493 return TTIImpl->getMaxNumArgs();
1494}
1495
1496bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
1497 return TTIImpl->shouldExpandReduction(II);
1498}
1499
1500TargetTransformInfo::ReductionShuffle
1501TargetTransformInfo::getPreferredExpandedReductionShuffle(
1502 const IntrinsicInst *II) const {
1503 return TTIImpl->getPreferredExpandedReductionShuffle(II);
1504}
1505
1506unsigned TargetTransformInfo::getGISelRematGlobalCost() const {
1507 return TTIImpl->getGISelRematGlobalCost();
1508}
1509
1510unsigned TargetTransformInfo::getMinTripCountTailFoldingThreshold() const {
1511 return TTIImpl->getMinTripCountTailFoldingThreshold();
1512}
1513
1514bool TargetTransformInfo::supportsScalableVectors() const {
1515 return TTIImpl->supportsScalableVectors();
1516}
1517
1518bool TargetTransformInfo::enableScalableVectorization() const {
1519 return TTIImpl->enableScalableVectorization();
1520}
1521
1522bool TargetTransformInfo::hasActiveVectorLength() const {
1523 return TTIImpl->hasActiveVectorLength();
1524}
1525
1526bool TargetTransformInfo::isProfitableToSinkOperands(
1527 Instruction *I, SmallVectorImpl<Use *> &OpsToSink) const {
1528 return TTIImpl->isProfitableToSinkOperands(I, Ops&: OpsToSink);
1529}
1530
1531bool TargetTransformInfo::isVectorShiftByScalarCheap(Type *Ty) const {
1532 return TTIImpl->isVectorShiftByScalarCheap(Ty);
1533}
1534
1535unsigned
1536TargetTransformInfo::getNumBytesToPadGlobalArray(unsigned Size,
1537 Type *ArrayType) const {
1538 return TTIImpl->getNumBytesToPadGlobalArray(Size, ArrayType);
1539}
1540
1541void TargetTransformInfo::collectKernelLaunchBounds(
1542 const Function &F,
1543 SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const {
1544 return TTIImpl->collectKernelLaunchBounds(F, LB);
1545}
1546
1547bool TargetTransformInfo::allowVectorElementIndexingUsingGEP() const {
1548 return TTIImpl->allowVectorElementIndexingUsingGEP();
1549}
1550
1551TargetTransformInfoImplBase::~TargetTransformInfoImplBase() = default;
1552
1553TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
1554
1555TargetIRAnalysis::TargetIRAnalysis(
1556 std::function<Result(const Function &)> TTICallback)
1557 : TTICallback(std::move(TTICallback)) {}
1558
1559TargetIRAnalysis::Result TargetIRAnalysis::run(const Function &F,
1560 FunctionAnalysisManager &) {
1561 assert(!F.isIntrinsic() && "Should not request TTI for intrinsics");
1562 return TTICallback(F);
1563}
1564
1565AnalysisKey TargetIRAnalysis::Key;
1566
1567TargetIRAnalysis::Result TargetIRAnalysis::getDefaultTTI(const Function &F) {
1568 return Result(F.getDataLayout());
1569}
1570
1571// Register the basic pass.
1572INITIALIZE_PASS(TargetTransformInfoWrapperPass, "tti",
1573 "Target Transform Information", false, true)
1574char TargetTransformInfoWrapperPass::ID = 0;
1575
1576void TargetTransformInfoWrapperPass::anchor() {}
1577
1578TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass()
1579 : ImmutablePass(ID) {}
1580
1581TargetTransformInfoWrapperPass::TargetTransformInfoWrapperPass(
1582 TargetIRAnalysis TIRA)
1583 : ImmutablePass(ID), TIRA(std::move(TIRA)) {}
1584
1585TargetTransformInfo &TargetTransformInfoWrapperPass::getTTI(const Function &F) {
1586 FunctionAnalysisManager DummyFAM;
1587 TTI = TIRA.run(F, DummyFAM);
1588 return *TTI;
1589}
1590
1591ImmutablePass *
1592llvm::createTargetTransformInfoWrapperPass(TargetIRAnalysis TIRA) {
1593 return new TargetTransformInfoWrapperPass(std::move(TIRA));
1594}
1595