1//===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "PPCTargetTransformInfo.h"
10#include "llvm/Analysis/CodeMetrics.h"
11#include "llvm/Analysis/TargetLibraryInfo.h"
12#include "llvm/Analysis/TargetTransformInfo.h"
13#include "llvm/CodeGen/BasicTTIImpl.h"
14#include "llvm/CodeGen/TargetLowering.h"
15#include "llvm/CodeGen/TargetSchedule.h"
16#include "llvm/IR/IntrinsicsPowerPC.h"
17#include "llvm/IR/ProfDataUtils.h"
18#include "llvm/Support/CommandLine.h"
19#include "llvm/Transforms/InstCombine/InstCombiner.h"
20#include "llvm/Transforms/Utils/Local.h"
21#include <optional>
22
23using namespace llvm;
24
25#define DEBUG_TYPE "ppctti"
26
27static cl::opt<bool> PPCEVL("ppc-evl",
28 cl::desc("Allow EVL type vp.load/vp.store"),
29 cl::init(Val: false), cl::Hidden);
30
31static cl::opt<bool> Pwr9EVL("ppc-pwr9-evl",
32 cl::desc("Allow vp.load and vp.store for pwr9"),
33 cl::init(Val: false), cl::Hidden);
34
35static cl::opt<bool> VecMaskCost("ppc-vec-mask-cost",
36cl::desc("add masking cost for i1 vectors"), cl::init(Val: true), cl::Hidden);
37
38static cl::opt<bool> DisablePPCConstHoist("disable-ppc-constant-hoisting",
39cl::desc("disable constant hoisting on PPC"), cl::init(Val: false), cl::Hidden);
40
41static cl::opt<bool>
42EnablePPCColdCC("ppc-enable-coldcc", cl::Hidden, cl::init(Val: false),
43 cl::desc("Enable using coldcc calling conv for cold "
44 "internal functions"));
45
46static cl::opt<bool>
47LsrNoInsnsCost("ppc-lsr-no-insns-cost", cl::Hidden, cl::init(Val: false),
48 cl::desc("Do not add instruction count to lsr cost model"));
49
50// The latency of mtctr is only justified if there are more than 4
51// comparisons that will be removed as a result.
52static cl::opt<unsigned>
53SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(Val: 4), cl::Hidden,
54 cl::desc("Loops with a constant trip count smaller than "
55 "this value will not use the count register."));
56
57//===----------------------------------------------------------------------===//
58//
59// PPC cost model.
60//
61//===----------------------------------------------------------------------===//
62
63TargetTransformInfo::PopcntSupportKind
64PPCTTIImpl::getPopcntSupport(unsigned TyWidth) const {
65 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
66 if (ST->hasPOPCNTD() != PPCSubtarget::POPCNTD_Unavailable && TyWidth <= 64)
67 return ST->hasPOPCNTD() == PPCSubtarget::POPCNTD_Slow ?
68 TTI::PSK_SlowHardware : TTI::PSK_FastHardware;
69 return TTI::PSK_Software;
70}
71
72std::optional<Instruction *>
73PPCTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
74 Intrinsic::ID IID = II.getIntrinsicID();
75 switch (IID) {
76 default:
77 break;
78 case Intrinsic::ppc_altivec_lvx:
79 case Intrinsic::ppc_altivec_lvxl:
80 // Turn PPC lvx -> load if the pointer is known aligned.
81 if (getOrEnforceKnownAlignment(
82 V: II.getArgOperand(i: 0), PrefAlign: Align(16), DL: IC.getDataLayout(), CtxI: &II,
83 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree()) >= 16) {
84 Value *Ptr = II.getArgOperand(i: 0);
85 return new LoadInst(II.getType(), Ptr, "", false, Align(16));
86 }
87 break;
88 case Intrinsic::ppc_vsx_lxvw4x:
89 case Intrinsic::ppc_vsx_lxvd2x: {
90 // Turn PPC VSX loads into normal loads.
91 Value *Ptr = II.getArgOperand(i: 0);
92 return new LoadInst(II.getType(), Ptr, Twine(""), false, Align(1));
93 }
94 case Intrinsic::ppc_altivec_stvx:
95 case Intrinsic::ppc_altivec_stvxl:
96 // Turn stvx -> store if the pointer is known aligned.
97 if (getOrEnforceKnownAlignment(
98 V: II.getArgOperand(i: 1), PrefAlign: Align(16), DL: IC.getDataLayout(), CtxI: &II,
99 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree()) >= 16) {
100 Value *Ptr = II.getArgOperand(i: 1);
101 return new StoreInst(II.getArgOperand(i: 0), Ptr, false, Align(16));
102 }
103 break;
104 case Intrinsic::ppc_vsx_stxvw4x:
105 case Intrinsic::ppc_vsx_stxvd2x: {
106 // Turn PPC VSX stores into normal stores.
107 Value *Ptr = II.getArgOperand(i: 1);
108 return new StoreInst(II.getArgOperand(i: 0), Ptr, false, Align(1));
109 }
110 case Intrinsic::ppc_altivec_vperm:
111 // Turn vperm(V1,V2,mask) -> shuffle(V1,V2,mask) if mask is a constant.
112 // Note that ppc_altivec_vperm has a big-endian bias, so when creating
113 // a vectorshuffle for little endian, we must undo the transformation
114 // performed on vec_perm in altivec.h. That is, we must complement
115 // the permutation mask with respect to 31 and reverse the order of
116 // V1 and V2.
117 if (Constant *Mask = dyn_cast<Constant>(Val: II.getArgOperand(i: 2))) {
118 assert(cast<FixedVectorType>(Mask->getType())->getNumElements() == 16 &&
119 "Bad type for intrinsic!");
120
121 // Check that all of the elements are integer constants or undefs.
122 bool AllEltsOk = true;
123 for (unsigned I = 0; I != 16; ++I) {
124 Constant *Elt = Mask->getAggregateElement(Elt: I);
125 if (!Elt || !(isa<ConstantInt>(Val: Elt) || isa<UndefValue>(Val: Elt))) {
126 AllEltsOk = false;
127 break;
128 }
129 }
130
131 if (AllEltsOk) {
132 // Cast the input vectors to byte vectors.
133 Value *Op0 =
134 IC.Builder.CreateBitCast(V: II.getArgOperand(i: 0), DestTy: Mask->getType());
135 Value *Op1 =
136 IC.Builder.CreateBitCast(V: II.getArgOperand(i: 1), DestTy: Mask->getType());
137 Value *Result = PoisonValue::get(T: Op0->getType());
138
139 // Only extract each element once.
140 Value *ExtractedElts[32];
141 memset(s: ExtractedElts, c: 0, n: sizeof(ExtractedElts));
142
143 for (unsigned I = 0; I != 16; ++I) {
144 if (isa<UndefValue>(Val: Mask->getAggregateElement(Elt: I)))
145 continue;
146 unsigned Idx =
147 cast<ConstantInt>(Val: Mask->getAggregateElement(Elt: I))->getZExtValue();
148 Idx &= 31; // Match the hardware behavior.
149 if (DL.isLittleEndian())
150 Idx = 31 - Idx;
151
152 if (!ExtractedElts[Idx]) {
153 Value *Op0ToUse = (DL.isLittleEndian()) ? Op1 : Op0;
154 Value *Op1ToUse = (DL.isLittleEndian()) ? Op0 : Op1;
155 ExtractedElts[Idx] = IC.Builder.CreateExtractElement(
156 Vec: Idx < 16 ? Op0ToUse : Op1ToUse, Idx: Idx & 15);
157 }
158
159 // Insert this value into the result vector.
160 Result =
161 IC.Builder.CreateInsertElement(Vec: Result, NewElt: ExtractedElts[Idx], Idx: I);
162 }
163 return CastInst::Create(Instruction::BitCast, S: Result, Ty: II.getType());
164 }
165 }
166 break;
167 }
168 return std::nullopt;
169}
170
171InstructionCost PPCTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
172 TTI::TargetCostKind CostKind) const {
173 if (DisablePPCConstHoist)
174 return BaseT::getIntImmCost(Imm, Ty, CostKind);
175
176 assert(Ty->isIntegerTy());
177
178 unsigned BitSize = Ty->getPrimitiveSizeInBits();
179 if (BitSize == 0)
180 return ~0U;
181
182 if (Imm == 0)
183 return TTI::TCC_Free;
184
185 if (Imm.getBitWidth() <= 64) {
186 if (isInt<16>(x: Imm.getSExtValue()))
187 return TTI::TCC_Basic;
188
189 if (isInt<32>(x: Imm.getSExtValue())) {
190 // A constant that can be materialized using lis.
191 if ((Imm.getZExtValue() & 0xFFFF) == 0)
192 return TTI::TCC_Basic;
193
194 return 2 * TTI::TCC_Basic;
195 }
196 }
197
198 return 4 * TTI::TCC_Basic;
199}
200
201InstructionCost
202PPCTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
203 const APInt &Imm, Type *Ty,
204 TTI::TargetCostKind CostKind) const {
205 if (DisablePPCConstHoist)
206 return BaseT::getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
207
208 assert(Ty->isIntegerTy());
209
210 unsigned BitSize = Ty->getPrimitiveSizeInBits();
211 if (BitSize == 0)
212 return ~0U;
213
214 switch (IID) {
215 default:
216 return TTI::TCC_Free;
217 case Intrinsic::sadd_with_overflow:
218 case Intrinsic::uadd_with_overflow:
219 case Intrinsic::ssub_with_overflow:
220 case Intrinsic::usub_with_overflow:
221 if ((Idx == 1) && Imm.getBitWidth() <= 64 && isInt<16>(x: Imm.getSExtValue()))
222 return TTI::TCC_Free;
223 break;
224 case Intrinsic::experimental_stackmap:
225 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
226 return TTI::TCC_Free;
227 break;
228 case Intrinsic::experimental_patchpoint_void:
229 case Intrinsic::experimental_patchpoint:
230 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
231 return TTI::TCC_Free;
232 break;
233 }
234 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind);
235}
236
237InstructionCost PPCTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
238 const APInt &Imm, Type *Ty,
239 TTI::TargetCostKind CostKind,
240 Instruction *Inst) const {
241 if (DisablePPCConstHoist)
242 return BaseT::getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind, Inst);
243
244 assert(Ty->isIntegerTy());
245
246 unsigned BitSize = Ty->getPrimitiveSizeInBits();
247 if (BitSize == 0)
248 return ~0U;
249
250 unsigned ImmIdx = ~0U;
251 bool ShiftedFree = false, RunFree = false, UnsignedFree = false,
252 ZeroFree = false;
253 switch (Opcode) {
254 default:
255 return TTI::TCC_Free;
256 case Instruction::GetElementPtr:
257 // Always hoist the base address of a GetElementPtr. This prevents the
258 // creation of new constants for every base constant that gets constant
259 // folded with the offset.
260 if (Idx == 0)
261 return 2 * TTI::TCC_Basic;
262 return TTI::TCC_Free;
263 case Instruction::And:
264 RunFree = true; // (for the rotate-and-mask instructions)
265 [[fallthrough]];
266 case Instruction::Add:
267 case Instruction::Or:
268 case Instruction::Xor:
269 ShiftedFree = true;
270 [[fallthrough]];
271 case Instruction::Sub:
272 case Instruction::Mul:
273 case Instruction::Shl:
274 case Instruction::LShr:
275 case Instruction::AShr:
276 ImmIdx = 1;
277 break;
278 case Instruction::ICmp:
279 UnsignedFree = true;
280 ImmIdx = 1;
281 // Zero comparisons can use record-form instructions.
282 [[fallthrough]];
283 case Instruction::Select:
284 ZeroFree = true;
285 break;
286 case Instruction::PHI:
287 case Instruction::Call:
288 case Instruction::Ret:
289 case Instruction::Load:
290 case Instruction::Store:
291 break;
292 }
293
294 if (ZeroFree && Imm == 0)
295 return TTI::TCC_Free;
296
297 if (Idx == ImmIdx && Imm.getBitWidth() <= 64) {
298 if (isInt<16>(x: Imm.getSExtValue()))
299 return TTI::TCC_Free;
300
301 if (RunFree) {
302 if (Imm.getBitWidth() <= 32 &&
303 (isShiftedMask_32(Value: Imm.getZExtValue()) ||
304 isShiftedMask_32(Value: ~Imm.getZExtValue())))
305 return TTI::TCC_Free;
306
307 if (ST->isPPC64() &&
308 (isShiftedMask_64(Value: Imm.getZExtValue()) ||
309 isShiftedMask_64(Value: ~Imm.getZExtValue())))
310 return TTI::TCC_Free;
311 }
312
313 if (UnsignedFree && isUInt<16>(x: Imm.getZExtValue()))
314 return TTI::TCC_Free;
315
316 if (ShiftedFree && (Imm.getZExtValue() & 0xFFFF) == 0)
317 return TTI::TCC_Free;
318 }
319
320 return PPCTTIImpl::getIntImmCost(Imm, Ty, CostKind);
321}
322
323// Check if the current Type is an MMA vector type. Valid MMA types are
324// v256i1 and v512i1 respectively.
325static bool isMMAType(Type *Ty) {
326 return Ty->isVectorTy() && (Ty->getScalarSizeInBits() == 1) &&
327 (Ty->getPrimitiveSizeInBits() > 128);
328}
329
330InstructionCost
331PPCTTIImpl::getInstructionCost(const User *U, ArrayRef<const Value *> Operands,
332 TTI::TargetCostKind CostKind) const {
333 // We already implement getCastInstrCost and getMemoryOpCost where we perform
334 // the vector adjustment there.
335 if (isa<CastInst>(Val: U) || isa<LoadInst>(Val: U) || isa<StoreInst>(Val: U))
336 return BaseT::getInstructionCost(U, Operands, CostKind);
337
338 if (U->getType()->isVectorTy()) {
339 // Instructions that need to be split should cost more.
340 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: U->getType());
341 return LT.first * BaseT::getInstructionCost(U, Operands, CostKind);
342 }
343
344 return BaseT::getInstructionCost(U, Operands, CostKind);
345}
346
347bool PPCTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
348 AssumptionCache &AC,
349 TargetLibraryInfo *LibInfo,
350 HardwareLoopInfo &HWLoopInfo) const {
351 const PPCTargetMachine &TM = ST->getTargetMachine();
352 TargetSchedModel SchedModel;
353 SchedModel.init(TSInfo: ST);
354
355 // Do not convert small short loops to CTR loop.
356 unsigned ConstTripCount = SE.getSmallConstantTripCount(L);
357 if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) {
358 SmallPtrSet<const Value *, 32> EphValues;
359 CodeMetrics::collectEphemeralValues(L, AC: &AC, EphValues);
360 InstructionCost NumInsts;
361 for (BasicBlock *BB : L->blocks()) {
362 for (Instruction &I : *BB) {
363 if (EphValues.count(Ptr: &I))
364 continue;
365 SmallVector<const Value *, 4> Operands(I.operand_values());
366 NumInsts += getInstructionCost(U: &I, Operands, CostKind: TTI::TCK_CodeSize);
367 }
368 }
369 // 6 is an approximate latency for the mtctr instruction.
370 if (NumInsts <= (6 * SchedModel.getIssueWidth()))
371 return false;
372 }
373
374 // Check that there is no hardware loop related intrinsics in the loop.
375 for (auto *BB : L->getBlocks())
376 for (auto &I : *BB)
377 if (auto *Call = dyn_cast<IntrinsicInst>(Val: &I))
378 if (Call->getIntrinsicID() == Intrinsic::set_loop_iterations ||
379 Call->getIntrinsicID() == Intrinsic::loop_decrement)
380 return false;
381
382 SmallVector<BasicBlock*, 4> ExitingBlocks;
383 L->getExitingBlocks(ExitingBlocks);
384
385 // If there is an exit edge known to be frequently taken,
386 // we should not transform this loop.
387 for (auto &BB : ExitingBlocks) {
388 Instruction *TI = BB->getTerminator();
389 if (!TI) continue;
390
391 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: TI)) {
392 uint64_t TrueWeight = 0, FalseWeight = 0;
393 if (!extractBranchWeights(I: *BI, TrueVal&: TrueWeight, FalseVal&: FalseWeight))
394 continue;
395
396 // If the exit path is more frequent than the loop path,
397 // we return here without further analysis for this loop.
398 bool TrueIsExit = !L->contains(BB: BI->getSuccessor(i: 0));
399 if (( TrueIsExit && FalseWeight < TrueWeight) ||
400 (!TrueIsExit && FalseWeight > TrueWeight))
401 return false;
402 }
403 }
404
405 LLVMContext &C = L->getHeader()->getContext();
406 HWLoopInfo.CountType = TM.isPPC64() ?
407 Type::getInt64Ty(C) : Type::getInt32Ty(C);
408 HWLoopInfo.LoopDecrement = ConstantInt::get(Ty: HWLoopInfo.CountType, V: 1);
409 return true;
410}
411
412void PPCTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
413 TTI::UnrollingPreferences &UP,
414 OptimizationRemarkEmitter *ORE) const {
415 if (ST->getCPUDirective() == PPC::DIR_A2) {
416 // The A2 is in-order with a deep pipeline, and concatenation unrolling
417 // helps expose latency-hiding opportunities to the instruction scheduler.
418 UP.Partial = UP.Runtime = true;
419
420 // We unroll a lot on the A2 (hundreds of instructions), and the benefits
421 // often outweigh the cost of a division to compute the trip count.
422 UP.AllowExpensiveTripCount = true;
423 }
424
425 BaseT::getUnrollingPreferences(L, SE, UP, ORE);
426}
427
428void PPCTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
429 TTI::PeelingPreferences &PP) const {
430 BaseT::getPeelingPreferences(L, SE, PP);
431}
432// This function returns true to allow using coldcc calling convention.
433// Returning true results in coldcc being used for functions which are cold at
434// all call sites when the callers of the functions are not calling any other
435// non coldcc functions.
436bool PPCTTIImpl::useColdCCForColdCall(Function &F) const {
437 return EnablePPCColdCC;
438}
439
440bool PPCTTIImpl::enableAggressiveInterleaving(bool LoopHasReductions) const {
441 // On the A2, always unroll aggressively.
442 if (ST->getCPUDirective() == PPC::DIR_A2)
443 return true;
444
445 return LoopHasReductions;
446}
447
448PPCTTIImpl::TTI::MemCmpExpansionOptions
449PPCTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
450 TTI::MemCmpExpansionOptions Options;
451 if (getST()->hasAltivec())
452 Options.LoadSizes = {16, 8, 4, 2, 1};
453 else
454 Options.LoadSizes = {8, 4, 2, 1};
455
456 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
457 return Options;
458}
459
460bool PPCTTIImpl::enableInterleavedAccessVectorization() const { return true; }
461
462unsigned PPCTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
463 assert(ClassID == GPRRC || ClassID == FPRRC ||
464 ClassID == VRRC || ClassID == VSXRC);
465 if (ST->hasVSX()) {
466 assert(ClassID == GPRRC || ClassID == VSXRC || ClassID == VRRC);
467 return ClassID == VSXRC ? 64 : 32;
468 }
469 assert(ClassID == GPRRC || ClassID == FPRRC || ClassID == VRRC);
470 return 32;
471}
472
473unsigned PPCTTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const {
474 if (Vector)
475 return ST->hasVSX() ? VSXRC : VRRC;
476 if (Ty &&
477 (Ty->getScalarType()->isFloatTy() || Ty->getScalarType()->isDoubleTy()))
478 return ST->hasVSX() ? VSXRC : FPRRC;
479 if (Ty && (Ty->getScalarType()->isFP128Ty() ||
480 Ty->getScalarType()->isPPC_FP128Ty()))
481 return VRRC;
482 if (Ty && Ty->getScalarType()->isHalfTy())
483 return VSXRC;
484 return GPRRC;
485}
486
487const char* PPCTTIImpl::getRegisterClassName(unsigned ClassID) const {
488
489 switch (ClassID) {
490 default:
491 llvm_unreachable("unknown register class");
492 return "PPC::unknown register class";
493 case GPRRC: return "PPC::GPRRC";
494 case FPRRC: return "PPC::FPRRC";
495 case VRRC: return "PPC::VRRC";
496 case VSXRC: return "PPC::VSXRC";
497 }
498}
499
500TypeSize
501PPCTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
502 switch (K) {
503 case TargetTransformInfo::RGK_Scalar:
504 return TypeSize::getFixed(ExactSize: ST->isPPC64() ? 64 : 32);
505 case TargetTransformInfo::RGK_FixedWidthVector:
506 return TypeSize::getFixed(ExactSize: ST->hasAltivec() ? 128 : 0);
507 case TargetTransformInfo::RGK_ScalableVector:
508 return TypeSize::getScalable(MinimumSize: 0);
509 }
510
511 llvm_unreachable("Unsupported register kind");
512}
513
514unsigned PPCTTIImpl::getCacheLineSize() const {
515 // Starting with P7 we have a cache line size of 128.
516 unsigned Directive = ST->getCPUDirective();
517 // Assume that Future CPU has the same cache line size as the others.
518 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 ||
519 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 ||
520 Directive == PPC::DIR_PWR11 || Directive == PPC::DIR_PWR_FUTURE)
521 return 128;
522
523 // On other processors return a default of 64 bytes.
524 return 64;
525}
526
527unsigned PPCTTIImpl::getPrefetchDistance() const {
528 return 300;
529}
530
531unsigned PPCTTIImpl::getMaxInterleaveFactor(ElementCount VF,
532 bool HasUnorderedReductions) const {
533 unsigned Directive = ST->getCPUDirective();
534 // The 440 has no SIMD support, but floating-point instructions
535 // have a 5-cycle latency, so unroll by 5x for latency hiding.
536 if (Directive == PPC::DIR_440)
537 return 5;
538
539 // The A2 has no SIMD support, but floating-point instructions
540 // have a 6-cycle latency, so unroll by 6x for latency hiding.
541 if (Directive == PPC::DIR_A2)
542 return 6;
543
544 // FIXME: For lack of any better information, do no harm...
545 if (Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500)
546 return 1;
547
548 // For P7 and P8, floating-point instructions have a 6-cycle latency and
549 // there are two execution units, so unroll by 12x for latency hiding.
550 // FIXME: the same for P9 as previous gen until POWER9 scheduling is ready
551 // FIXME: the same for P10 as previous gen until POWER10 scheduling is ready
552 // Assume that future is the same as the others.
553 if (Directive == PPC::DIR_PWR7 || Directive == PPC::DIR_PWR8 ||
554 Directive == PPC::DIR_PWR9 || Directive == PPC::DIR_PWR10 ||
555 Directive == PPC::DIR_PWR11 || Directive == PPC::DIR_PWR_FUTURE)
556 return 12;
557
558 // For most things, modern systems have two execution units (and
559 // out-of-order execution).
560 return 2;
561}
562
563// Returns a cost adjustment factor to adjust the cost of vector instructions
564// on targets which there is overlap between the vector and scalar units,
565// thereby reducing the overall throughput of vector code wrt. scalar code.
566// An invalid instruction cost is returned if the type is an MMA vector type.
567InstructionCost PPCTTIImpl::vectorCostAdjustmentFactor(unsigned Opcode,
568 Type *Ty1,
569 Type *Ty2) const {
570 // If the vector type is of an MMA type (v256i1, v512i1), an invalid
571 // instruction cost is returned. This is to signify to other cost computing
572 // functions to return the maximum instruction cost in order to prevent any
573 // opportunities for the optimizer to produce MMA types within the IR.
574 if (isMMAType(Ty: Ty1))
575 return InstructionCost::getInvalid();
576
577 if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy())
578 return InstructionCost(1);
579
580 std::pair<InstructionCost, MVT> LT1 = getTypeLegalizationCost(Ty: Ty1);
581 // If type legalization involves splitting the vector, we don't want to
582 // double the cost at every step - only the last step.
583 if (LT1.first != 1 || !LT1.second.isVector())
584 return InstructionCost(1);
585
586 int ISD = TLI->InstructionOpcodeToISD(Opcode);
587 if (TLI->isOperationExpand(Op: ISD, VT: LT1.second))
588 return InstructionCost(1);
589
590 if (Ty2) {
591 std::pair<InstructionCost, MVT> LT2 = getTypeLegalizationCost(Ty: Ty2);
592 if (LT2.first != 1 || !LT2.second.isVector())
593 return InstructionCost(1);
594 }
595
596 return InstructionCost(2);
597}
598
599InstructionCost PPCTTIImpl::getArithmeticInstrCost(
600 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
601 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
602 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
603 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
604
605 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty1: Ty, Ty2: nullptr);
606 if (!CostFactor.isValid())
607 return InstructionCost::getMax();
608
609 // TODO: Handle more cost kinds.
610 if (CostKind != TTI::TCK_RecipThroughput)
611 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
612 Opd2Info: Op2Info, Args, CtxI);
613
614 // Fallback to the default implementation.
615 InstructionCost Cost = BaseT::getArithmeticInstrCost(
616 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
617 return Cost * CostFactor;
618}
619
620InstructionCost PPCTTIImpl::getShuffleCost(
621 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
622 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
623 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
624 TTI::VectorInstrContext VIC) const {
625
626 InstructionCost CostFactor =
627 vectorCostAdjustmentFactor(Opcode: Instruction::ShuffleVector, Ty1: SrcTy, Ty2: nullptr);
628 if (!CostFactor.isValid())
629 return InstructionCost::getMax();
630
631 // Legalize the type.
632 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
633
634 // PPC, for both Altivec/VSX, support cheap arbitrary permutations
635 // (at least in the sense that there need only be one non-loop-invariant
636 // instruction). We need one such shuffle instruction for each actual
637 // register (this is not true for arbitrary shuffles, but is true for the
638 // structured types of shuffles covered by TTI::ShuffleKind).
639 return LT.first * CostFactor;
640}
641
642InstructionCost PPCTTIImpl::getCFInstrCost(unsigned Opcode,
643 TTI::TargetCostKind CostKind,
644 const Instruction *I) const {
645 if (CostKind != TTI::TCK_RecipThroughput)
646 return Opcode == Instruction::PHI ? 0 : 1;
647 // Branches are assumed to be predicted.
648 return 0;
649}
650
651InstructionCost PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
652 Type *Src,
653 TTI::CastContextHint CCH,
654 TTI::TargetCostKind CostKind,
655 const Instruction *I) const {
656 assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
657
658 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty1: Dst, Ty2: Src);
659 if (!CostFactor.isValid())
660 return InstructionCost::getMax();
661
662 InstructionCost Cost =
663 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
664 Cost *= CostFactor;
665 // TODO: Allow non-throughput costs that aren't binary.
666 if (CostKind != TTI::TCK_RecipThroughput)
667 return Cost == 0 ? 0 : 1;
668 return Cost;
669}
670
671InstructionCost PPCTTIImpl::getCmpSelInstrCost(
672 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
673 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
674 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
675 InstructionCost CostFactor =
676 vectorCostAdjustmentFactor(Opcode, Ty1: ValTy, Ty2: nullptr);
677 if (!CostFactor.isValid())
678 return InstructionCost::getMax();
679
680 InstructionCost Cost = BaseT::getCmpSelInstrCost(
681 Opcode, ValTy, CondTy, VecPred, CostKind, Op1Info, Op2Info, I);
682 // TODO: Handle other cost kinds.
683 if (CostKind != TTI::TCK_RecipThroughput)
684 return Cost;
685 return Cost * CostFactor;
686}
687
688InstructionCost PPCTTIImpl::getVectorInstrCost(
689 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
690 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
691 assert(Val->isVectorTy() && "This must be a vector type");
692
693 int ISD = TLI->InstructionOpcodeToISD(Opcode);
694 assert(ISD && "Invalid opcode");
695
696 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty1: Val, Ty2: nullptr);
697 if (!CostFactor.isValid())
698 return InstructionCost::getMax();
699
700 InstructionCost Cost =
701 BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1, VIC);
702 Cost *= CostFactor;
703
704 if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) {
705 // Double-precision scalars are already located in index #0 (or #1 if LE).
706 if (ISD == ISD::EXTRACT_VECTOR_ELT &&
707 Index == (ST->isLittleEndian() ? 1 : 0))
708 return 0;
709
710 return Cost;
711 }
712 if (Val->getScalarType()->isIntegerTy()) {
713 unsigned EltSize = Val->getScalarSizeInBits();
714 // Computing on 1 bit values requires extra mask or compare operations.
715 unsigned MaskCostForOneBitSize = (VecMaskCost && EltSize == 1) ? 1 : 0;
716 // Computing on non const index requires extra mask or compare operations.
717 unsigned MaskCostForIdx = (Index != -1U) ? 0 : 1;
718 if (ST->hasP9Altivec()) {
719 // P10 has vxform insert which can handle non const index. The
720 // MaskCostForIdx is for masking the index.
721 // P9 has insert for const index. A move-to VSR and a permute/insert.
722 // Assume vector operation cost for both (cost will be 2x on P9).
723 if (ISD == ISD::INSERT_VECTOR_ELT) {
724 if (ST->hasP10Vector())
725 return CostFactor + MaskCostForIdx;
726 if (Index != -1U)
727 return 2 * CostFactor;
728 } else if (ISD == ISD::EXTRACT_VECTOR_ELT) {
729 // It's an extract. Maybe we can do a cheap move-from VSR.
730 unsigned EltSize = Val->getScalarSizeInBits();
731 // P9 has both mfvsrd and mfvsrld for 64 bit integer.
732 if (EltSize == 64 && Index != -1U)
733 return 1;
734 if (EltSize == 32) {
735 unsigned MfvsrwzIndex = ST->isLittleEndian() ? 2 : 1;
736 if (Index == MfvsrwzIndex)
737 return 1;
738
739 // For other indexs like non const, P9 has vxform extract. The
740 // MaskCostForIdx is for masking the index.
741 return CostFactor + MaskCostForIdx;
742 }
743
744 // We need a vector extract (or mfvsrld). Assume vector operation cost.
745 // The cost of the load constant for a vector extract is disregarded
746 // (invariant, easily schedulable).
747 return CostFactor + MaskCostForOneBitSize + MaskCostForIdx;
748 }
749 } else if (ST->hasDirectMove() && Index != -1U) {
750 // Assume permute has standard cost.
751 // Assume move-to/move-from VSR have 2x standard cost.
752 if (ISD == ISD::INSERT_VECTOR_ELT)
753 return 3;
754 return 3 + MaskCostForOneBitSize;
755 }
756 }
757
758 // Estimated cost of a load-hit-store delay. This was obtained
759 // experimentally as a minimum needed to prevent unprofitable
760 // vectorization for the paq8p benchmark. It may need to be
761 // raised further if other unprofitable cases remain.
762 unsigned LHSPenalty = 2;
763 if (ISD == ISD::INSERT_VECTOR_ELT)
764 LHSPenalty += 7;
765
766 // Vector element insert/extract with Altivec is very expensive,
767 // because they require store and reload with the attendant
768 // processor stall for load-hit-store. Until VSX is available,
769 // these need to be estimated as very costly.
770 if (ISD == ISD::EXTRACT_VECTOR_ELT ||
771 ISD == ISD::INSERT_VECTOR_ELT)
772 return LHSPenalty + Cost;
773
774 return Cost;
775}
776
777InstructionCost PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
778 Align Alignment,
779 unsigned AddressSpace,
780 TTI::TargetCostKind CostKind,
781 TTI::OperandValueInfo OpInfo,
782 const Instruction *I) const {
783 InstructionCost CostFactor = vectorCostAdjustmentFactor(Opcode, Ty1: Src, Ty2: nullptr);
784 if (!CostFactor.isValid())
785 return InstructionCost::getMax();
786
787 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
788 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
789 CostKind);
790 // Legalize the type.
791 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
792 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
793 "Invalid Opcode");
794
795 InstructionCost Cost =
796 BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
797 // TODO: Handle other cost kinds.
798 if (CostKind != TTI::TCK_RecipThroughput)
799 return Cost;
800
801 Cost *= CostFactor;
802
803 bool IsAltivecType = ST->hasAltivec() &&
804 (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 ||
805 LT.second == MVT::v4i32 || LT.second == MVT::v4f32);
806 bool IsVSXType = ST->hasVSX() &&
807 (LT.second == MVT::v2f64 || LT.second == MVT::v2i64);
808
809 // VSX has 32b/64b load instructions. Legalization can handle loading of
810 // 32b/64b to VSR correctly and cheaply. But BaseT::getMemoryOpCost and
811 // PPCTargetLowering can't compute the cost appropriately. So here we
812 // explicitly check this case. There are also corresponding store
813 // instructions.
814 unsigned MemBits = Src->getPrimitiveSizeInBits();
815 unsigned SrcBytes = LT.second.getStoreSize();
816 if (ST->hasVSX() && IsAltivecType) {
817 if (MemBits == 64 || (ST->hasP8Vector() && MemBits == 32))
818 return 1;
819
820 // Use lfiwax/xxspltw
821 if (Opcode == Instruction::Load && MemBits == 32 && Alignment < SrcBytes)
822 return 2;
823 }
824
825 // Aligned loads and stores are easy.
826 if (!SrcBytes || Alignment >= SrcBytes)
827 return Cost;
828
829 // If we can use the permutation-based load sequence, then this is also
830 // relatively cheap (not counting loop-invariant instructions): one load plus
831 // one permute (the last load in a series has extra cost, but we're
832 // neglecting that here). Note that on the P7, we could do unaligned loads
833 // for Altivec types using the VSX instructions, but that's more expensive
834 // than using the permutation-based load sequence. On the P8, that's no
835 // longer true.
836 if (Opcode == Instruction::Load && (!ST->hasP8Vector() && IsAltivecType) &&
837 Alignment >= LT.second.getScalarType().getStoreSize())
838 return Cost + LT.first; // Add the cost of the permutations.
839
840 // For VSX, we can do unaligned loads and stores on Altivec/VSX types. On the
841 // P7, unaligned vector loads are more expensive than the permutation-based
842 // load sequence, so that might be used instead, but regardless, the net cost
843 // is about the same (not counting loop-invariant instructions).
844 if (IsVSXType || (ST->hasVSX() && IsAltivecType))
845 return Cost;
846
847 // Newer PPC supports unaligned memory access.
848 if (TLI->allowsMisalignedMemoryAccesses(VT: LT.second, AddrSpace: 0))
849 return Cost;
850
851 // PPC in general does not support unaligned loads and stores. They'll need
852 // to be decomposed based on the alignment factor.
853
854 // Add the cost of each scalar load or store.
855 Cost += LT.first * ((SrcBytes / Alignment.value()) - 1);
856
857 // For a vector type, there is also scalarization overhead (only for
858 // stores, loads are expanded using the vector-load + permutation sequence,
859 // which is much less expensive).
860 if (Src->isVectorTy() && Opcode == Instruction::Store)
861 for (int I = 0, E = cast<FixedVectorType>(Val: Src)->getNumElements(); I < E;
862 ++I)
863 Cost +=
864 getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Src, CostKind, Index: I,
865 Op0: nullptr, Op1: nullptr, VIC: TTI::VectorInstrContext::None);
866
867 return Cost;
868}
869
870InstructionCost PPCTTIImpl::getInterleavedMemoryOpCost(
871 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
872 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
873 bool UseMaskForCond, bool UseMaskForGaps) const {
874 InstructionCost CostFactor =
875 vectorCostAdjustmentFactor(Opcode, Ty1: VecTy, Ty2: nullptr);
876 if (!CostFactor.isValid())
877 return InstructionCost::getMax();
878
879 if (UseMaskForCond || UseMaskForGaps)
880 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
881 Alignment, AddressSpace, CostKind,
882 UseMaskForCond, UseMaskForGaps);
883
884 assert(isa<VectorType>(VecTy) &&
885 "Expect a vector type for interleaved memory op");
886
887 // Legalize the type.
888 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VecTy);
889
890 // Firstly, the cost of load/store operation.
891 InstructionCost Cost =
892 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
893
894 // PPC, for both Altivec/VSX, support cheap arbitrary permutations
895 // (at least in the sense that there need only be one non-loop-invariant
896 // instruction). For each result vector, we need one shuffle per incoming
897 // vector (except that the first shuffle can take two incoming vectors
898 // because it does not need to take itself).
899 Cost += Factor*(LT.first-1);
900
901 return Cost;
902}
903
904InstructionCost
905PPCTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
906 TTI::TargetCostKind CostKind) const {
907
908 if (!VPIntrinsic::isVPIntrinsic(ICA.getID()))
909 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
910
911 if (ICA.getID() == Intrinsic::vp_load) {
912 MemIntrinsicCostAttributes MICA(Intrinsic::masked_load, ICA.getReturnType(),
913 Align(1), 0);
914 return getMemIntrinsicInstrCost(MICA, CostKind);
915 }
916
917 if (ICA.getID() == Intrinsic::vp_store) {
918 MemIntrinsicCostAttributes MICA(Intrinsic::masked_store,
919 ICA.getArgTypes()[0], Align(1), 0);
920 return getMemIntrinsicInstrCost(MICA, CostKind);
921 }
922
923 return InstructionCost::getInvalid();
924}
925
926bool PPCTTIImpl::areTypesABICompatible(const Function *Caller,
927 const Function *Callee,
928 ArrayRef<Type *> Types) const {
929
930 // We need to ensure that argument promotion does not
931 // attempt to promote pointers to MMA types (__vector_pair
932 // and __vector_quad) since these types explicitly cannot be
933 // passed as arguments. Both of these types are larger than
934 // the 128-bit Altivec vectors and have a scalar size of 1 bit.
935 if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
936 return false;
937
938 return llvm::none_of(Range&: Types, P: [](Type *Ty) {
939 if (Ty->isSized())
940 return Ty->isIntOrIntVectorTy(BitWidth: 1) && Ty->getPrimitiveSizeInBits() > 128;
941 return false;
942 });
943}
944
945bool PPCTTIImpl::canSaveCmp(Loop *L, CondBrInst **BI, ScalarEvolution *SE,
946 LoopInfo *LI, DominatorTree *DT,
947 AssumptionCache *AC,
948 TargetLibraryInfo *LibInfo) const {
949 // Process nested loops first.
950 for (Loop *I : *L)
951 if (canSaveCmp(L: I, BI, SE, LI, DT, AC, LibInfo))
952 return false; // Stop search.
953
954 HardwareLoopInfo HWLoopInfo(L);
955
956 if (!HWLoopInfo.canAnalyze(LI&: *LI))
957 return false;
958
959 if (!isHardwareLoopProfitable(L, SE&: *SE, AC&: *AC, LibInfo, HWLoopInfo))
960 return false;
961
962 if (!HWLoopInfo.isHardwareLoopCandidate(SE&: *SE, LI&: *LI, DT&: *DT))
963 return false;
964
965 *BI = HWLoopInfo.ExitBranch;
966 return true;
967}
968
969bool PPCTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
970 const TargetTransformInfo::LSRCost &C2) const {
971 // PowerPC default behaviour here is "instruction number 1st priority".
972 // If LsrNoInsnsCost is set, call default implementation.
973 if (!LsrNoInsnsCost)
974 return std::tie(args: C1.Insns, args: C1.NumRegs, args: C1.AddRecCost, args: C1.NumIVMuls,
975 args: C1.NumBaseAdds, args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
976 std::tie(args: C2.Insns, args: C2.NumRegs, args: C2.AddRecCost, args: C2.NumIVMuls,
977 args: C2.NumBaseAdds, args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
978 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
979}
980
981bool PPCTTIImpl::isNumRegsMajorCostOfLSR() const { return false; }
982
983bool PPCTTIImpl::shouldBuildRelLookupTables() const {
984 // XCOFF hasn't implemented lowerRelativeReference, disable non-ELF for now.
985 if (!ST->isELFv2ABI())
986 return false;
987 return BaseT::shouldBuildRelLookupTables();
988}
989
990bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
991 MemIntrinsicInfo &Info) const {
992 switch (Inst->getIntrinsicID()) {
993 case Intrinsic::ppc_altivec_lvx:
994 case Intrinsic::ppc_altivec_lvxl:
995 case Intrinsic::ppc_altivec_lvebx:
996 case Intrinsic::ppc_altivec_lvehx:
997 case Intrinsic::ppc_altivec_lvewx:
998 case Intrinsic::ppc_vsx_lxvd2x:
999 case Intrinsic::ppc_vsx_lxvw4x:
1000 case Intrinsic::ppc_vsx_lxvd2x_be:
1001 case Intrinsic::ppc_vsx_lxvw4x_be:
1002 case Intrinsic::ppc_vsx_lxvl:
1003 case Intrinsic::ppc_vsx_lxvll:
1004 case Intrinsic::ppc_vsx_lxvp: {
1005 Info.PtrVal = Inst->getArgOperand(i: 0);
1006 Info.ReadMem = true;
1007 Info.WriteMem = false;
1008 return true;
1009 }
1010 case Intrinsic::ppc_altivec_stvx:
1011 case Intrinsic::ppc_altivec_stvxl:
1012 case Intrinsic::ppc_altivec_stvebx:
1013 case Intrinsic::ppc_altivec_stvehx:
1014 case Intrinsic::ppc_altivec_stvewx:
1015 case Intrinsic::ppc_vsx_stxvd2x:
1016 case Intrinsic::ppc_vsx_stxvw4x:
1017 case Intrinsic::ppc_vsx_stxvd2x_be:
1018 case Intrinsic::ppc_vsx_stxvw4x_be:
1019 case Intrinsic::ppc_vsx_stxvl:
1020 case Intrinsic::ppc_vsx_stxvll:
1021 case Intrinsic::ppc_vsx_stxvp: {
1022 Info.PtrVal = Inst->getArgOperand(i: 1);
1023 Info.ReadMem = false;
1024 Info.WriteMem = true;
1025 return true;
1026 }
1027 case Intrinsic::ppc_stbcx:
1028 case Intrinsic::ppc_sthcx:
1029 case Intrinsic::ppc_stdcx:
1030 case Intrinsic::ppc_stwcx: {
1031 Info.PtrVal = Inst->getArgOperand(i: 0);
1032 Info.ReadMem = false;
1033 Info.WriteMem = true;
1034 return true;
1035 }
1036 default:
1037 break;
1038 }
1039
1040 return false;
1041}
1042
1043bool PPCTTIImpl::supportsTailCallFor(const CallBase *CB) const {
1044 return TLI->supportsTailCallFor(CB);
1045}
1046
1047// Target hook used by CodeGen to decide whether to expand vector predication
1048// intrinsics into scalar operations or to use special ISD nodes to represent
1049// them. The Target will not see the intrinsics.
1050TargetTransformInfo::VPLegalization
1051PPCTTIImpl::getVPLegalizationStrategy(const VPIntrinsic &PI) const {
1052 using VPLegalization = TargetTransformInfo::VPLegalization;
1053 unsigned Directive = ST->getCPUDirective();
1054 VPLegalization DefaultLegalization = BaseT::getVPLegalizationStrategy(PI);
1055 if (Directive != PPC::DIR_PWR10 && Directive != PPC::DIR_PWR_FUTURE &&
1056 (!Pwr9EVL || Directive != PPC::DIR_PWR9))
1057 return DefaultLegalization;
1058
1059 if (!ST->isPPC64())
1060 return DefaultLegalization;
1061
1062 unsigned IID = PI.getIntrinsicID();
1063 if (IID != Intrinsic::vp_load && IID != Intrinsic::vp_store)
1064 return DefaultLegalization;
1065
1066 bool IsLoad = IID == Intrinsic::vp_load;
1067 Type *VecTy = IsLoad ? PI.getType() : PI.getOperand(i_nocapture: 0)->getType();
1068 EVT VT = TLI->getValueType(DL, Ty: VecTy, AllowUnknown: true);
1069 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 &&
1070 VT != MVT::v16i8)
1071 return DefaultLegalization;
1072
1073 auto IsAllTrueMask = [](Value *MaskVal) {
1074 if (Value *SplattedVal = getSplatValue(V: MaskVal))
1075 if (auto *ConstValue = dyn_cast<Constant>(Val: SplattedVal))
1076 return ConstValue->isAllOnesValue();
1077 return false;
1078 };
1079 unsigned MaskIx = IsLoad ? 1 : 2;
1080 if (!IsAllTrueMask(PI.getOperand(i_nocapture: MaskIx)))
1081 return DefaultLegalization;
1082
1083 return VPLegalization(VPLegalization::Legal, VPLegalization::Legal);
1084}
1085
1086bool PPCTTIImpl::hasActiveVectorLength() const {
1087 if (!PPCEVL || !ST->isPPC64())
1088 return false;
1089 unsigned CPU = ST->getCPUDirective();
1090 return CPU == PPC::DIR_PWR10 || CPU == PPC::DIR_PWR_FUTURE ||
1091 (Pwr9EVL && CPU == PPC::DIR_PWR9);
1092}
1093
1094bool PPCTTIImpl::isLegalMaskedLoad(Type *DataType, Align Alignment,
1095 unsigned AddressSpace,
1096 TTI::MaskKind MaskKind) const {
1097 if (!hasActiveVectorLength())
1098 return false;
1099
1100 auto IsLegalLoadWithLengthType = [](EVT VT) {
1101 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16 && VT != MVT::i8)
1102 return false;
1103 return true;
1104 };
1105
1106 return IsLegalLoadWithLengthType(TLI->getValueType(DL, Ty: DataType, AllowUnknown: true));
1107}
1108
1109bool PPCTTIImpl::isLegalMaskedStore(Type *DataType, Align Alignment,
1110 unsigned AddressSpace,
1111 TTI::MaskKind MaskKind) const {
1112 return isLegalMaskedLoad(DataType, Alignment, AddressSpace);
1113}
1114
1115InstructionCost
1116PPCTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1117 TTI::TargetCostKind CostKind) const {
1118
1119 InstructionCost BaseCost = BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1120
1121 unsigned Opcode;
1122 switch (MICA.getID()) {
1123 case Intrinsic::masked_load:
1124 Opcode = Instruction::Load;
1125 break;
1126 case Intrinsic::masked_store:
1127 Opcode = Instruction::Store;
1128 break;
1129 default:
1130 return BaseCost;
1131 }
1132
1133 Type *DataTy = MICA.getDataType();
1134 Align Alignment = MICA.getAlignment();
1135 unsigned AddressSpace = MICA.getAddressSpace();
1136
1137 auto VecTy = dyn_cast<FixedVectorType>(Val: DataTy);
1138 if (!VecTy)
1139 return BaseCost;
1140 if (Opcode == Instruction::Load) {
1141 if (!isLegalMaskedLoad(DataType: VecTy->getScalarType(), Alignment, AddressSpace))
1142 return BaseCost;
1143 } else {
1144 if (!isLegalMaskedStore(DataType: VecTy->getScalarType(), Alignment, AddressSpace))
1145 return BaseCost;
1146 }
1147 if (VecTy->getPrimitiveSizeInBits() > 128)
1148 return BaseCost;
1149
1150 // Cost is 1 (scalar compare) + 1 (scalar select) +
1151 // 1 * vectorCostAdjustmentFactor (vector load with length)
1152 // Maybe + 1 (scalar shift)
1153 InstructionCost Cost =
1154 1 + 1 + vectorCostAdjustmentFactor(Opcode, Ty1: DataTy, Ty2: nullptr);
1155 if (ST->getCPUDirective() != PPC::DIR_PWR_FUTURE ||
1156 VecTy->getScalarSizeInBits() != 8)
1157 Cost += 1; // need shift for length
1158 return Cost;
1159}
1160
1161InstructionCost PPCTTIImpl::getPartialReductionCost(
1162 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
1163 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
1164 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
1165 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
1166 InstructionCost Invalid = InstructionCost::getInvalid();
1167
1168 if (!getST()->hasAltivec())
1169 return Invalid;
1170 if (Opcode != Instruction::Add)
1171 return Invalid;
1172
1173 EVT AccVT = TLI->getValueType(DL, Ty: AccumType, AllowUnknown: true);
1174 if (AccVT != MVT::i32)
1175 return Invalid;
1176
1177 Type *ATy = VectorType::get(ElementType: InputTypeA, EC: VF);
1178 EVT AVT = TLI->getValueType(DL, Ty: ATy, AllowUnknown: true);
1179
1180 if (OpAExtend != TTI::PR_SignExtend && OpAExtend != TTI::PR_ZeroExtend)
1181 return Invalid;
1182
1183 if (!BinOp) {
1184 // It's just an add, so only the LHS type matters. RHS is just 1s.
1185 if (AVT != MVT::v16i8 && AVT != MVT::v8i16)
1186 return Invalid;
1187 // We can do either (sext/-) or (zext/-) for v16i8 and v8i16
1188 return vectorCostAdjustmentFactor(Opcode: Instruction::Add, Ty1: ATy, Ty2: nullptr);
1189 }
1190
1191 if (BinOp.value() != Instruction::Mul)
1192 return Invalid;
1193 if (InputTypeA != InputTypeB)
1194 return Invalid;
1195 if (OpBExtend != TTI::PR_SignExtend && OpBExtend != TTI::PR_ZeroExtend)
1196 return Invalid;
1197
1198 if (AVT == MVT::v16i8) {
1199 // For v16i8 PPC supports vmsumubm (zext/zext) and vmsummbm (sext/zext)
1200 if (OpBExtend != TTI::PR_ZeroExtend)
1201 return Invalid;
1202 } else if (AVT == MVT::v8i16) {
1203 // For v8i16 PPC supports vmsumuhm (zext/zext) and vmsumshm (sext/sext)
1204 if (OpAExtend != OpBExtend)
1205 return Invalid;
1206 } else {
1207 return Invalid;
1208 }
1209
1210 return vectorCostAdjustmentFactor(Opcode: Instruction::Add, Ty1: ATy, Ty2: nullptr);
1211}
1212