1//===-- X86TargetTransformInfo.cpp - X86 specific TTI pass ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements a TargetTransformInfo analysis pass specific to the
10/// X86 target machine. It uses the target's detailed information to provide
11/// more precise answers to certain TTI queries, while letting the target
12/// independent and default TTI implementations handle the rest.
13///
14//===----------------------------------------------------------------------===//
15/// About Cost Model numbers used below it's necessary to say the following:
16/// the numbers correspond to some "generic" X86 CPU instead of usage of a
17/// specific CPU model. Usually the numbers correspond to the CPU where the
18/// feature first appeared. For example, if we do Subtarget.hasSSE42() in
19/// the lookups below the cost is based on Nehalem as that was the first CPU
20/// to support that feature level and thus has most likely the worst case cost,
21/// although we may discard an outlying worst cost from one CPU (e.g. Atom).
22///
23/// Some examples of other technologies/CPUs:
24/// SSE 3 - Pentium4 / Athlon64
25/// SSE 4.1 - Penryn
26/// SSE 4.2 - Nehalem / Silvermont
27/// AVX - Sandy Bridge / Jaguar / Bulldozer
28/// AVX2 - Haswell / Ryzen
29/// AVX-512 - Xeon Phi / Skylake
30///
31/// And some examples of instruction target dependent costs (latency)
32/// divss sqrtss rsqrtss
33/// AMD K7 11-16 19 3
34/// Piledriver 9-24 13-15 5
35/// Jaguar 14 16 2
36/// Pentium II,III 18 30 2
37/// Nehalem 7-14 7-18 3
38/// Haswell 10-13 11 5
39///
40/// Interpreting the 4 TargetCostKind types:
41/// TCK_RecipThroughput and TCK_Latency should try to match the worst case
42/// values reported by the CPU scheduler models (and llvm-mca).
43/// TCK_CodeSize should match the instruction count (e.g. divss = 1), NOT the
44/// actual encoding size of the instruction.
45/// TCK_SizeAndLatency should match the worst case micro-op counts reported by
46/// by the CPU scheduler models (and llvm-mca), to ensure that they are
47/// compatible with the MicroOpBufferSize and LoopMicroOpBufferSize values which are
48/// often used as the cost thresholds where TCK_SizeAndLatency is requested.
49//===----------------------------------------------------------------------===//
50
51#include "X86TargetTransformInfo.h"
52#include "llvm/ADT/SmallBitVector.h"
53#include "llvm/Analysis/TargetTransformInfo.h"
54#include "llvm/CodeGen/Analysis.h"
55#include "llvm/CodeGen/BasicTTIImpl.h"
56#include "llvm/CodeGen/CostTable.h"
57#include "llvm/CodeGen/TargetLowering.h"
58#include "llvm/IR/InstIterator.h"
59#include "llvm/IR/IntrinsicInst.h"
60#include <optional>
61
62using namespace llvm;
63
64#define DEBUG_TYPE "x86tti"
65
66//===----------------------------------------------------------------------===//
67//
68// X86 cost model.
69//
70//===----------------------------------------------------------------------===//
71
72// Helper struct to store/access costs for each cost kind.
73// TODO: Move this to allow other targets to use it?
74namespace {
75struct CostKindCosts {
76 unsigned RecipThroughputCost = ~0U;
77 unsigned LatencyCost = ~0U;
78 unsigned CodeSizeCost = ~0U;
79 unsigned SizeAndLatencyCost = ~0U;
80
81 std::optional<unsigned>
82 operator[](TargetTransformInfo::TargetCostKind Kind) const {
83 unsigned Cost = ~0U;
84 switch (Kind) {
85 case TargetTransformInfo::TCK_RecipThroughput:
86 Cost = RecipThroughputCost;
87 break;
88 case TargetTransformInfo::TCK_Latency:
89 Cost = LatencyCost;
90 break;
91 case TargetTransformInfo::TCK_CodeSize:
92 Cost = CodeSizeCost;
93 break;
94 case TargetTransformInfo::TCK_SizeAndLatency:
95 Cost = SizeAndLatencyCost;
96 break;
97 }
98 if (Cost == ~0U)
99 return std::nullopt;
100 return Cost;
101 }
102};
103} // namespace
104using CostKindTblEntry = CostTblEntryT<CostKindCosts>;
105using TypeConversionCostKindTblEntry = TypeConversionCostTblEntryT<CostKindCosts>;
106
107TargetTransformInfo::PopcntSupportKind
108X86TTIImpl::getPopcntSupport(unsigned TyWidth) const {
109 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
110 // TODO: Currently the __builtin_popcount() implementation using SSE3
111 // instructions is inefficient. Once the problem is fixed, we should
112 // call ST->hasSSE3() instead of ST->hasPOPCNT().
113 return ST->hasPOPCNT() ? TTI::PSK_FastHardware : TTI::PSK_Software;
114}
115
116std::optional<unsigned> X86TTIImpl::getCacheSize(
117 TargetTransformInfo::CacheLevel Level) const {
118 switch (Level) {
119 case TargetTransformInfo::CacheLevel::L1D:
120 // - Penryn
121 // - Nehalem
122 // - Westmere
123 // - Sandy Bridge
124 // - Ivy Bridge
125 // - Haswell
126 // - Broadwell
127 // - Skylake
128 // - Kabylake
129 return 32 * 1024; // 32 KiB
130 case TargetTransformInfo::CacheLevel::L2D:
131 // - Penryn
132 // - Nehalem
133 // - Westmere
134 // - Sandy Bridge
135 // - Ivy Bridge
136 // - Haswell
137 // - Broadwell
138 // - Skylake
139 // - Kabylake
140 return 256 * 1024; // 256 KiB
141 }
142
143 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
144}
145
146std::optional<unsigned> X86TTIImpl::getCacheAssociativity(
147 TargetTransformInfo::CacheLevel Level) const {
148 // - Penryn
149 // - Nehalem
150 // - Westmere
151 // - Sandy Bridge
152 // - Ivy Bridge
153 // - Haswell
154 // - Broadwell
155 // - Skylake
156 // - Kabylake
157 switch (Level) {
158 case TargetTransformInfo::CacheLevel::L1D:
159 [[fallthrough]];
160 case TargetTransformInfo::CacheLevel::L2D:
161 return 8;
162 }
163
164 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
165}
166
167enum ClassIDEnum { GPRClass = 0, VectorClass = 1, ScalarFPClass = 2 };
168
169unsigned X86TTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const {
170 return Vector ? VectorClass
171 : Ty && Ty->isFloatingPointTy() ? ScalarFPClass
172 : GPRClass;
173}
174
175unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
176 if (ClassID == VectorClass && !ST->hasSSE1())
177 return 0;
178
179 if (!ST->is64Bit())
180 return 8;
181
182 if ((ClassID == GPRClass && ST->hasEGPR()) ||
183 (ClassID != GPRClass && ST->hasAVX512()))
184 return 32;
185
186 return 16;
187}
188
189bool X86TTIImpl::hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const {
190 if (!ST->hasCF())
191 return false;
192 if (!Ty)
193 return true;
194 // Conditional faulting is supported by CFCMOV, which only accepts
195 // 16/32/64-bit operands.
196 // TODO: Support f32/f64 with VMOVSS/VMOVSD with zero mask when it's
197 // profitable.
198 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
199 if (!Ty->isIntegerTy() && (!VTy || VTy->getNumElements() != 1))
200 return false;
201 auto *ScalarTy = Ty->getScalarType();
202 switch (cast<IntegerType>(Val: ScalarTy)->getBitWidth()) {
203 default:
204 return false;
205 case 16:
206 case 32:
207 case 64:
208 return true;
209 }
210}
211
212TypeSize
213X86TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
214 unsigned PreferVectorWidth = ST->getPreferVectorWidth();
215 switch (K) {
216 case TargetTransformInfo::RGK_Scalar:
217 return TypeSize::getFixed(ExactSize: ST->is64Bit() ? 64 : 32);
218 case TargetTransformInfo::RGK_FixedWidthVector:
219 if (ST->hasAVX512() && PreferVectorWidth >= 512)
220 return TypeSize::getFixed(ExactSize: 512);
221 if (ST->hasAVX() && PreferVectorWidth >= 256)
222 return TypeSize::getFixed(ExactSize: 256);
223 if (ST->hasSSE1() && PreferVectorWidth >= 128)
224 return TypeSize::getFixed(ExactSize: 128);
225 return TypeSize::getFixed(ExactSize: 0);
226 case TargetTransformInfo::RGK_ScalableVector:
227 return TypeSize::getScalable(MinimumSize: 0);
228 }
229
230 llvm_unreachable("Unsupported register kind");
231}
232
233unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
234 return getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector)
235 .getFixedValue();
236}
237
238unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF,
239 bool HasUnorderedReductions) const {
240 // If the loop will not be vectorized, don't interleave the loop.
241 // Let regular unroll to unroll the loop, which saves the overflow
242 // check and memory check cost.
243 if (VF.isScalar())
244 return 1;
245
246 if (ST->isAtom())
247 return 1;
248
249 // Sandybridge and Haswell have multiple execution ports and pipelined
250 // vector units.
251 if (ST->hasAVX())
252 return 4;
253
254 return 2;
255}
256
257InstructionCost X86TTIImpl::getArithmeticInstrCost(
258 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
259 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
260 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
261
262 // vXi8 multiplications are always promoted to vXi16.
263 // Sub-128-bit types can be extended/packed more efficiently.
264 if (Opcode == Instruction::Mul && Ty->isVectorTy() &&
265 Ty->getPrimitiveSizeInBits() <= 64 && Ty->getScalarSizeInBits() == 8) {
266 Type *WideVecTy =
267 VectorType::getExtendedElementVectorType(VTy: cast<VectorType>(Val: Ty));
268 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideVecTy, Src: Ty,
269 CCH: TargetTransformInfo::CastContextHint::None,
270 CostKind) +
271 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
272 CCH: TargetTransformInfo::CastContextHint::None,
273 CostKind) +
274 getArithmeticInstrCost(Opcode, Ty: WideVecTy, CostKind, Op1Info, Op2Info);
275 }
276
277 // Legalize the type.
278 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
279
280 int ISD = TLI->InstructionOpcodeToISD(Opcode);
281 assert(ISD && "Invalid opcode");
282
283 if (ISD == ISD::MUL && Args.size() == 2 && LT.second.isVector() &&
284 (LT.second.getScalarType() == MVT::i32 ||
285 LT.second.getScalarType() == MVT::i64)) {
286 // Check if the operands can be represented as a smaller datatype.
287 bool Op1Signed = false, Op2Signed = false;
288 unsigned Op1MinSize = BaseT::minRequiredElementSize(Val: Args[0], isSigned&: Op1Signed);
289 unsigned Op2MinSize = BaseT::minRequiredElementSize(Val: Args[1], isSigned&: Op2Signed);
290 unsigned OpMinSize = std::max(a: Op1MinSize, b: Op2MinSize);
291 bool SignedMode = Op1Signed || Op2Signed;
292
293 // If both vXi32 are representable as i15 and at least one is constant,
294 // zero-extended, or sign-extended from vXi16 (or less pre-SSE41) then we
295 // can treat this as PMADDWD which has the same costs as a vXi16 multiply.
296 if (OpMinSize <= 15 && !ST->isPMADDWDSlow() &&
297 LT.second.getScalarType() == MVT::i32) {
298 bool Op1Constant =
299 isa<ConstantDataVector>(Val: Args[0]) || isa<ConstantVector>(Val: Args[0]);
300 bool Op2Constant =
301 isa<ConstantDataVector>(Val: Args[1]) || isa<ConstantVector>(Val: Args[1]);
302 bool Op1Sext = isa<SExtInst>(Val: Args[0]) &&
303 (Op1MinSize == 15 || (Op1MinSize < 15 && !ST->hasSSE41()));
304 bool Op2Sext = isa<SExtInst>(Val: Args[1]) &&
305 (Op2MinSize == 15 || (Op2MinSize < 15 && !ST->hasSSE41()));
306
307 bool IsZeroExtended = !Op1Signed || !Op2Signed;
308 bool IsConstant = Op1Constant || Op2Constant;
309 bool IsSext = Op1Sext || Op2Sext;
310 if (IsConstant || IsZeroExtended || IsSext)
311 LT.second =
312 MVT::getVectorVT(VT: MVT::i16, NumElements: 2 * LT.second.getVectorNumElements());
313 }
314
315 // Check if the vXi32 operands can be shrunk into a smaller datatype.
316 // This should match the codegen from reduceVMULWidth.
317 // TODO: Make this generic (!ST->SSE41 || ST->isPMULLDSlow()).
318 if (ST->useSLMArithCosts() && LT.second == MVT::v4i32) {
319 if (OpMinSize <= 7)
320 return LT.first * 3; // pmullw/sext
321 if (!SignedMode && OpMinSize <= 8)
322 return LT.first * 3; // pmullw/zext
323 if (OpMinSize <= 15)
324 return LT.first * 5; // pmullw/pmulhw/pshuf
325 if (!SignedMode && OpMinSize <= 16)
326 return LT.first * 5; // pmullw/pmulhw/pshuf
327 }
328
329 // If both vXi64 are representable as (unsigned) i32, then we can perform
330 // the multiple with a single PMULUDQ instruction.
331 // TODO: Add (SSE41+) PMULDQ handling for signed extensions.
332 if (!SignedMode && OpMinSize <= 32 && LT.second.getScalarType() == MVT::i64)
333 ISD = X86ISD::PMULUDQ;
334 }
335
336 // Vector multiply by pow2 will be simplified to shifts.
337 // Vector multiply by -pow2 will be simplified to shifts/negates.
338 if (ISD == ISD::MUL && Op2Info.isConstant() &&
339 (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2())) {
340 InstructionCost Cost =
341 getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind,
342 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
343 if (Op2Info.isNegatedPowerOf2())
344 Cost += getArithmeticInstrCost(Opcode: Instruction::Sub, Ty, CostKind);
345 return Cost;
346 }
347
348 // On X86, vector signed division by constants power-of-two are
349 // normally expanded to the sequence SRA + SRL + ADD + SRA.
350 // The OperandValue properties may not be the same as that of the previous
351 // operation; conservatively assume OP_None.
352 if ((ISD == ISD::SDIV || ISD == ISD::SREM) &&
353 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
354 InstructionCost Cost =
355 2 * getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
356 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
357 Cost += getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
358 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
359 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
360 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
361
362 if (ISD == ISD::SREM) {
363 // For SREM: (X % C) is the equivalent of (X - (X/C)*C)
364 Cost += getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind, Op1Info: Op1Info.getNoProps(),
365 Op2Info: Op2Info.getNoProps());
366 Cost += getArithmeticInstrCost(Opcode: Instruction::Sub, Ty, CostKind, Op1Info: Op1Info.getNoProps(),
367 Op2Info: Op2Info.getNoProps());
368 }
369
370 return Cost;
371 }
372
373 // Vector unsigned division/remainder will be simplified to shifts/masks.
374 if ((ISD == ISD::UDIV || ISD == ISD::UREM) &&
375 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
376 if (ISD == ISD::UDIV)
377 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
378 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
379 // UREM
380 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind,
381 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
382 }
383
384 // A rem paired with a div on the same operands in the same block is free:
385 // the pair costs the div alone. Power-of-two divisors are handled above.
386 if ((ISD == ISD::UREM || ISD == ISD::SREM) && CostKind != TTI::TCK_Latency &&
387 CtxI && CtxI->getOpcode() == Opcode) {
388 using namespace PatternMatch;
389 // Same use limit as SLP, bounds the walk over the operand users.
390 constexpr unsigned MaxUses = 64;
391 unsigned DivOpcode =
392 ISD == ISD::SREM ? Instruction::SDiv : Instruction::UDiv;
393 const Value *X = CtxI->getOperand(i: 0);
394 const Value *Y = CtxI->getOperand(i: 1);
395 // Constants have context-wide use lists, walk the variable operand.
396 const Value *V = isa<Constant>(Val: Y) ? X : Y;
397 if (!isa<Constant>(Val: V) && !V->hasNUsesOrMore(N: MaxUses) &&
398 any_of(Range: V->users(), P: [&](const User *U) {
399 return match(V: U, P: m_BinOp(Opcode: DivOpcode, L: m_Specific(V: X), R: m_Specific(V: Y))) &&
400 cast<Instruction>(Val: U)->getParent() == CtxI->getParent();
401 }))
402 return TTI::TCC_Free;
403 }
404
405 // A scalar integer divide/remainder by a constant is not a hardware divide;
406 // it lowers to a magic-number multiply-high plus a few fixup ops. Cost it as
407 // that sequence rather than the generic single-instruction divide, so the
408 // vectorizers do not compare against an artificially cheap scalar lane. The
409 // power-of-two cases are handled above; negated powers of two are left to the
410 // generic handling.
411 if (!Ty->isVectorTy() && Op2Info.isConstant() && !Op2Info.isNegatedPowerOf2() &&
412 (ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
413 ISD == ISD::SREM)) {
414 unsigned Cost = ISD == ISD::UREM || ISD == ISD::SREM ? 6 : 5;
415 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
416 Cost += 2;
417 return LT.first * Cost;
418 }
419
420 static const CostKindTblEntry GFNIUniformConstCostTable[] = {
421 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
422 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
423 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
424 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
425 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
426 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
427 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
428 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
429 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
430 };
431
432 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasGFNI())
433 if (const auto *Entry =
434 CostTableLookup(Table: GFNIUniformConstCostTable, ISD, Ty: LT.second))
435 if (auto KindCost = Entry->Cost[CostKind])
436 return LT.first * *KindCost;
437
438 static const CostKindTblEntry AVX512BWUniformConstCostTable[] = {
439 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
440 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
441 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw, pand, pxor, psubb.
442 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
443 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
444 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw, pand, pxor, psubb.
445 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
446 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
447 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
448
449 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
450 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
451 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
452 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
453 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
454 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
455 };
456
457 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasBWI())
458 if (const auto *Entry =
459 CostTableLookup(Table: AVX512BWUniformConstCostTable, ISD, Ty: LT.second))
460 if (auto KindCost = Entry->Cost[CostKind])
461 return LT.first * *KindCost;
462
463 static const CostKindTblEntry AVX512DQUniformConstCostTable[] = {
464 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15 } }, // vpmullq-based MULHS sequence
465 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17 } }, // vpmullq-based MULHS+mul+sub sequence
466 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 15 } }, // vpmullq-based MULHS sequence
467 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 17 } }, // vpmullq-based MULHS+mul+sub sequence
468 // The remainder's multiply-back is a single vpmullq with DQ, just like the
469 // pmulld the vXi32 entries above rely on. Without DQ it is another
470 // vpmuludq schoolbook, so the AVX512/AVX2 tables charge more.
471 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17 } }, // MULHU + vpmullq + sub sequence
472 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 17 } }, // MULHU + vpmullq + sub sequence
473 };
474
475 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasDQI())
476 if (const auto *Entry =
477 CostTableLookup(Table: AVX512DQUniformConstCostTable, ISD, Ty: LT.second))
478 if (auto KindCost = Entry->Cost[CostKind])
479 return LT.first * *KindCost;
480
481 static const CostKindTblEntry AVX512UniformConstCostTable[] = {
482 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 12, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psllw + pand.
483 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 12, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw + pand.
484 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 12, .SizeAndLatencyCost: 12 } }, // psrlw, pand, pxor, psubb.
485
486 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psllw + split.
487 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psrlw + split.
488 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psraw + split.
489
490 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
491 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
492 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
493 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
494 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
495 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
496
497 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
498 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
499 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
500 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
501 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
502 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
503 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
504
505 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
506 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
507 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
508 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
509
510 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq-based MULHU sequence
511 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 21 } }, // pmuludq-based MULHU+mul+sub sequence
512 };
513
514 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX512())
515 if (const auto *Entry =
516 CostTableLookup(Table: AVX512UniformConstCostTable, ISD, Ty: LT.second))
517 if (auto KindCost = Entry->Cost[CostKind])
518 return LT.first * *KindCost;
519
520 static const CostKindTblEntry AVX2UniformConstCostTable[] = {
521 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
522 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
523 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
524 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // psllw + pand.
525 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // psrlw + pand.
526 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // psrlw, pand, pxor, psubb.
527
528 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
529 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
530 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw
531 { .ISD: ISD::SHL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw
532 { .ISD: ISD::SRL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw
533 { .ISD: ISD::SRA, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw
534
535 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
536 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
537 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
538 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
539 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld
540 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad
541
542 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
543 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
544 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // psrad + shuffle.
545 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
546 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
547 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // psrad + shuffle + split.
548
549 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
550 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
551 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
552 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
553
554 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq-based MULHU sequence
555 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 21 } }, // pmuludq-based MULHU+mul+sub sequence
556 };
557
558 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX2())
559 if (const auto *Entry =
560 CostTableLookup(Table: AVX2UniformConstCostTable, ISD, Ty: LT.second))
561 if (auto KindCost = Entry->Cost[CostKind])
562 return LT.first * *KindCost;
563
564 static const CostKindTblEntry AVXUniformConstCostTable[] = {
565 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
566 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
567 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
568 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2*(psllw + pand) + split.
569 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2*(psrlw + pand) + split.
570 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 12, .SizeAndLatencyCost: 13 } }, // 2*(psrlw, pand, pxor, psubb) + split.
571
572 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw.
573 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw.
574 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw.
575 { .ISD: ISD::SHL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psllw + split.
576 { .ISD: ISD::SRL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw + split.
577 { .ISD: ISD::SRA, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psraw + split.
578
579 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld.
580 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld.
581 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad.
582 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // pslld + split.
583 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrld + split.
584 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrad + split.
585
586 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq.
587 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq.
588 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // psrad + shuffle.
589 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // 2 x psllq + split.
590 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // 2 x psllq + split.
591 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } }, // 2 x psrad + shuffle + split.
592
593 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmuludq sequence + split.
594 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmuludq+mul+sub sequence + split.
595 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 12 } }, // 2*pmuludq sequence + split.
596 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } }, // 2*pmuludq+mul+sub sequence + split.
597 };
598
599 // XOP has faster vXi8 shifts.
600 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX() &&
601 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
602 if (const auto *Entry =
603 CostTableLookup(Table: AVXUniformConstCostTable, ISD, Ty: LT.second))
604 if (auto KindCost = Entry->Cost[CostKind])
605 return LT.first * *KindCost;
606
607 static const CostKindTblEntry SSE2UniformConstCostTable[] = {
608 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
609 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
610 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
611
612 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw.
613 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw.
614 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw.
615
616 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
617 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld.
618 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad.
619
620 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq.
621 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq.
622 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } }, // 2 x psrad + shuffle.
623
624 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
625 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
626 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
627 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
628 };
629
630 // XOP has faster vXi8 shifts.
631 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasSSE2() &&
632 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
633 if (const auto *Entry =
634 CostTableLookup(Table: SSE2UniformConstCostTable, ISD, Ty: LT.second))
635 if (auto KindCost = Entry->Cost[CostKind])
636 return LT.first * *KindCost;
637
638 static const CostKindTblEntry AVX512BWConstCostTable[] = {
639 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
640 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
641 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
642 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
643
644 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhw sequence
645 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhw+mul+sub sequence
646 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhuw sequence
647 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhuw+mul+sub sequence
648 };
649
650 if (Op2Info.isConstant() && ST->hasBWI())
651 if (const auto *Entry =
652 CostTableLookup(Table: AVX512BWConstCostTable, ISD, Ty: LT.second))
653 if (auto KindCost = Entry->Cost[CostKind])
654 return LT.first * *KindCost;
655
656 static const CostKindTblEntry AVX512DQConstCostTable[] = {
657 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 19 } }, // vpmullq-based MULHS sequence
658 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 21 } }, // vpmullq-based MULHS+mul+sub sequence
659 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 19 } }, // vpmullq-based MULHS sequence
660 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 21 } }, // vpmullq-based MULHS+mul+sub sequence
661 // The remainder's multiply-back is a single vpmullq with DQ, whereas the
662 // AVX512/AVX2 tables have to charge for another vpmuludq schoolbook.
663 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24 } }, // MULHU + vpmullq + sub sequence
664 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 24 } }, // MULHU + vpmullq + sub sequence
665 };
666
667 if (Op2Info.isConstant() && ST->hasDQI())
668 if (const auto *Entry =
669 CostTableLookup(Table: AVX512DQConstCostTable, ISD, Ty: LT.second))
670 if (auto KindCost = Entry->Cost[CostKind])
671 return LT.first * *KindCost;
672
673 static const CostKindTblEntry AVX512ConstCostTable[] = {
674 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 28 } }, // 4*ext+4*pmulhw sequence
675 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
676 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 28 } }, // 4*ext+4*pmulhw sequence
677 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
678
679 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 12 } }, // 2*vpmulhw sequence
680 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 16 } }, // 2*vpmulhw+mul+sub sequence
681 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 12 } }, // 2*vpmulhuw sequence
682 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 16 } }, // 2*vpmulhuw+mul+sub sequence
683
684 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuldq sequence
685 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 17 } }, // vpmuldq+mul+sub sequence
686 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
687 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 17 } }, // vpmuludq+mul+sub sequence
688
689 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 22 } }, // vpmuludq-based MULHU sequence
690 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 28 } }, // vpmuludq-based MULHU+mul+sub sequence
691 };
692
693 if (Op2Info.isConstant() && ST->hasAVX512())
694 if (const auto *Entry =
695 CostTableLookup(Table: AVX512ConstCostTable, ISD, Ty: LT.second))
696 if (auto KindCost = Entry->Cost[CostKind])
697 return LT.first * *KindCost;
698
699 static const CostKindTblEntry AVX2ConstCostTable[] = {
700 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
701 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
702 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
703 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
704
705 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhw sequence
706 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhw+mul+sub sequence
707 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhuw sequence
708 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhuw+mul+sub sequence
709
710 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuldq sequence
711 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 19 } }, // vpmuldq+mul+sub sequence
712 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
713 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 19 } }, // vpmuludq+mul+sub sequence
714
715 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 22 } }, // vpmuludq-based MULHU sequence
716 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 28 } }, // vpmuludq-based MULHU+mul+sub sequence
717 };
718
719 if (Op2Info.isConstant() && ST->hasAVX2())
720 if (const auto *Entry = CostTableLookup(Table: AVX2ConstCostTable, ISD, Ty: LT.second))
721 if (auto KindCost = Entry->Cost[CostKind])
722 return LT.first * *KindCost;
723
724 static const CostKindTblEntry AVXConstCostTable[] = {
725 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 30 } }, // 4*ext+4*pmulhw sequence + split.
726 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
727 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 30 } }, // 4*ext+4*pmulhw sequence + split.
728 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
729
730 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmulhw sequence + split.
731 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmulhw+mul+sub sequence + split.
732 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmulhuw sequence + split.
733 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmulhuw+mul+sub sequence + split.
734
735 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 32 } }, // vpmuludq sequence
736 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 38 } }, // vpmuludq+mul+sub sequence
737 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 32 } }, // 2*pmuludq sequence + split.
738 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 42 } }, // 2*pmuludq+mul+sub sequence + split.
739 };
740
741 if (Op2Info.isConstant() && ST->hasAVX())
742 if (const auto *Entry = CostTableLookup(Table: AVXConstCostTable, ISD, Ty: LT.second))
743 if (auto KindCost = Entry->Cost[CostKind])
744 return LT.first * *KindCost;
745
746 static const CostKindTblEntry SSE41ConstCostTable[] = {
747 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
748 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20 } }, // vpmuludq+mul+sub sequence
749 };
750
751 if (Op2Info.isConstant() && ST->hasSSE41())
752 if (const auto *Entry =
753 CostTableLookup(Table: SSE41ConstCostTable, ISD, Ty: LT.second))
754 if (auto KindCost = Entry->Cost[CostKind])
755 return LT.first * *KindCost;
756
757 static const CostKindTblEntry SSE2ConstCostTable[] = {
758 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
759 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
760 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
761 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
762
763 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6 } }, // pmulhw sequence
764 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8 } }, // pmulhw+mul+sub sequence
765 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6 } }, // pmulhuw sequence
766 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8 } }, // pmulhuw+mul+sub sequence
767
768 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19 } }, // pmuludq sequence
769 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 24 } }, // pmuludq+mul+sub sequence
770 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq sequence
771 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20 } }, // pmuludq+mul+sub sequence
772 };
773
774 if (Op2Info.isConstant() && ST->hasSSE2())
775 if (const auto *Entry = CostTableLookup(Table: SSE2ConstCostTable, ISD, Ty: LT.second))
776 if (auto KindCost = Entry->Cost[CostKind])
777 return LT.first * *KindCost;
778
779 // rem matches div because the divider returns the remainder for free.
780 static const CostKindTblEntry ScalarVarDivCostTable[] = {
781 { .ISD: ISD::SDIV, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
782 { .ISD: ISD::UDIV, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
783 { .ISD: ISD::SREM, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
784 { .ISD: ISD::UREM, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
785 { .ISD: ISD::SDIV, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
786 { .ISD: ISD::UDIV, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
787 { .ISD: ISD::SREM, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
788 { .ISD: ISD::UREM, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
789 { .ISD: ISD::SDIV, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
790 { .ISD: ISD::UDIV, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
791 { .ISD: ISD::SREM, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
792 { .ISD: ISD::UREM, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
793 { .ISD: ISD::SDIV, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
794 { .ISD: ISD::UDIV, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
795 { .ISD: ISD::SREM, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
796 { .ISD: ISD::UREM, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
797 };
798
799 if (!LT.second.isVector() && !Op2Info.isConstant())
800 if (const auto *Entry =
801 CostTableLookup(Table: ScalarVarDivCostTable, ISD, Ty: LT.second))
802 if (auto KindCost = Entry->Cost[CostKind])
803 return LT.first * *KindCost;
804
805 // Variable divisors lower through a float divide. strictfp needs SAE
806 // rounding which is 512-bit only.
807 bool IsStrictFP =
808 CtxI && CtxI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP);
809 bool IsDivRem = ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
810 ISD == ISD::SREM;
811 bool VarDivToFP = IsDivRem && !Op2Info.isConstant() &&
812 (!IsStrictFP || ST->useAVX512Regs());
813
814 // i64 needs the qq converts, which are AVX512DQ only. Two tables because the
815 // lowering picks by operand value and not by type.
816 static const CostKindTblEntry AVX512DQExactVarDivCostTable[] = {
817 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } }, // cvt+divpd sequence
818 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
819 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
820 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
821 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
822 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
823 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
824 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
825 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
826 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
827 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
828 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
829 };
830
831 static const CostKindTblEntry AVX512DQVarDivCostTable[] = {
832 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
833 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
834 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
835 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
836 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
837 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
838 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
839 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
840 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
841 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 18 } },
842 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
843 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 18 } },
844 };
845
846 // The DAG combine picks between the two sequences with these same two
847 // queries, so the cost cannot disagree with what codegen emits.
848 bool IsSignedDiv = ISD == ISD::SDIV || ISD == ISD::SREM;
849 auto OperandsFit = [&](unsigned Mantissa) {
850 if (Args.size() != 2 || !CtxI)
851 return false;
852 unsigned EltBits = LT.second.getScalarSizeInBits();
853 const DataLayout &DL = CtxI->getDataLayout();
854 auto Fits = [&](const Value *V) {
855 if (IsSignedDiv)
856 return ComputeNumSignBits(Op: V, DL, /*AC=*/nullptr, CtxI) + Mantissa >
857 EltBits;
858 return computeKnownBits(V, DL, /*AC=*/nullptr, CtxI)
859 .countMaxActiveBits() <= Mantissa;
860 };
861 return Fits(Args[0]) && Fits(Args[1]);
862 };
863
864 // An i32 divide goes through f32 when both operands fit in 24 bits, and
865 // through f64 at twice the vector width when they do not.
866 bool ExactI32 =
867 VarDivToFP && LT.second.getScalarType() == MVT::i32 &&
868 OperandsFit(APFloat::semanticsPrecision(APFloat::IEEEsingle()));
869
870 if (VarDivToFP && ST->hasDQI() && ST->useAVX512Regs() &&
871 LT.second.getScalarType() == MVT::i64) {
872 bool ExactFPDiv =
873 OperandsFit(APFloat::semanticsPrecision(APFloat::IEEEdouble()));
874
875 // Only the reciprocal chain multiplies, so only it is vpmullq gated.
876 bool SlowMultiply =
877 !ExactFPDiv && LT.second == MVT::v2i64 && ST->isPMULLQSlow();
878
879 if (!SlowMultiply) {
880 ArrayRef<CostKindTblEntry> Tbl =
881 ExactFPDiv ? AVX512DQExactVarDivCostTable : AVX512DQVarDivCostTable;
882 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
883 if (auto KindCost = Entry->Cost[CostKind])
884 return LT.first * *KindCost;
885 }
886 }
887
888 static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
889 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } }, // unpack+cvt+divps sequence
890 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
891 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
892 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
893 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
894 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
895 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
896 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
897 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
898 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
899 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
900 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
901 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
902 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
903 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
904 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
905 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
906 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
907 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
908 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
909 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
910 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
911 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
912 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
913 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } }, // cvt+divpd sequence
914 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
915 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
916 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
917 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
918 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
919 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
920 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
921 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
922 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
923 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
924 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
925 };
926
927 static const CostKindTblEntry AVX512BWExactVarDivCostTable[] = {
928 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } }, // cvt+divps sequence
929 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
930 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
931 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
932 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
933 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
934 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
935 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
936 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
937 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
938 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
939 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
940 };
941
942 if (VarDivToFP && ST->hasBWI()) {
943 ArrayRef<CostKindTblEntry> Tbl = AVX512BWVarDivCostTable;
944 if (ExactI32)
945 Tbl = AVX512BWExactVarDivCostTable;
946 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
947 if (auto KindCost = Entry->Cost[CostKind])
948 return LT.first * *KindCost;
949 }
950
951 static const CostKindTblEntry AVX512VarDivCostTable[] = {
952 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // unpack+cvt+divps sequence
953 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
954 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
955 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
956 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
957 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
958 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
959 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
960 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
961 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
962 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
963 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
964 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
965 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
966 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
967 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
968 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
969 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
970 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
971 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
972 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
973 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
974 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
975 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
976 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } }, // cvt+divpd sequence
977 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
978 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
979 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
980 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
981 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
982 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
983 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
984 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
985 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
986 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
987 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
988 };
989
990 static const CostKindTblEntry AVX512ExactVarDivCostTable[] = {
991 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } }, // cvt+divps sequence
992 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
993 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
994 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
995 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
996 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
997 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
998 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
999 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
1000 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
1001 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
1002 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
1003 };
1004
1005 if (VarDivToFP && ST->hasAVX512()) {
1006 ArrayRef<CostKindTblEntry> Tbl = AVX512VarDivCostTable;
1007 if (ExactI32)
1008 Tbl = AVX512ExactVarDivCostTable;
1009 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1010 if (auto KindCost = Entry->Cost[CostKind])
1011 return LT.first * *KindCost;
1012 }
1013
1014 static const CostKindTblEntry AVX2VarDivCostTable[] = {
1015 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } }, // unpack+cvt+divps sequence
1016 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
1017 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
1018 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
1019 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
1020 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
1021 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
1022 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
1023 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
1024 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
1025 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
1026 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
1027 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
1028 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
1029 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
1030 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
1031 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } }, // cvt+divpd sequence
1032 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1033 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1034 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1035 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1036 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1037 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1038 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1039 };
1040
1041 static const CostKindTblEntry AVX2ExactVarDivCostTable[] = {
1042 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9 } }, // cvt+divps sequence
1043 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
1044 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9 } },
1045 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
1046 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1047 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1048 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1049 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1050 };
1051
1052 if (VarDivToFP && ST->hasAVX2()) {
1053 ArrayRef<CostKindTblEntry> Tbl = AVX2VarDivCostTable;
1054 if (ExactI32)
1055 Tbl = AVX2ExactVarDivCostTable;
1056 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1057 if (auto KindCost = Entry->Cost[CostKind])
1058 return LT.first * *KindCost;
1059 }
1060
1061 // No unsigned i32 entries below AVX2, where the u32 to f64 converts are
1062 // emulated and the fold stays off.
1063 static const CostKindTblEntry AVX1VarDivCostTable[] = {
1064 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } }, // unpack+cvt+divps sequence
1065 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1066 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1067 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1068 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1069 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1070 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1071 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1072 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1073 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1074 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1075 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1076 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1077 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1078 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1079 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1080 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } }, // cvt+divpd sequence
1081 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } },
1082 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 88 } },
1083 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 88 } },
1084 };
1085
1086 static const CostKindTblEntry AVX1ExactVarDivCostTable[] = {
1087 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } }, // cvt+divps sequence
1088 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } },
1089 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
1090 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
1091 };
1092
1093 if (VarDivToFP && ST->hasAVX()) {
1094 ArrayRef<CostKindTblEntry> Tbl = AVX1VarDivCostTable;
1095 if (ExactI32)
1096 Tbl = AVX1ExactVarDivCostTable;
1097 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1098 if (auto KindCost = Entry->Cost[CostKind])
1099 return LT.first * *KindCost;
1100 }
1101
1102 static const CostKindTblEntry SSE2VarDivCostTable[] = {
1103 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } }, // unpack+cvt+divps sequence
1104 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1105 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1106 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1107 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1108 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1109 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1110 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1111 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } }, // cvt+divpd sequence
1112 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } },
1113 };
1114
1115 static const CostKindTblEntry SSE2ExactVarDivCostTable[] = {
1116 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } }, // cvt+divps sequence
1117 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } },
1118 };
1119
1120 if (VarDivToFP && ST->hasSSE2()) {
1121 ArrayRef<CostKindTblEntry> Tbl = SSE2VarDivCostTable;
1122 if (ExactI32)
1123 Tbl = SSE2ExactVarDivCostTable;
1124 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1125 if (auto KindCost = Entry->Cost[CostKind])
1126 return LT.first * *KindCost;
1127 }
1128
1129 static const CostKindTblEntry AVX512BWUniformCostTable[] = {
1130 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + pand.
1131 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1132 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 8,.SizeAndLatencyCost: 12 } }, // psrlw, pand, pxor, psubb.
1133 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1134 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1135 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1136 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1137 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // psrlw + pand.
1138 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 15 } }, // psrlw, pand, pxor, psubb.
1139
1140 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw
1141 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw
1142 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrqw
1143 };
1144
1145 if (ST->hasBWI() && Op2Info.isUniform())
1146 if (const auto *Entry =
1147 CostTableLookup(Table: AVX512BWUniformCostTable, ISD, Ty: LT.second))
1148 if (auto KindCost = Entry->Cost[CostKind])
1149 return LT.first * *KindCost;
1150
1151 static const CostKindTblEntry AVX512UniformCostTable[] = {
1152 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + split.
1153 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrlw + split.
1154 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psraw + split.
1155
1156 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pslld
1157 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrld
1158 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrad
1159
1160 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1161 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1162 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1163 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1164 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1165 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1166 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1167 };
1168
1169 if (ST->hasAVX512() && Op2Info.isUniform())
1170 if (const auto *Entry =
1171 CostTableLookup(Table: AVX512UniformCostTable, ISD, Ty: LT.second))
1172 if (auto KindCost = Entry->Cost[CostKind])
1173 return LT.first * *KindCost;
1174
1175 static const CostKindTblEntry AVX2UniformCostTable[] = {
1176 // Uniform splats are cheaper for the following instructions.
1177 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + pand.
1178 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1179 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1180 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1181 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1182 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // psrlw, pand, pxor, psubb.
1183
1184 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1185 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1186 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1187 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw.
1188 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw.
1189 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psraw.
1190
1191 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
1192 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld
1193 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad
1194 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pslld
1195 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrld
1196 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrad
1197
1198 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1199 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1200 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2 x psrad + shuffle.
1201 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1202 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1203 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // 2 x psrad + shuffle.
1204 };
1205
1206 if (ST->hasAVX2() && Op2Info.isUniform())
1207 if (const auto *Entry =
1208 CostTableLookup(Table: AVX2UniformCostTable, ISD, Ty: LT.second))
1209 if (auto KindCost = Entry->Cost[CostKind])
1210 return LT.first * *KindCost;
1211
1212 static const CostKindTblEntry AVXUniformCostTable[] = {
1213 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1214 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1215 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1216 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 8,.CodeSizeCost: 11,.SizeAndLatencyCost: 14 } }, // psllw + pand + split.
1217 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 9,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // psrlw + pand + split.
1218 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 10,.LatencyCost: 11,.CodeSizeCost: 16,.SizeAndLatencyCost: 21 } }, // psrlw, pand, pxor, psubb + split.
1219
1220 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1221 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1222 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1223 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + split.
1224 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrlw + split.
1225 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psraw + split.
1226
1227 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld.
1228 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld.
1229 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad.
1230 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // pslld + split.
1231 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrld + split.
1232 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrad + split.
1233
1234 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq.
1235 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq.
1236 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2 x psrad + shuffle.
1237 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psllq + split.
1238 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psrlq + split.
1239 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 10,.SizeAndLatencyCost: 13 } }, // 2 x (2 x psrad + shuffle) + split.
1240 };
1241
1242 // XOP has faster vXi8 shifts.
1243 if (ST->hasAVX() && Op2Info.isUniform() &&
1244 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1245 if (const auto *Entry =
1246 CostTableLookup(Table: AVXUniformCostTable, ISD, Ty: LT.second))
1247 if (auto KindCost = Entry->Cost[CostKind])
1248 return LT.first * *KindCost;
1249
1250 static const CostKindTblEntry SSE2UniformCostTable[] = {
1251 // Uniform splats are cheaper for the following instructions.
1252 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } }, // psllw + pand.
1253 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1254 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 15, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // pcmpgtb sequence.
1255
1256 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1257 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1258 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1259
1260 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
1261 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld.
1262 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad.
1263
1264 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq.
1265 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq.
1266 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2*psrlq + xor + sub.
1267 };
1268
1269 if (ST->hasSSE2() && Op2Info.isUniform() &&
1270 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1271 if (const auto *Entry =
1272 CostTableLookup(Table: SSE2UniformCostTable, ISD, Ty: LT.second))
1273 if (auto KindCost = Entry->Cost[CostKind])
1274 return LT.first * *KindCost;
1275
1276 static const CostKindTblEntry AVX512DQCostTable[] = {
1277 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmullq
1278 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmullq
1279 { .ISD: ISD::MUL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } } // pmullq
1280 };
1281
1282 // Look for AVX512DQ lowering tricks for custom cases.
1283 if (ST->hasDQI())
1284 if (const auto *Entry = CostTableLookup(Table: AVX512DQCostTable, ISD, Ty: LT.second))
1285 if (auto KindCost = Entry->Cost[CostKind])
1286 return LT.first * *KindCost;
1287
1288 static const CostKindTblEntry AVX512BWCostTable[] = {
1289 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsllvw/pack sequence.
1290 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsrlvw/pack sequence.
1291 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsravw/pack sequence.
1292 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 23,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // extend/vpsllvw/pack sequence.
1293 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 30,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // extend/vpsrlvw/pack sequence.
1294 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13,.CodeSizeCost: 24,.SizeAndLatencyCost: 30 } }, // extend/vpsravw/pack sequence.
1295 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 19,.CodeSizeCost: 13,.SizeAndLatencyCost: 15 } }, // extend/vpsllvw/pack sequence.
1296 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 27,.CodeSizeCost: 15,.SizeAndLatencyCost: 18 } }, // extend/vpsrlvw/pack sequence.
1297 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 15,.CodeSizeCost: 30,.SizeAndLatencyCost: 30 } }, // extend/vpsravw/pack sequence.
1298
1299 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1300 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1301 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1302 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1303 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1304 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1305 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1306 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1307 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1308
1309 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddb
1310 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddw
1311
1312 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddb
1313 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddw
1314 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddd
1315 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddq
1316
1317 { .ISD: ISD::SUB, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubb
1318 { .ISD: ISD::SUB, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubw
1319
1320 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/pmullw/trunc
1321 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // pmaddubsw
1322 { .ISD: ISD::MUL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // pmaddubsw
1323 { .ISD: ISD::MUL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1324
1325 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubb
1326 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubw
1327 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubd
1328 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubq
1329 };
1330
1331 // Look for AVX512BW lowering tricks for custom cases.
1332 if (ST->hasBWI())
1333 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTable, ISD, Ty: LT.second))
1334 if (auto KindCost = Entry->Cost[CostKind])
1335 return LT.first * *KindCost;
1336
1337 static const CostKindTblEntry AVX512CostTable[] = {
1338 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19,.CodeSizeCost: 27,.SizeAndLatencyCost: 33 } }, // vpblendv+split sequence.
1339 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19,.CodeSizeCost: 30,.SizeAndLatencyCost: 36 } }, // vpblendv+split sequence.
1340 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 37, .LatencyCost: 37,.CodeSizeCost: 51,.SizeAndLatencyCost: 63 } }, // vpblendv+split sequence.
1341
1342 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsrlvd/pack sequence.
1343 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsrlvd/pack sequence.
1344 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsravd/pack sequence.
1345
1346 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1347 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1348 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1349 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1350 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1351 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1352 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1353 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1354 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1355
1356 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1357 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1358 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1359 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1360 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1361 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1362 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1363 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1364 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1365
1366 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*paddb + split
1367 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*paddw + split
1368
1369 { .ISD: ISD::SUB, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*psubb + split
1370 { .ISD: ISD::SUB, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*psubw + split
1371
1372 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1373 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1374 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1375 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1376
1377 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1378 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1379 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1380 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1381
1382 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1383 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1384 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1385 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1386
1387 { .ISD: ISD::MUL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1388 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1389 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1390 { .ISD: ISD::MUL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // 3*pmuludq/3*shift/2*add
1391 { .ISD: ISD::MUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Skylake from http://www.agner.org/
1392
1393 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1394
1395 { .ISD: ISD::FNEG, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Skylake from http://www.agner.org/
1396 { .ISD: ISD::FADD, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1397 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1398 { .ISD: ISD::FSUB, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1399 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1400 { .ISD: ISD::FMUL, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1401 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1402 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1403 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1404
1405 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1406 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1407 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1408 { .ISD: ISD::FDIV, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 23, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
1409
1410 { .ISD: ISD::FNEG, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Skylake from http://www.agner.org/
1411 { .ISD: ISD::FADD, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1412 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1413 { .ISD: ISD::FSUB, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1414 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1415 { .ISD: ISD::FMUL, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1416 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1417 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1418 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1419
1420 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1421 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1422 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1423 { .ISD: ISD::FDIV, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
1424 };
1425
1426 if (ST->hasAVX512())
1427 if (const auto *Entry = CostTableLookup(Table: AVX512CostTable, ISD, Ty: LT.second))
1428 if (auto KindCost = Entry->Cost[CostKind])
1429 return LT.first * *KindCost;
1430
1431 static const CostKindTblEntry AVX2ShiftCostTable[] = {
1432 // Shifts on vXi64/vXi32 on AVX2 is legal even though we declare to
1433 // customize them to detect the cases where shift amount is a scalar one.
1434 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsllvd (Haswell from agner.org)
1435 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsrlvd (Haswell from agner.org)
1436 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsravd (Haswell from agner.org)
1437 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsllvd (Haswell from agner.org)
1438 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsrlvd (Haswell from agner.org)
1439 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsravd (Haswell from agner.org)
1440 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvq (Haswell from agner.org)
1441 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvq (Haswell from agner.org)
1442 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpsllvq (Haswell from agner.org)
1443 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpsrlvq (Haswell from agner.org)
1444 };
1445
1446 if (ST->hasAVX512()) {
1447 if (ISD == ISD::SHL && LT.second == MVT::v32i16 && Op2Info.isConstant())
1448 // On AVX512, a packed v32i16 shift left by a constant build_vector
1449 // is lowered into a vector multiply (vpmullw).
1450 return getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
1451 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1452 }
1453
1454 // Look for AVX2 lowering tricks (XOP is always better at v4i32 shifts).
1455 if (ST->hasAVX2() && !(ST->hasXOP() && LT.second == MVT::v4i32)) {
1456 if (ISD == ISD::SHL && LT.second == MVT::v16i16 &&
1457 Op2Info.isConstant())
1458 // On AVX2, a packed v16i16 shift left by a constant build_vector
1459 // is lowered into a vector multiply (vpmullw).
1460 return getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
1461 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1462
1463 if (const auto *Entry = CostTableLookup(Table: AVX2ShiftCostTable, ISD, Ty: LT.second))
1464 if (auto KindCost = Entry->Cost[CostKind])
1465 return LT.first * *KindCost;
1466 }
1467
1468 static const CostKindTblEntry XOPShiftCostTable[] = {
1469 // 128bit shifts take 1cy, but right shifts require negation beforehand.
1470 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1471 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1472 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1473 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1474 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1475 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1476 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1477 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1478 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1479 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1480 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1481 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1482 // 256bit shifts require splitting if AVX2 didn't catch them above.
1483 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1484 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1485 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1486 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1487 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1488 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1489 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1490 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1491 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1492 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1493 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1494 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1495 };
1496
1497 // Look for XOP lowering tricks.
1498 if (ST->hasXOP()) {
1499 // If the right shift is constant then we'll fold the negation so
1500 // it's as cheap as a left shift.
1501 int ShiftISD = ISD;
1502 if ((ShiftISD == ISD::SRL || ShiftISD == ISD::SRA) && Op2Info.isConstant())
1503 ShiftISD = ISD::SHL;
1504 if (const auto *Entry =
1505 CostTableLookup(Table: XOPShiftCostTable, ISD: ShiftISD, Ty: LT.second))
1506 if (auto KindCost = Entry->Cost[CostKind])
1507 return LT.first * *KindCost;
1508 }
1509
1510 if (ISD == ISD::SHL && !Op2Info.isUniform() && Op2Info.isConstant()) {
1511 MVT VT = LT.second;
1512 // Vector shift left by non uniform constant can be lowered
1513 // into vector multiply.
1514 if (((VT == MVT::v8i16 || VT == MVT::v4i32) && ST->hasSSE2()) ||
1515 ((VT == MVT::v16i16 || VT == MVT::v8i32) && ST->hasAVX()))
1516 ISD = ISD::MUL;
1517 }
1518
1519 static const CostKindTblEntry GLMCostTable[] = {
1520 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 19, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divss
1521 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 35, .LatencyCost: 36, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divps
1522 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 33, .LatencyCost: 34, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divsd
1523 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 65, .LatencyCost: 66, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divpd
1524 };
1525
1526 if (ST->useGLMDivSqrtCosts())
1527 if (const auto *Entry = CostTableLookup(Table: GLMCostTable, ISD, Ty: LT.second))
1528 if (auto KindCost = Entry->Cost[CostKind])
1529 return LT.first * *KindCost;
1530
1531 static const CostKindTblEntry SLMCostTable[] = {
1532 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } }, // pmulld
1533 { .ISD: ISD::MUL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1534 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulsd
1535 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulss
1536 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulpd
1537 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulps
1538 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 19, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divss
1539 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 39, .LatencyCost: 39, .CodeSizeCost: 1, .SizeAndLatencyCost: 6 } }, // divps
1540 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 34, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divsd
1541 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 69, .LatencyCost: 69, .CodeSizeCost: 1, .SizeAndLatencyCost: 6 } }, // divpd
1542 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // addpd
1543 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // subpd
1544 // v2i64/v4i64 mul is custom lowered as a series of long:
1545 // multiplies(3), shifts(3) and adds(2)
1546 // slm muldq version throughput is 2 and addq throughput 4
1547 // thus: 3X2 (muldq throughput) + 3X1 (shift throughput) +
1548 // 3X4 (addq throughput) = 17
1549 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 22, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
1550 // slm addq\subq throughput is 4
1551 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1552 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1553 };
1554
1555 if (ST->useSLMArithCosts())
1556 if (const auto *Entry = CostTableLookup(Table: SLMCostTable, ISD, Ty: LT.second))
1557 if (auto KindCost = Entry->Cost[CostKind])
1558 return LT.first * *KindCost;
1559
1560 static const CostKindTblEntry AVX2CostTable[] = {
1561 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 21,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // vpblendvb sequence.
1562 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 23,.CodeSizeCost: 11,.SizeAndLatencyCost: 22 } }, // vpblendvb sequence.
1563 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsrlvd/pack sequence.
1564 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsrlvd/pack sequence.
1565
1566 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 27,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // vpblendvb sequence.
1567 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 30,.CodeSizeCost: 12,.SizeAndLatencyCost: 24 } }, // vpblendvb sequence.
1568 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsrlvd/pack sequence.
1569 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsrlvd/pack sequence.
1570
1571 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 17,.CodeSizeCost: 24,.SizeAndLatencyCost: 30 } }, // vpblendvb sequence.
1572 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 20,.CodeSizeCost: 24,.SizeAndLatencyCost: 43 } }, // vpblendvb sequence.
1573 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsravd/pack sequence.
1574 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsravd/pack sequence.
1575 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // srl/xor/sub sequence.
1576 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // srl/xor/sub sequence.
1577
1578 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubb
1579 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddb
1580 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubw
1581 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddw
1582 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubd
1583 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddd
1584 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubq
1585 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddq
1586
1587 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18, .CodeSizeCost: 6,.SizeAndLatencyCost: 12 } }, // extend/pmullw/pack
1588 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 8,.SizeAndLatencyCost: 16 } }, // pmaddubsw
1589 { .ISD: ISD::MUL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmullw
1590 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld
1591 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld
1592 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 8,.SizeAndLatencyCost: 13 } }, // 3*pmuludq/3*shift/2*add
1593 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // 3*pmuludq/3*shift/2*add
1594
1595 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1596
1597 { .ISD: ISD::FNEG, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorpd
1598 { .ISD: ISD::FNEG, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1599
1600 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddsd
1601 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddss
1602 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddpd
1603 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddps
1604 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vaddpd
1605 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vaddps
1606
1607 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubsd
1608 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubss
1609 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubpd
1610 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubps
1611 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vsubpd
1612 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vsubps
1613
1614 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulsd
1615 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulss
1616 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulpd
1617 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulps
1618 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vmulpd
1619 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vmulps
1620
1621 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivss
1622 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivps
1623 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vdivps
1624 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivsd
1625 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivpd
1626 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vdivpd
1627 };
1628
1629 // Look for AVX2 lowering tricks for custom cases.
1630 if (ST->hasAVX2())
1631 if (const auto *Entry = CostTableLookup(Table: AVX2CostTable, ISD, Ty: LT.second))
1632 if (auto KindCost = Entry->Cost[CostKind])
1633 return LT.first * *KindCost;
1634
1635 static const CostKindTblEntry AVX1CostTable[] = {
1636 // We don't have to scalarize unsupported ops. We can issue two half-sized
1637 // operations and we only need to extract the upper YMM half.
1638 // Two ops + 1 extract + 1 insert = 4.
1639 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 11, .CodeSizeCost: 18, .SizeAndLatencyCost: 19 } }, // pmaddubsw + split
1640 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } }, // 2*pmaddubsw/3*and/psllw/or
1641 { .ISD: ISD::MUL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // pmullw + split
1642 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } }, // pmulld + split
1643 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmulld
1644 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 15, .CodeSizeCost: 19, .SizeAndLatencyCost: 20 } },
1645
1646 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // pmuludq + split
1647
1648 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1649 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1650 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1651 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1652
1653 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1654 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1655 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1656 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1657
1658 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1659 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1660 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1661 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1662
1663 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubb + split
1664 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddb + split
1665 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubw + split
1666 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddw + split
1667 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubd + split
1668 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddd + split
1669 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubq + split
1670 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddq + split
1671 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubq
1672 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddq
1673
1674 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 21,.CodeSizeCost: 11,.SizeAndLatencyCost: 17 } }, // pblendvb sequence.
1675 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22,.CodeSizeCost: 27,.SizeAndLatencyCost: 40 } }, // pblendvb sequence + split.
1676 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9,.CodeSizeCost: 11,.SizeAndLatencyCost: 11 } }, // pblendvb sequence.
1677 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 24,.SizeAndLatencyCost: 25 } }, // pblendvb sequence + split.
1678 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // pslld/paddd/cvttps2dq/pmulld
1679 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11,.CodeSizeCost: 12,.SizeAndLatencyCost: 17 } }, // pslld/paddd/cvttps2dq/pmulld + split
1680 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // Shift each lane + blend.
1681 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // Shift each lane + blend + split.
1682
1683 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 27,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // pblendvb sequence.
1684 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 23, .LatencyCost: 23,.CodeSizeCost: 30,.SizeAndLatencyCost: 43 } }, // pblendvb sequence + split.
1685 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 14,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1686 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30,.CodeSizeCost: 31,.SizeAndLatencyCost: 48 } }, // pblendvb sequence + split.
1687 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // Shift each lane + blend.
1688 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14,.CodeSizeCost: 26,.SizeAndLatencyCost: 34 } }, // Shift each lane + blend + split.
1689 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // Shift each lane + blend.
1690 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // Shift each lane + blend + split.
1691
1692 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 22,.CodeSizeCost: 24,.SizeAndLatencyCost: 36 } }, // pblendvb sequence.
1693 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 45,.CodeSizeCost: 51,.SizeAndLatencyCost: 76 } }, // pblendvb sequence + split.
1694 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 14,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1695 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30,.CodeSizeCost: 31,.SizeAndLatencyCost: 48 } }, // pblendvb sequence + split.
1696 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // Shift each lane + blend.
1697 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14,.CodeSizeCost: 26,.SizeAndLatencyCost: 34 } }, // Shift each lane + blend + split.
1698 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // Shift each lane + blend.
1699 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 22,.SizeAndLatencyCost: 30 } }, // Shift each lane + blend + split.
1700
1701 { .ISD: ISD::FNEG, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1702 { .ISD: ISD::FNEG, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1703
1704 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1705 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1706 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1707 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1708 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1709 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1710
1711 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1712 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1713 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1714 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1715 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1716 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1717
1718 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1719 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1720 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1721 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1722 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1723 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1724
1725 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1726 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1727 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 29, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // SNB from http://www.agner.org/
1728 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1729 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1730 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 45, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // SNB from http://www.agner.org/
1731 };
1732
1733 if (ST->hasAVX())
1734 if (const auto *Entry = CostTableLookup(Table: AVX1CostTable, ISD, Ty: LT.second))
1735 if (auto KindCost = Entry->Cost[CostKind])
1736 return LT.first * *KindCost;
1737
1738 static const CostKindTblEntry SSE42CostTable[] = {
1739 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1740 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1741 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1742 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1743
1744 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1745 { .ISD: ISD::FSUB, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1746 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1747 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1748
1749 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1750 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1751 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1752 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1753
1754 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1755 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1756 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1757 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1758
1759 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 10 } } // 3*pmuludq/3*shift/2*add
1760 };
1761
1762 if (ST->hasSSE42())
1763 if (const auto *Entry = CostTableLookup(Table: SSE42CostTable, ISD, Ty: LT.second))
1764 if (auto KindCost = Entry->Cost[CostKind])
1765 return LT.first * *KindCost;
1766
1767 static const CostKindTblEntry SSE41CostTable[] = {
1768 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 24,.CodeSizeCost: 17,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1769 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11 } }, // pblendvb sequence.
1770 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 4,.SizeAndLatencyCost: 10 } }, // pslld/paddd/cvttps2dq/pmulld
1771
1772 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 27,.CodeSizeCost: 18,.SizeAndLatencyCost: 24 } }, // pblendvb sequence.
1773 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 26,.CodeSizeCost: 23,.SizeAndLatencyCost: 27 } }, // pblendvb sequence.
1774 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 17,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1775 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1776
1777 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 41,.CodeSizeCost: 30,.SizeAndLatencyCost: 36 } }, // pblendvb sequence.
1778 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 26,.CodeSizeCost: 23,.SizeAndLatencyCost: 27 } }, // pblendvb sequence.
1779 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 17,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1780 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 17, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1781
1782 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } } // pmulld (Nehalem from agner.org)
1783 };
1784
1785 if (ST->hasSSE41())
1786 if (const auto *Entry = CostTableLookup(Table: SSE41CostTable, ISD, Ty: LT.second))
1787 if (auto KindCost = Entry->Cost[CostKind])
1788 return LT.first * *KindCost;
1789
1790 static const CostKindTblEntry SSSE3CostTable[] = {
1791 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18,.CodeSizeCost: 10,.SizeAndLatencyCost: 12 } }, // 2*pmaddubsw/3*and/psllw/or
1792 };
1793
1794 if (ST->hasSSSE3())
1795 if (const auto *Entry = CostTableLookup(Table: SSSE3CostTable, ISD, Ty: LT.second))
1796 if (auto KindCost = Entry->Cost[CostKind])
1797 return LT.first * *KindCost;
1798
1799 static const CostKindTblEntry SSE2CostTable[] = {
1800 // We don't correctly identify costs of casts because they are marked as
1801 // custom.
1802 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 21,.CodeSizeCost: 26,.SizeAndLatencyCost: 28 } }, // cmpgtb sequence.
1803 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 27,.CodeSizeCost: 16,.SizeAndLatencyCost: 20 } }, // cmpgtw sequence.
1804 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 19,.CodeSizeCost: 10,.SizeAndLatencyCost: 12 } }, // pslld/paddd/cvttps2dq/pmuludq.
1805 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1806
1807 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 28,.CodeSizeCost: 27,.SizeAndLatencyCost: 30 } }, // cmpgtb sequence.
1808 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19,.CodeSizeCost: 31,.SizeAndLatencyCost: 31 } }, // cmpgtw sequence.
1809 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1810 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1811
1812 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 30,.CodeSizeCost: 54,.SizeAndLatencyCost: 54 } }, // unpacked cmpgtb sequence.
1813 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19,.CodeSizeCost: 31,.SizeAndLatencyCost: 31 } }, // cmpgtw sequence.
1814 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1815 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // srl/xor/sub splat+shuffle sequence.
1816
1817 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1818 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1819 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1820 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1821
1822 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1823 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1824 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1825 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1826
1827 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1828 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1829 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1830 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1831
1832 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddq
1833 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubq
1834
1835 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18,.CodeSizeCost: 12,.SizeAndLatencyCost: 12 } }, // 2*unpack/2*pmullw/2*and/pack
1836 { .ISD: ISD::MUL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1837 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // 3*pmuludq/4*shuffle
1838 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 10 } }, // 3*pmuludq/3*shift/2*add
1839
1840 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1841
1842 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 23, .LatencyCost: 23, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1843 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 39, .LatencyCost: 39, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1844 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 38, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1845 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 69, .LatencyCost: 69, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1846
1847 { .ISD: ISD::FNEG, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1848 { .ISD: ISD::FNEG, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1849 { .ISD: ISD::FNEG, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1850 { .ISD: ISD::FNEG, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1851
1852 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1853 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1854 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1855
1856 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1857 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1858 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1859
1860 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1861 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1862 };
1863
1864 if (ST->hasSSE2())
1865 if (const auto *Entry = CostTableLookup(Table: SSE2CostTable, ISD, Ty: LT.second))
1866 if (auto KindCost = Entry->Cost[CostKind])
1867 return LT.first * *KindCost;
1868
1869 static const CostKindTblEntry SSE1CostTable[] = {
1870 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1871 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 34, .LatencyCost: 48, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1872
1873 { .ISD: ISD::FNEG, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
1874 { .ISD: ISD::FNEG, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
1875
1876 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1877 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1878
1879 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1880 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1881
1882 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1883 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1884 };
1885
1886 if (ST->hasSSE1())
1887 if (const auto *Entry = CostTableLookup(Table: SSE1CostTable, ISD, Ty: LT.second))
1888 if (auto KindCost = Entry->Cost[CostKind])
1889 return LT.first * *KindCost;
1890
1891 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
1892 { .ISD: ISD::ADD, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Core (Merom) from http://www.agner.org/
1893 { .ISD: ISD::SUB, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Core (Merom) from http://www.agner.org/
1894 { .ISD: ISD::MUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1895 };
1896
1897 if (ST->is64Bit())
1898 if (const auto *Entry = CostTableLookup(Table: X64CostTbl, ISD, Ty: LT.second))
1899 if (auto KindCost = Entry->Cost[CostKind])
1900 return LT.first * *KindCost;
1901
1902 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
1903 { .ISD: ISD::ADD, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1904 { .ISD: ISD::ADD, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1905 { .ISD: ISD::ADD, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1906
1907 { .ISD: ISD::SUB, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1908 { .ISD: ISD::SUB, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1909 { .ISD: ISD::SUB, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1910
1911 { .ISD: ISD::MUL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1912 { .ISD: ISD::MUL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1913 { .ISD: ISD::MUL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1914
1915 { .ISD: ISD::FNEG, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // (x87)
1916 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1917 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1918 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1919 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 38, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1920 };
1921
1922 if (const auto *Entry = CostTableLookup(Table: X86CostTbl, ISD, Ty: LT.second))
1923 if (auto KindCost = Entry->Cost[CostKind])
1924 return LT.first * *KindCost;
1925
1926 // It is not a good idea to vectorize division. We have to scalarize it and
1927 // in the process we will often end up having to spilling regular
1928 // registers. The overhead of division is going to dominate most kernels
1929 // anyways so try hard to prevent vectorization of division - it is
1930 // generally a bad idea. Assume somewhat arbitrarily that we have to be able
1931 // to hide "20 cycles" for each lane.
1932 if (CostKind == TTI::TCK_RecipThroughput && LT.second.isVector() &&
1933 (ISD == ISD::SDIV || ISD == ISD::SREM || ISD == ISD::UDIV ||
1934 ISD == ISD::UREM)) {
1935 InstructionCost ScalarCost =
1936 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind,
1937 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1938 return 20 * LT.first * LT.second.getVectorNumElements() * ScalarCost;
1939 }
1940
1941 // Handle some basic single instruction code size cases.
1942 if (CostKind == TTI::TCK_CodeSize) {
1943 switch (ISD) {
1944 case ISD::FADD:
1945 case ISD::FSUB:
1946 case ISD::FMUL:
1947 case ISD::FDIV:
1948 case ISD::FNEG:
1949 case ISD::AND:
1950 case ISD::OR:
1951 case ISD::XOR:
1952 return LT.first;
1953 break;
1954 }
1955 }
1956
1957 // Fallback to the default implementation.
1958 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
1959 Args, CtxI);
1960}
1961
1962InstructionCost
1963X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
1964 unsigned Opcode1, const SmallBitVector &OpcodeMask,
1965 TTI::TargetCostKind CostKind,
1966 ArrayRef<const Value *> Scalars) const {
1967 if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask, Scalars))
1968 return TTI::TCC_Basic;
1969 return InstructionCost::getInvalid();
1970}
1971
1972InstructionCost X86TTIImpl::getShuffleCost(
1973 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1974 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1975 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1976 TTI::VectorInstrContext VIC) const {
1977 assert((Mask.empty() || DstTy->isScalableTy() ||
1978 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1979 "Expected the Mask to match the return size if given");
1980 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1981 "Expected the same scalar types");
1982
1983 // 64-bit packed float vectors (v2f32) are widened to type v4f32.
1984 // 64-bit packed integer vectors (v2i32) are widened to type v4i32.
1985 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1986
1987 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1988
1989 // If all args are constant than this will be constant folded away.
1990 if (!Args.empty() &&
1991 all_of(Range&: Args, P: [](const Value *Arg) { return isa<Constant>(Val: Arg); }))
1992 return TTI::TCC_Free;
1993
1994 // Recognize a basic concat_vector shuffle.
1995 if (Kind == TTI::SK_PermuteTwoSrc &&
1996 Mask.size() == (2 * SrcTy->getElementCount().getKnownMinValue()) &&
1997 ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts: Mask.size()))
1998 return getShuffleCost(Kind: TTI::SK_InsertSubvector,
1999 DstTy: VectorType::getDoubleElementsVectorType(VTy: SrcTy),
2000 SrcTy: VectorType::getDoubleElementsVectorType(VTy: SrcTy),
2001 CostKind, Mask, Index: Mask.size() / 2, SubTp: SrcTy);
2002
2003 // Treat Transpose as 2-op shuffles - there's no difference in lowering.
2004 if (Kind == TTI::SK_Transpose)
2005 if (LT.second != MVT::v4f64 && LT.second != MVT::v4i64)
2006 Kind = TTI::SK_PermuteTwoSrc;
2007
2008 if (Kind == TTI::SK_Broadcast) {
2009 // For Broadcasts we are splatting the first element from the first input
2010 // register, so only need to reference that input and all the output
2011 // registers are the same.
2012 LT.first = 1;
2013
2014 // If we're broadcasting a load then AVX/AVX2 can do this for free.
2015 // If many-used-load whose every use is one of a small set of operations
2016 // that SLP can rewrite into a single vector lane, codegen can fold it into
2017 // the free broadcast.
2018 using namespace PatternMatch;
2019 auto IsBroadcastLoadFoldUser = [&](const User *U) {
2020 if (isa<InsertElementInst>(Val: U) && U->getOperand(i: 1) == Args[0])
2021 return true;
2022 if (U->getType()->isVectorTy())
2023 return false;
2024 // Terminators (return/branch/switch/indirectbr/resume/invoke EH)
2025 // and phis carry the value across control flow.
2026 if (const auto *I = dyn_cast<Instruction>(Val: U))
2027 if (I->isTerminator() ||
2028 isa<PHINode, AtomicRMWInst, AtomicCmpXchgInst>(Val: I))
2029 return false;
2030 // Only pure calls can be folded.
2031 if (const auto *CB = dyn_cast<CallBase>(Val: U))
2032 return CB->doesNotAccessMemory() && !CB->mayHaveSideEffects();
2033 return true;
2034 };
2035 auto IsFoldableSLPBroadcastLoad = [&]() {
2036 if (!match(V: Args[0], P: m_Load(Op: m_Value())))
2037 return false;
2038 auto *FVT = dyn_cast<FixedVectorType>(Val: DstTy);
2039 if (!FVT)
2040 return false;
2041 // getNumUses() counts each Use, matching the per-lane broadcast
2042 // accounting (a use like `op %x, %x` consumes two broadcast lanes).
2043 if (Args[0]->getNumUses() != FVT->getNumElements())
2044 return false;
2045 return all_of(Range: Args[0]->users(), P: IsBroadcastLoadFoldUser);
2046 };
2047 if (!Args.empty() &&
2048 (match(V: Args[0], P: m_OneUse(SubPattern: m_Load(Op: m_Value()))) ||
2049 IsFoldableSLPBroadcastLoad()) &&
2050 (ST->hasAVX2() ||
2051 (ST->hasAVX() && LT.second.getScalarSizeInBits() >= 32)))
2052 return TTI::TCC_Free;
2053 }
2054
2055 // Attempt to detect a cheaper inlane shuffle, avoiding 128-bit subvector
2056 // permutation.
2057 // Attempt to detect a shuffle mask with a single defined element.
2058 bool IsInLaneShuffle = false;
2059 bool IsSingleElementMask = false;
2060 if (SrcTy->getPrimitiveSizeInBits() > 0 &&
2061 (SrcTy->getPrimitiveSizeInBits() % 128) == 0 &&
2062 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
2063 Mask.size() == SrcTy->getElementCount().getKnownMinValue()) {
2064 unsigned NumLanes = SrcTy->getPrimitiveSizeInBits() / 128;
2065 unsigned NumEltsPerLane = Mask.size() / NumLanes;
2066 if ((Mask.size() % NumLanes) == 0) {
2067 IsInLaneShuffle = all_of(Range: enumerate(First&: Mask), P: [&](const auto &P) {
2068 return P.value() == PoisonMaskElem ||
2069 ((P.value() % Mask.size()) / NumEltsPerLane) ==
2070 (P.index() / NumEltsPerLane);
2071 });
2072 IsSingleElementMask =
2073 (Mask.size() - 1) == static_cast<unsigned>(count_if(Range&: Mask, P: [](int M) {
2074 return M == PoisonMaskElem;
2075 }));
2076 }
2077 }
2078
2079 // Treat <X x bfloat> shuffles as <X x half>.
2080 if (LT.second.isVectorOf(EltVT: MVT::bf16))
2081 LT.second = LT.second.changeVectorElementType(EltVT: MVT::f16);
2082
2083 // Subvector extractions are free if they start at the beginning of a
2084 // vector and cheap if the subvectors are aligned.
2085 if (Kind == TTI::SK_ExtractSubvector && LT.second.isVector()) {
2086 int NumElts = LT.second.getVectorNumElements();
2087 if ((Index % NumElts) == 0)
2088 return TTI::TCC_Free;
2089 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
2090 if (SubLT.second.isVector()) {
2091 int NumSubElts = SubLT.second.getVectorNumElements();
2092 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2093 return SubLT.first;
2094 // Handle some cases for widening legalization. For now we only handle
2095 // cases where the original subvector was naturally aligned and evenly
2096 // fit in its legalized subvector type.
2097 // FIXME: Remove some of the alignment restrictions.
2098 // FIXME: We can use permq for 64-bit or larger extracts from 256-bit
2099 // vectors.
2100 int OrigSubElts = cast<FixedVectorType>(Val: SubTp)->getNumElements();
2101 if (NumSubElts > OrigSubElts && (Index % OrigSubElts) == 0 &&
2102 (NumSubElts % OrigSubElts) == 0 &&
2103 LT.second.getVectorElementType() ==
2104 SubLT.second.getVectorElementType() &&
2105 LT.second.getVectorElementType().getSizeInBits() ==
2106 SrcTy->getElementType()->getPrimitiveSizeInBits()) {
2107 assert(NumElts >= NumSubElts && NumElts > OrigSubElts &&
2108 "Unexpected number of elements!");
2109 auto *VecTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2110 NumElts: LT.second.getVectorNumElements());
2111 auto *SubTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2112 NumElts: SubLT.second.getVectorNumElements());
2113 int ExtractIndex = alignDown(Value: (Index % NumElts), Align: NumSubElts);
2114 InstructionCost ExtractCost =
2115 getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: {},
2116 Index: ExtractIndex, SubTp: SubTy);
2117
2118 // If the original size is 32-bits or more, we can use pshufd. Otherwise
2119 // if we have SSSE3 we can use pshufb.
2120 if (SubTp->getPrimitiveSizeInBits() >= 32 || ST->hasSSSE3())
2121 return ExtractCost + 1; // pshufd or pshufb
2122
2123 assert(SubTp->getPrimitiveSizeInBits() == 16 &&
2124 "Unexpected vector size");
2125
2126 return ExtractCost + 2; // worst case pshufhw + pshufd
2127 }
2128 }
2129 // If the extract subvector is not optimal, treat it as single op shuffle.
2130 Kind = TTI::SK_PermuteSingleSrc;
2131 }
2132
2133 // Subvector insertions are cheap if the subvectors are aligned.
2134 // Note that in general, the insertion starting at the beginning of a vector
2135 // isn't free, because we need to preserve the rest of the wide vector,
2136 // but if the destination vector legalizes to the same width as the subvector
2137 // then the insertion will simplify to a (free) register copy.
2138 if (Kind == TTI::SK_InsertSubvector && LT.second.isVector()) {
2139 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: DstTy);
2140 int NumElts = DstLT.second.getVectorNumElements();
2141 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
2142 if (SubLT.second.isVector()) {
2143 int NumSubElts = SubLT.second.getVectorNumElements();
2144 bool MatchingTypes =
2145 NumElts == NumSubElts &&
2146 (SubTp->getElementCount().getKnownMinValue() % NumSubElts) == 0;
2147 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2148 return MatchingTypes ? TTI::TCC_Free : SubLT.first;
2149 }
2150
2151 // Attempt to match MOVSS (Idx == 0) or INSERTPS pattern. This will have
2152 // been matched by improveShuffleKindFromMask as a SK_InsertSubvector of
2153 // v1f32 (legalised to f32) into a v4f32.
2154 if (LT.first == 1 && LT.second == MVT::v4f32 && SubLT.first == 1 &&
2155 SubLT.second == MVT::f32 && (Index == 0 || ST->hasSSE41()))
2156 return 1;
2157
2158 // If the insertion is the lowest subvector then it will be blended
2159 // otherwise treat it like a 2-op shuffle.
2160 Kind =
2161 (Index == 0 && LT.first == 1) ? TTI::SK_Select : TTI::SK_PermuteTwoSrc;
2162 }
2163
2164 // Handle some common (illegal) sub-vector types as they are often very cheap
2165 // to shuffle even on targets without PSHUFB.
2166 EVT VT = TLI->getValueType(DL, Ty: SrcTy);
2167 if (VT.isSimple() && VT.isVector() && VT.getSizeInBits() < 128 &&
2168 !ST->hasSSSE3()) {
2169 static const CostKindTblEntry SSE2SubVectorShuffleTbl[] = {
2170 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2171 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2172 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2173 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2174 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2175
2176 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2177 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2178 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // punpck/pshuflw/packus
2179 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2180
2181 {.ISD: TTI::SK_Splice, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2182 {.ISD: TTI::SK_Splice, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2183 {.ISD: TTI::SK_Splice, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2184 {.ISD: TTI::SK_Splice, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2185
2186 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2187 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2188 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 7,.LatencyCost: 7,.CodeSizeCost: 7,.SizeAndLatencyCost: 7}}, // punpck/pshuflw
2189 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // punpck/pshuflw
2190 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck
2191
2192 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2193 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2194 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // punpck/pshuflw
2195 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // punpck/pshuflw
2196 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2197 };
2198
2199 if (ST->hasSSE2())
2200 if (const auto *Entry =
2201 CostTableLookup(Table: SSE2SubVectorShuffleTbl, ISD: Kind, Ty: VT.getSimpleVT()))
2202 if (auto KindCost = Entry->Cost[CostKind])
2203 return LT.first * *KindCost;
2204 }
2205
2206 // We are going to permute multiple sources and the result will be in multiple
2207 // destinations. Providing an accurate cost only for splits where the element
2208 // type remains the same.
2209 if (LT.first != 1) {
2210 MVT LegalVT = LT.second;
2211 if (LegalVT.isVector() &&
2212 LegalVT.getVectorElementType().getSizeInBits() ==
2213 SrcTy->getElementType()->getPrimitiveSizeInBits() &&
2214 LegalVT.getVectorNumElements() <
2215 cast<FixedVectorType>(Val: SrcTy)->getNumElements()) {
2216 unsigned VecTySize = DL.getTypeStoreSize(Ty: SrcTy);
2217 unsigned LegalVTSize = LegalVT.getStoreSize();
2218 // Number of source vectors after legalization:
2219 unsigned NumOfSrcs = (VecTySize + LegalVTSize - 1) / LegalVTSize;
2220 // Number of destination vectors after legalization:
2221 InstructionCost NumOfDests = LT.first;
2222
2223 auto *SingleOpTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2224 NumElts: LegalVT.getVectorNumElements());
2225
2226 if (!Mask.empty() && NumOfDests.isValid()) {
2227 // Try to perform better estimation of the permutation.
2228 // 1. Split the source/destination vectors into real registers.
2229 // 2. Do the mask analysis to identify which real registers are
2230 // permuted. If more than 1 source registers are used for the
2231 // destination register building, the cost for this destination register
2232 // is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
2233 // source register is used, build mask and calculate the cost as a cost
2234 // of PermuteSingleSrc.
2235 // Also, for the single register permute we try to identify if the
2236 // destination register is just a copy of the source register or the
2237 // copy of the previous destination register (the cost is
2238 // TTI::TCC_Basic). If the source register is just reused, the cost for
2239 // this operation is TTI::TCC_Free.
2240 NumOfDests =
2241 getTypeLegalizationCost(
2242 Ty: FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: Mask.size()))
2243 .first;
2244 unsigned E = NumOfDests.getValue();
2245 unsigned NormalizedVF =
2246 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
2247 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
2248 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
2249 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
2250 copy(Range&: Mask, Out: NormalizedMask.begin());
2251 unsigned PrevSrcReg = 0;
2252 ArrayRef<int> PrevRegMask;
2253 InstructionCost Cost = 0;
2254 processShuffleMasks(
2255 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
2256 SingleInputAction: [this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
2257 &Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
2258 if (!ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size())) {
2259 // Check if the previous register can be just copied to the next
2260 // one.
2261 if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
2262 PrevRegMask != RegMask)
2263 Cost +=
2264 getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
2265 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
2266 else
2267 // Just a copy of previous destination register.
2268 Cost += TTI::TCC_Basic;
2269 return;
2270 }
2271 if (SrcReg != DestReg &&
2272 any_of(Range&: RegMask, P: not_equal_to(Arg: PoisonMaskElem))) {
2273 // Just a copy of the source register.
2274 Cost += TTI::TCC_Free;
2275 }
2276 PrevSrcReg = SrcReg;
2277 PrevRegMask = RegMask;
2278 },
2279 ManyInputsAction: [this, SingleOpTy, CostKind,
2280 &Cost](ArrayRef<int> RegMask, unsigned /*Unused*/,
2281 unsigned /*Unused*/, bool /*Unused*/) {
2282 Cost += getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
2283 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
2284 });
2285 return Cost;
2286 }
2287
2288 InstructionCost NumOfShuffles = (NumOfSrcs - 1) * NumOfDests;
2289 return NumOfShuffles * getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
2290 SrcTy: SingleOpTy, CostKind, Mask: {}, Index: 0,
2291 SubTp: nullptr);
2292 }
2293
2294 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2295 SubTp);
2296 }
2297
2298 // If we're just moving a single element around (probably as an alternative to
2299 // extracting it), we can assume this is cheap.
2300 if (LT.first == 1 && IsInLaneShuffle && IsSingleElementMask)
2301 return TTI::TCC_Basic;
2302
2303 static const CostKindTblEntry AVX512VBMIShuffleTbl[] = {
2304 { .ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2305 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2306 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2307 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2308 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2b
2309 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2b
2310 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } } // vpermt2b
2311 };
2312
2313 if (ST->hasVBMI())
2314 if (const auto *Entry =
2315 CostTableLookup(Table: AVX512VBMIShuffleTbl, ISD: Kind, Ty: LT.second))
2316 if (auto KindCost = Entry->Cost[CostKind])
2317 return LT.first * *KindCost;
2318
2319 static const CostKindTblEntry AVX512BWShuffleTbl[] = {
2320 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2321 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2322 { .ISD: TTI::SK_Broadcast, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2323
2324 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermw
2325 { .ISD: TTI::SK_Reverse, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermw
2326 { .ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2327 { .ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2328 { .ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pshufb + vshufi64x2
2329
2330 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2331 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2332 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2333 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2334 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // extend to v32i16
2335
2336 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2337 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32f16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2338 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2339 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2340 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 19, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } }, // 6 * v32i8 + 1
2341
2342 { .ISD: TTI::SK_Select, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmw
2343 { .ISD: TTI::SK_Select, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmb
2344
2345 { .ISD: TTI::SK_Splice, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2346 { .ISD: TTI::SK_Splice, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2347 { .ISD: TTI::SK_Splice, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2348 };
2349
2350 if (ST->hasBWI())
2351 if (const auto *Entry =
2352 CostTableLookup(Table: AVX512BWShuffleTbl, ISD: Kind, Ty: LT.second))
2353 if (auto KindCost = Entry->Cost[CostKind])
2354 return LT.first * *KindCost;
2355
2356 static const CostKindTblEntry AVX512InLaneShuffleTbl[] = {
2357 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2358 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2359 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2360 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2361 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2362 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2363 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2364 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2365 };
2366
2367 if (IsInLaneShuffle && ST->hasAVX512())
2368 if (const auto *Entry =
2369 CostTableLookup(Table: AVX512InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2370 if (auto KindCost = Entry->Cost[CostKind])
2371 return LT.first * *KindCost;
2372
2373 static const CostKindTblEntry AVX512ShuffleTbl[] = {
2374 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastsd
2375 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastsd
2376 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastss
2377 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastss
2378 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastq
2379 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastq
2380 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastd
2381 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastd
2382 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2383 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2384 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2385 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2386 {.ISD: TTI::SK_Broadcast, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2387 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 }}, // vpbroadcastb
2388
2389 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermpd
2390 {.ISD: TTI::SK_Reverse, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermps
2391 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermq
2392 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermd
2393 {.ISD: TTI::SK_Reverse, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2394 {.ISD: TTI::SK_Reverse, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2395 {.ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2396
2397 {.ISD: TTI::SK_Splice, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2398 {.ISD: TTI::SK_Splice, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2399 {.ISD: TTI::SK_Splice, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2400 {.ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2401 {.ISD: TTI::SK_Splice, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2402 {.ISD: TTI::SK_Splice, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2403 {.ISD: TTI::SK_Splice, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2404 {.ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2405 {.ISD: TTI::SK_Splice, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2406 {.ISD: TTI::SK_Splice, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2407 {.ISD: TTI::SK_Splice, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2408
2409 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2410 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2411 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2412 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2413 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2414 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2415 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2416 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2417 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2418 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2419 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2420 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2421 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pshufb
2422
2423 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2pd
2424 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2ps
2425 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2q
2426 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2d
2427 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2pd
2428 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2ps
2429 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2q
2430 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2d
2431 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2432 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2433 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2434 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2435
2436 // FIXME: This just applies the type legalization cost rules above
2437 // assuming these completely split.
2438 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2439 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2440 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2441 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2442 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2443 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2444
2445 {.ISD: TTI::SK_Select, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2446 {.ISD: TTI::SK_Select, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2447 {.ISD: TTI::SK_Select, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2448 {.ISD: TTI::SK_Select, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmpd
2449 {.ISD: TTI::SK_Select, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmps
2450 {.ISD: TTI::SK_Select, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmq
2451 {.ISD: TTI::SK_Select, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmd
2452 };
2453
2454 if (ST->hasAVX512())
2455 if (const auto *Entry = CostTableLookup(Table: AVX512ShuffleTbl, ISD: Kind, Ty: LT.second))
2456 if (auto KindCost = Entry->Cost[CostKind])
2457 return LT.first * *KindCost;
2458
2459 static const CostKindTblEntry AVX2InLaneShuffleTbl[] = {
2460 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2461 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2462 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2463
2464 { .ISD: TTI::SK_Transpose, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2465 { .ISD: TTI::SK_Transpose, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2466
2467 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufpd + vblendpd
2468 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufps + vblendps
2469 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufd + vpblendd
2470 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufd + vpblendd
2471 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2472 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2473 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2474 };
2475
2476 if (IsInLaneShuffle && ST->hasAVX2())
2477 if (const auto *Entry =
2478 CostTableLookup(Table: AVX2InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2479 if (auto KindCost = Entry->Cost[CostKind])
2480 return LT.first * *KindCost;
2481
2482 static const CostKindTblEntry AVX2ShuffleTbl[] = {
2483 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vbroadcastpd
2484 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vbroadcastps
2485 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastq
2486 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastd
2487 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastw
2488 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2489 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastw
2490 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2491 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastb
2492 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2493
2494 { .ISD: TTI::SK_Reverse, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpermpd
2495 { .ISD: TTI::SK_Reverse, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermps
2496 { .ISD: TTI::SK_Reverse, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpermq
2497 { .ISD: TTI::SK_Reverse, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermd
2498 { .ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2499 { .ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2500 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2501
2502 { .ISD: TTI::SK_Select, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2503 { .ISD: TTI::SK_Select, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2504 { .ISD: TTI::SK_Select, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2505
2506 { .ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2507 { .ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2508 { .ISD: TTI::SK_Splice, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2509 { .ISD: TTI::SK_Splice, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2510 { .ISD: TTI::SK_Splice, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2511
2512 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2513 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2514 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2515 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2516 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2517 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2518 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2519
2520 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermpd + vblendpd
2521 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermps + vblendps
2522 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermq + vpblendd
2523 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermd + vpblendd
2524 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2525 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2526 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2527 };
2528
2529 if (ST->hasAVX2())
2530 if (const auto *Entry = CostTableLookup(Table: AVX2ShuffleTbl, ISD: Kind, Ty: LT.second))
2531 if (auto KindCost = Entry->Cost[CostKind])
2532 return LT.first * *KindCost;
2533
2534 static const CostKindTblEntry XOPShuffleTbl[] = {
2535 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2pd
2536 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2ps
2537 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2pd
2538 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2ps
2539 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*vpperm
2540 // + vinsertf128
2541 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*vpperm
2542 // + vinsertf128
2543
2544 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 6*vpperm
2545 // + vinsertf128
2546
2547 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpperm
2548 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 6*vpperm
2549 // + vinsertf128
2550 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpperm
2551 };
2552
2553 if (ST->hasXOP())
2554 if (const auto *Entry = CostTableLookup(Table: XOPShuffleTbl, ISD: Kind, Ty: LT.second))
2555 if (auto KindCost = Entry->Cost[CostKind])
2556 return LT.first * *KindCost;
2557
2558 static const CostKindTblEntry AVX1InLaneShuffleTbl[] = {
2559 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilpd
2560 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilpd
2561 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilps
2562 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilps
2563
2564 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2565 // + vpor + vinsertf128
2566 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2567 // + vpor + vinsertf128
2568 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2569 // + vpor + vinsertf128
2570
2571 { .ISD: TTI::SK_Transpose, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2572 { .ISD: TTI::SK_Transpose, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2573
2574 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufpd + vblendpd
2575 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufps + vblendps
2576 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpermilpd + vblendpd
2577 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpermilps + vblendps
2578 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2579 // + 2*vpor + vinsertf128
2580 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2581 // + 2*vpor + vinsertf128
2582 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2583 // + 2*vpor + vinsertf128
2584 };
2585
2586 if (IsInLaneShuffle && ST->hasAVX())
2587 if (const auto *Entry =
2588 CostTableLookup(Table: AVX1InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2589 if (auto KindCost = Entry->Cost[CostKind])
2590 return LT.first * *KindCost;
2591
2592 static const CostKindTblEntry AVX1ShuffleTbl[] = {
2593 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilpd
2594 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilps
2595 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilpd
2596 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilps
2597 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 4}}, // vpshuflw + vpshufd + vinsertf128
2598 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 4}}, // vpshuflw + vpshufd + vinsertf128
2599 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 4,.CodeSizeCost: 3,.SizeAndLatencyCost: 6}}, // vpshufb + vinsertf128
2600
2601 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 6,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vpermilpd
2602 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 7,.CodeSizeCost: 2,.SizeAndLatencyCost: 4}}, // vperm2f128 + vpermilps
2603 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 6,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vpermilpd
2604 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 7,.CodeSizeCost: 2,.SizeAndLatencyCost: 4}}, // vperm2f128 + vpermilps
2605 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2606 // + vinsertf128
2607 {.ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2608 // + vinsertf128
2609 {.ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2610 // + vinsertf128
2611
2612 {.ISD: TTI::SK_Select, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendpd
2613 {.ISD: TTI::SK_Select, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendpd
2614 {.ISD: TTI::SK_Select, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendps
2615 {.ISD: TTI::SK_Select, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendps
2616 {.ISD: TTI::SK_Select, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2617 {.ISD: TTI::SK_Select, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2618 {.ISD: TTI::SK_Select, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2619
2620 {.ISD: TTI::SK_Splice, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + shufpd
2621 {.ISD: TTI::SK_Splice, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + shufpd
2622 {.ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2623 {.ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2624 {.ISD: TTI::SK_Splice, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2625 {.ISD: TTI::SK_Splice, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2626 {.ISD: TTI::SK_Splice, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2627
2628 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vshufpd
2629 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vshufpd
2630 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2631 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2632 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16,.Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2633 // + 2*por + vinsertf128
2634 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16,.Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2635 // + 2*por + vinsertf128
2636 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2637 // + 2*por + vinsertf128
2638
2639 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // 2*vperm2f128 + vshufpd
2640 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // 2*vperm2f128 + vshufpd
2641 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2642 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2643 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16,.Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2644 // + 4*por + vinsertf128
2645 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16,.Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2646 // + 4*por + vinsertf128
2647 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2648 // + 4*por + vinsertf128
2649 };
2650
2651 if (ST->hasAVX())
2652 if (const auto *Entry = CostTableLookup(Table: AVX1ShuffleTbl, ISD: Kind, Ty: LT.second))
2653 if (auto KindCost = Entry->Cost[CostKind])
2654 return LT.first * *KindCost;
2655
2656 static const CostKindTblEntry SSE41ShuffleTbl[] = {
2657 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2658 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // movsd
2659 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2660 {.ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // blendps
2661 {.ISD: TTI::SK_Select, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2662 {.ISD: TTI::SK_Select, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2663 {.ISD: TTI::SK_Select, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}} // pblendvb
2664 };
2665
2666 if (ST->hasSSE41())
2667 if (const auto *Entry = CostTableLookup(Table: SSE41ShuffleTbl, ISD: Kind, Ty: LT.second))
2668 if (auto KindCost = Entry->Cost[CostKind])
2669 return LT.first * *KindCost;
2670
2671 static const CostKindTblEntry SSSE3ShuffleTbl[] = {
2672 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2673 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2674 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2675
2676 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2677 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2678 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2679
2680 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2681 {.ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2682 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2683 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2684 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2685
2686 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2687 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2688 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2689
2690 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2691 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2692 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2693 };
2694
2695 if (ST->hasSSSE3())
2696 if (const auto *Entry = CostTableLookup(Table: SSSE3ShuffleTbl, ISD: Kind, Ty: LT.second))
2697 if (auto KindCost = Entry->Cost[CostKind])
2698 return LT.first * *KindCost;
2699
2700 static const CostKindTblEntry SSE2ShuffleTbl[] = {
2701 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2702 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2703 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2704 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshuflw + pshufd
2705 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshuflw + pshufd
2706 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 4}}, // unpck + pshuflw + pshufd
2707
2708 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2709 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2710 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2711 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pshuflw + pshufhw + pshufd
2712 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pshuflw + pshufhw + pshufd
2713 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 6,.CodeSizeCost: 11,.SizeAndLatencyCost: 11}}, // 2*pshuflw + 2*pshufhw
2714 // + 2*pshufd + 2*unpck + packus
2715
2716 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // movsd
2717 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // movsd
2718 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*shufps
2719 {.ISD: TTI::SK_Select, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2720 {.ISD: TTI::SK_Select, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2721 {.ISD: TTI::SK_Select, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2722
2723 {.ISD: TTI::SK_Splice, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2724 {.ISD: TTI::SK_Splice, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2725 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*{unpck,movsd,pshufd}
2726 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2727 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2728 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2729
2730 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2731 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2732 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2733 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}}, // 2*pshuflw + 2*pshufhw
2734 // + pshufd/unpck
2735 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}}, // 2*pshuflw + 2*pshufhw
2736 // + pshufd/unpck
2737 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 8, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 10}}, // 2*pshuflw + 2*pshufhw
2738 // + 2*pshufd + 2*unpck + 2*packus
2739
2740 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2741 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2742 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*{unpck,movsd,pshufd}
2743 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}}, // blend+permute
2744 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}}, // blend+permute
2745 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 13, .CodeSizeCost: 13, .SizeAndLatencyCost: 13}}, // blend+permute
2746 };
2747
2748 static const CostTblEntry SSE3BroadcastLoadTbl[] = {
2749 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: 0}, // broadcast handled by movddup
2750 };
2751
2752 if (ST->hasSSE2()) {
2753 bool IsLoad =
2754 llvm::any_of(Range&: Args, P: [](const auto &V) { return isa<LoadInst>(V); });
2755 if (ST->hasSSE3() && IsLoad)
2756 if (const auto *Entry =
2757 CostTableLookup(Table: SSE3BroadcastLoadTbl, ISD: Kind, Ty: LT.second)) {
2758 assert(isLegalBroadcastLoad(SrcTy->getElementType(),
2759 LT.second.getVectorElementCount()) &&
2760 "Table entry missing from isLegalBroadcastLoad()");
2761 return LT.first * Entry->Cost;
2762 }
2763
2764 if (const auto *Entry = CostTableLookup(Table: SSE2ShuffleTbl, ISD: Kind, Ty: LT.second))
2765 if (auto KindCost = Entry->Cost[CostKind])
2766 return LT.first * *KindCost;
2767 }
2768
2769 static const CostKindTblEntry SSE1ShuffleTbl[] = {
2770 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2771 { .ISD: TTI::SK_Reverse, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2772 { .ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2773 { .ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2774 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2775 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2776 };
2777
2778 if (ST->hasSSE1()) {
2779 if (LT.first == 1 && LT.second == MVT::v4f32 && Mask.size() == 4) {
2780 // SHUFPS: both pairs must come from the same source register.
2781 auto MatchSHUFPS = [](int X, int Y) {
2782 return X < 0 || Y < 0 || ((X & 4) == (Y & 4));
2783 };
2784 if (MatchSHUFPS(Mask[0], Mask[1]) && MatchSHUFPS(Mask[2], Mask[3]))
2785 return 1;
2786 }
2787 if (const auto *Entry = CostTableLookup(Table: SSE1ShuffleTbl, ISD: Kind, Ty: LT.second))
2788 if (auto KindCost = Entry->Cost[CostKind])
2789 return LT.first * *KindCost;
2790 }
2791
2792 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2793 SubTp);
2794}
2795
2796InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
2797 Type *Src,
2798 TTI::CastContextHint CCH,
2799 TTI::TargetCostKind CostKind,
2800 const Instruction *I) const {
2801 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2802 assert(ISD && "Invalid opcode");
2803
2804 // A narrow (i8/i16) zero-extension used as a GEP *index* can be folded into
2805 // the addressing mode of the consuming memory op, but only if the source is
2806 // already materialised zero-extended in a full register. X86's SIB form
2807 // [base + index*scale + disp] reads the index at full width and does NOT
2808 // zero-extend a narrow index (unlike AArch64's uxtw-extended addressing), so
2809 // a "dirty" narrow source (e.g. an i16 add result used only as an index)
2810 // still needs a dedicated movzx and is not free. Price it as free only with
2811 // positive evidence that no movzx is required.
2812 if (ISD == ISD::ZERO_EXTEND && I && I->hasOneUse() && Src->isIntegerTy() &&
2813 Src->getScalarSizeInBits() < 32) {
2814 const Use &U = *I->use_begin();
2815 if (isa<GetElementPtrInst>(Val: U.getUser()) &&
2816 U.getOperandNo() != GetElementPtrInst::getPointerOperandIndex()) {
2817 const Value *Op = I->getOperand(i: 0);
2818 // Clean sources: an extending load, a zeroext argument, or a value whose
2819 // high bits are provably zero (e.g. from a shift/mask). These mirror the
2820 // proof-based reasoning the middle end uses elsewhere (ValueTracking and
2821 // InstCombine's canEvaluateZExtd); we intentionally do NOT treat a merely
2822 // multiply-used operand as clean, since that is a guess rather than
2823 // proof.
2824 if (isa<LoadInst>(Val: Op))
2825 return TTI::TCC_Free;
2826 if (const auto *A = dyn_cast<Argument>(Val: Op))
2827 if (A->hasAttribute(Kind: Attribute::ZExt))
2828 return TTI::TCC_Free;
2829 if (computeKnownBits(V: Op, DL: I->getDataLayout(), /*AC=*/nullptr, CtxI: I)
2830 .countMinLeadingZeros() > 0)
2831 return TTI::TCC_Free;
2832 }
2833 }
2834
2835 // The cost tables include both specific, custom (non-legal) src/dst type
2836 // conversions and generic, legalized types. We test for customs first, before
2837 // falling back to legalization.
2838 // FIXME: Need a better design of the cost table to handle non-simple types of
2839 // potential massive combinations (elem_num x src_type x dst_type).
2840 static const TypeConversionCostKindTblEntry AVX512BWConversionTbl[]{
2841 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2842 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2843
2844 // Mask sign extend has an instruction.
2845 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2846 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2847 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2848 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2849 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2850 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2851 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2852 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2853 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2854 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2855 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2856 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2857 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2858 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2859 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2860 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v64i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2861 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2862
2863 // Mask zero extend is a sext + shift.
2864 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2865 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2866 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2867 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2868 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2869 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2870 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2871 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2872 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2873 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2874 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2875 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2876 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2877 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2878 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2879 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v64i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2880 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2881
2882 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2883 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2884 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2885 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2886 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2887 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2888 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2889 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2890 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2891 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2892 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2893 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2894 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2895 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2896 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2897 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2898 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2899
2900 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2901 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // widen to zmm
2902 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2903 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2904 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2905 };
2906
2907 static const TypeConversionCostKindTblEntry AVX512DQConversionTbl[] = {
2908 // Mask sign extend has an instruction.
2909 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2910 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2911 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2912 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2913 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2914 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2915 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2916 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2917
2918 // Mask zero extend is a sext + shift.
2919 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2920 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2921 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2922 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2923 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2924 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2925 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2926 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2927
2928 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2929 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2930 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2931 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2932 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2933 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2934 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2935 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2936
2937 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2938 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2939
2940 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2941 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2942
2943 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2944 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i64, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2945
2946 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2947 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i64, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2948 };
2949
2950 // TODO: For AVX512DQ + AVX512VL, we also have cheap casts for 128-bit and
2951 // 256-bit wide vectors.
2952
2953 static const TypeConversionCostKindTblEntry AVX512FConversionTbl[] = {
2954 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2955 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2956 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v16f64, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // 2*vcvtps2pd+vextractf64x4
2957 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v16f32, .Src: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps
2958 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
2959 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2960 { .ISD: ISD::FP_ROUND, .Dst: MVT::v16f16, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtps2ph
2961
2962 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2963 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2964 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2965 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2966 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2967 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2968 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2969 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2970 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2971 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2972 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2973 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
2974 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpsllq+vptestmq
2975 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpsllq+vptestmq
2976 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
2977 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2978 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2979 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2980 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2981 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2982 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdw
2983 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdw
2984 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2985 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2986 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2987 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2988 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2989 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2990 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2991 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2992 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2993 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqd
2994 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpmovqd
2995 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },// 2*vpmovqd+concat+vpmovdb
2996
2997 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // extend to v16i32
2998 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2999 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3000
3001 // Sign extend is zmm vpternlogd+vptruncdb.
3002 // Zero extend is zmm broadcast load+vptruncdw.
3003 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3004 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3005 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3006 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3007 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3008 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3009 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3010 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3011
3012 // Sign extend is zmm vpternlogd+vptruncdw.
3013 // Zero extend is zmm vpternlogd+vptruncdw+vpsrlw.
3014 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3015 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3016 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3017 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3018 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3019 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3020 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3021 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3022
3023 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3024 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3025 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3026 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3027 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3028 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3029 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq
3030 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq+psrlq
3031 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq
3032 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq+psrlq
3033
3034 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3035 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3036 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3037 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3038
3039 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3040 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3041 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3042 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3043 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3044 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3045 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3046 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3047 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3048 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3049
3050 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // FIXME: May not be right
3051 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // FIXME: May not be right
3052
3053 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3054 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3055 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3056 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3057 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3058 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3059 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3060 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3061
3062 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3063 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3064 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3065 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3066 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3067 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3068 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3069 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3070 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: {.RecipThroughputCost: 26, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3071 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3072
3073 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3074 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3075 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v32f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3076 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v64i8, .Src: MVT::v64f32, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3077 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v64i8, .Src: MVT::v64f64, .Cost: {.RecipThroughputCost: 31, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3078 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3079 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3080 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i16, .Src: MVT::v32f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3081 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i16, .Src: MVT::v32f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3082 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3083 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i32, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3084
3085 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3086 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3087 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i8, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3088 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i32, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3089 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3090 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3091 };
3092
3093 static const TypeConversionCostKindTblEntry AVX512BWVLConversionTbl[] {
3094 // Mask sign extend has an instruction.
3095 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3096 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3097 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3098 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3099 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3100 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3101 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3102 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3103 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3104 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3105 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3106 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3107 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3108 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3109 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3110 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3111 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3112
3113 // Mask zero extend is a sext + shift.
3114 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3115 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3116 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3117 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3118 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3119 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3120 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3121 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3122 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3123 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3124 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3125 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3126 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3127 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3128 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3129 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3130 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3131
3132 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3133 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3134 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3135 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3136 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3137 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3138 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3139 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3140 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3141 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3142 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3143 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3144 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3145 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3146 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3147 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3148 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3149
3150 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3151 };
3152
3153 static const TypeConversionCostKindTblEntry AVX512DQVLConversionTbl[] = {
3154 // Mask sign extend has an instruction.
3155 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3156 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3157 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3158 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3159 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3160 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3161 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3162 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3163
3164 // Mask zero extend is a sext + shift.
3165 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3166 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3167 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3168 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3169 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3170 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3171 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3172 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3173
3174 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3175 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3176 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3177 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3178 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3179 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3180 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3181 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3182
3183 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3184 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3185 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3186 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3187
3188 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3189 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3190 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3191 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3192
3193 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3194 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3195 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3196 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i64, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3197
3198 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3199 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3200 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3201 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i64, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3202 };
3203
3204 static const TypeConversionCostKindTblEntry AVX512VLConversionTbl[] = {
3205 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3206 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3207 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3208 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // split+2*v8i8
3209 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3210 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3211 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3212 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // split+2*v8i16
3213 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3214 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3215 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3216 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3217 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
3218 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
3219 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqd
3220 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
3221 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
3222 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
3223
3224 // sign extend is vpcmpeq+maskedmove+vpmovdw+vpacksswb
3225 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw+vpackuswb
3226 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3227 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3228 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3229 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3230 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3231 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3232 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3233 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3234
3235 // sign extend is vpcmpeq+maskedmove+vpmovdw
3236 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw
3237 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3238 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3239 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3240 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3241 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3242 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3243 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3244 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3245
3246 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3247 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3248 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3249 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3250 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3251 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3252 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3253 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3254
3255 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3256 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3257 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3258 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3259
3260 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3261 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3262 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3263 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3264 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3265 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3266 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3267 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3268 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3269 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3270 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3271 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3272
3273 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3274 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3275 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3276 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3277
3278 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3279 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3280 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3281 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3282 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3283 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3284 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3285 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3286 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3287 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3288 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3289 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3290 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3291
3292 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3293 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3294 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v32f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3295
3296 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3297 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3298 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3299 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3300 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3301 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3302 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3303 };
3304
3305 static const TypeConversionCostKindTblEntry AVX2ConversionTbl[] = {
3306 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3307 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3308 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3309 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3310 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3311 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3312
3313 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3314 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3315 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3316 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3317 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3318 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3319 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3320 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3321 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3322 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3323 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3324 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3325 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3326 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3327
3328 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3329
3330 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3331 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3332 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3333 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3334 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3335 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3336 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3337 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3338 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3339 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3340 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3341 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3342
3343 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3344 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3345
3346 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3347 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3348 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3349 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3350
3351 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3352 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3353 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3354 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3355 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3356 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3357 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3358 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3359
3360 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3361 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3362 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3363 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3364 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3365 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3366 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3367
3368 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3369 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3370 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3371 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3372 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3373 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3374 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3375 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3376 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3377 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3378 };
3379
3380 static const TypeConversionCostKindTblEntry AVXConversionTbl[] = {
3381 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3382 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3383 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3384 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3385 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3386 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3387
3388 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3389 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3390 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3391 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3392 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3393 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3394 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3395 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3396 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3397 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3398 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3399 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3400
3401 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3402 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3403 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3404 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3405 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3406
3407 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3408 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3409 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // and+extract+packuswb
3410 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3411 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3412 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3413 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // and+extract+2*packusdw
3414 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3415
3416 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3417 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3418 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3419 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3420 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3421 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3422 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3423 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3424 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3425 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3426 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3427 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3428
3429 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3430 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3431 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3432 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3433 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3434 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3435 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3436 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3437 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3438 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3439 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3440 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3441 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3442 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3443 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 18, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3444 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3445 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3446
3447 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3448 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3449 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3450 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3451 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3452 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3453 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3454 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3455 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3456 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3457 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3458
3459 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3460 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3461 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v32i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3462 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v32i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3463 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3464 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3465 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3466 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3467 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3468 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3469 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3470 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3471 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3472
3473 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3474 { .ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3475 };
3476
3477 static const TypeConversionCostKindTblEntry SSE41ConversionTbl[] = {
3478 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3479 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3480 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3481 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3482 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3483 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3484 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3485 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3486 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3487 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3488 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3489 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3490
3491 // These truncates end up widening elements.
3492 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZBQ
3493 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZWQ
3494 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZBD
3495
3496 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3497 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3498 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3499
3500 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3501 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3502 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3503 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3504 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3505 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3506 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3507 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3508 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3509 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3510 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3511
3512 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3513 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3514 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3515 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3516 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3517 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3518 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3519 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3520 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3521 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3522 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3523 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3524 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 22, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3525 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3526
3527 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3528 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3529 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3530 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3531 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3532 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3533 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3534 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3535 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3536 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3537
3538 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3539 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3540 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3541 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3542 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3543 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3544 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3545 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3546 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3547 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3548 };
3549
3550 static const TypeConversionCostKindTblEntry SSE2ConversionTbl[] = {
3551 // These are somewhat magic numbers justified by comparing the
3552 // output of llvm-mca for our various supported scheduler models
3553 // and basing it off the worst case scenario.
3554 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3555 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3556 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3557 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3558 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3559 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3560 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3561 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3562 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3563 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3564 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3565 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3566
3567 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3568 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3569 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3570 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3571 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3572 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3573 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3574 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3575 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3576 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3577 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3578 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3579 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 18, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3580
3581 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3582 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3583 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3584 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3585 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3586 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3587 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3588 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3589 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3590 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3591
3592 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3593 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3594 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3595 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3596 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3597 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3598 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3599 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3600 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3601 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3602
3603 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3604 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3605 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3606 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3607 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3608 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3609 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3610 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3611 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3612 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3613 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3614 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3615
3616 // These truncates are really widening elements.
3617 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD
3618 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLWD+DQ
3619 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW+WD+PSHUFD
3620 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLWD
3621 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW+WD
3622 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW
3623
3624 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+PACKUSWB
3625 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3626 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+2*PACKUSWB
3627 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3628 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3629 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3630 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3631 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3632 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+3*PACKUSWB
3633 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD+PSHUFLW
3634 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD
3635 };
3636
3637 static const TypeConversionCostKindTblEntry F16ConversionTbl[] = {
3638 { .ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3639 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3640 { .ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3641 { .ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3642 { .ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
3643 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3644 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3645 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
3646 };
3647
3648 // Attempt to map directly to (simple) MVT types to let us match custom entries.
3649 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
3650 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
3651
3652 // If we're sign-extending a vector comparison result back to the comparison
3653 // width, this will be free without AVX512 (or for 8/16-bit types without
3654 // BWI).
3655 if (!ST->hasAVX512() || (!ST->hasBWI() && DstTy.getScalarSizeInBits() < 32)) {
3656 if (I && Opcode == Instruction::CastOps::SExt &&
3657 SrcTy.isFixedLengthVectorOf(EltVT: MVT::i1)) {
3658 if (auto *CmpI = dyn_cast<CmpInst>(Val: I->getOperand(i: 0))) {
3659 Type *CmpTy = CmpI->getOperand(i_nocapture: 0)->getType();
3660 if (CmpTy->getScalarSizeInBits() == DstTy.getScalarSizeInBits())
3661 return TTI::TCC_Free;
3662 }
3663 }
3664 }
3665
3666 // The function getSimpleVT only handles simple value types.
3667 if (SrcTy.isSimple() && DstTy.isSimple()) {
3668 MVT SimpleSrcTy = SrcTy.getSimpleVT();
3669 MVT SimpleDstTy = DstTy.getSimpleVT();
3670
3671 if (ST->useAVX512Regs()) {
3672 if (ST->hasBWI())
3673 if (const auto *Entry = ConvertCostTableLookup(
3674 Table: AVX512BWConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3675 if (auto KindCost = Entry->Cost[CostKind])
3676 return *KindCost;
3677
3678 if (ST->hasDQI())
3679 if (const auto *Entry = ConvertCostTableLookup(
3680 Table: AVX512DQConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3681 if (auto KindCost = Entry->Cost[CostKind])
3682 return *KindCost;
3683
3684 if (ST->hasAVX512())
3685 if (const auto *Entry = ConvertCostTableLookup(
3686 Table: AVX512FConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3687 if (auto KindCost = Entry->Cost[CostKind])
3688 return *KindCost;
3689 }
3690
3691 if (ST->hasBWI())
3692 if (const auto *Entry = ConvertCostTableLookup(
3693 Table: AVX512BWVLConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3694 if (auto KindCost = Entry->Cost[CostKind])
3695 return *KindCost;
3696
3697 if (ST->hasDQI())
3698 if (const auto *Entry = ConvertCostTableLookup(
3699 Table: AVX512DQVLConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3700 if (auto KindCost = Entry->Cost[CostKind])
3701 return *KindCost;
3702
3703 if (ST->hasAVX512())
3704 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512VLConversionTbl, ISD,
3705 Dst: SimpleDstTy, Src: SimpleSrcTy))
3706 if (auto KindCost = Entry->Cost[CostKind])
3707 return *KindCost;
3708
3709 if (ST->hasAVX2()) {
3710 if (const auto *Entry = ConvertCostTableLookup(Table: AVX2ConversionTbl, ISD,
3711 Dst: SimpleDstTy, Src: SimpleSrcTy))
3712 if (auto KindCost = Entry->Cost[CostKind])
3713 return *KindCost;
3714 }
3715
3716 if (ST->hasAVX()) {
3717 if (const auto *Entry = ConvertCostTableLookup(Table: AVXConversionTbl, ISD,
3718 Dst: SimpleDstTy, Src: SimpleSrcTy))
3719 if (auto KindCost = Entry->Cost[CostKind])
3720 return *KindCost;
3721 }
3722
3723 if (ST->hasF16C()) {
3724 if (const auto *Entry = ConvertCostTableLookup(Table: F16ConversionTbl, ISD,
3725 Dst: SimpleDstTy, Src: SimpleSrcTy))
3726 if (auto KindCost = Entry->Cost[CostKind])
3727 return *KindCost;
3728 }
3729
3730 if (ST->hasSSE41()) {
3731 if (const auto *Entry = ConvertCostTableLookup(Table: SSE41ConversionTbl, ISD,
3732 Dst: SimpleDstTy, Src: SimpleSrcTy))
3733 if (auto KindCost = Entry->Cost[CostKind])
3734 return *KindCost;
3735 }
3736
3737 if (ST->hasSSE2()) {
3738 if (const auto *Entry = ConvertCostTableLookup(Table: SSE2ConversionTbl, ISD,
3739 Dst: SimpleDstTy, Src: SimpleSrcTy))
3740 if (auto KindCost = Entry->Cost[CostKind])
3741 return *KindCost;
3742 }
3743
3744 if ((ISD == ISD::FP_ROUND && SimpleDstTy == MVT::f16) ||
3745 (ISD == ISD::FP_EXTEND && SimpleSrcTy == MVT::f16)) {
3746 // fp16 conversions not covered by any table entries require a libcall.
3747 // Return a large (arbitrary) number to model this.
3748 return InstructionCost(64);
3749 }
3750 }
3751
3752 // Fall back to legalized types.
3753 std::pair<InstructionCost, MVT> LTSrc = getTypeLegalizationCost(Ty: Src);
3754 std::pair<InstructionCost, MVT> LTDest = getTypeLegalizationCost(Ty: Dst);
3755
3756 // If we're truncating to the same legalized type - just assume its free.
3757 if (ISD == ISD::TRUNCATE && LTSrc.second == LTDest.second)
3758 return TTI::TCC_Free;
3759
3760 if (ST->useAVX512Regs()) {
3761 if (ST->hasBWI())
3762 if (const auto *Entry = ConvertCostTableLookup(
3763 Table: AVX512BWConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3764 if (auto KindCost = Entry->Cost[CostKind])
3765 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3766
3767 if (ST->hasDQI())
3768 if (const auto *Entry = ConvertCostTableLookup(
3769 Table: AVX512DQConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3770 if (auto KindCost = Entry->Cost[CostKind])
3771 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3772
3773 if (ST->hasAVX512())
3774 if (const auto *Entry = ConvertCostTableLookup(
3775 Table: AVX512FConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3776 if (auto KindCost = Entry->Cost[CostKind])
3777 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3778 }
3779
3780 if (ST->hasBWI())
3781 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512BWVLConversionTbl, ISD,
3782 Dst: LTDest.second, Src: LTSrc.second))
3783 if (auto KindCost = Entry->Cost[CostKind])
3784 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3785
3786 if (ST->hasDQI())
3787 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512DQVLConversionTbl, ISD,
3788 Dst: LTDest.second, Src: LTSrc.second))
3789 if (auto KindCost = Entry->Cost[CostKind])
3790 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3791
3792 if (ST->hasAVX512())
3793 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512VLConversionTbl, ISD,
3794 Dst: LTDest.second, Src: LTSrc.second))
3795 if (auto KindCost = Entry->Cost[CostKind])
3796 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3797
3798 if (ST->hasAVX2())
3799 if (const auto *Entry = ConvertCostTableLookup(Table: AVX2ConversionTbl, ISD,
3800 Dst: LTDest.second, Src: LTSrc.second))
3801 if (auto KindCost = Entry->Cost[CostKind])
3802 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3803
3804 if (ST->hasAVX())
3805 if (const auto *Entry = ConvertCostTableLookup(Table: AVXConversionTbl, ISD,
3806 Dst: LTDest.second, Src: LTSrc.second))
3807 if (auto KindCost = Entry->Cost[CostKind])
3808 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3809
3810 if (ST->hasF16C()) {
3811 if (const auto *Entry = ConvertCostTableLookup(Table: F16ConversionTbl, ISD,
3812 Dst: LTDest.second, Src: LTSrc.second))
3813 if (auto KindCost = Entry->Cost[CostKind])
3814 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3815 }
3816
3817 if (ST->hasSSE41())
3818 if (const auto *Entry = ConvertCostTableLookup(Table: SSE41ConversionTbl, ISD,
3819 Dst: LTDest.second, Src: LTSrc.second))
3820 if (auto KindCost = Entry->Cost[CostKind])
3821 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3822
3823 if (ST->hasSSE2())
3824 if (const auto *Entry = ConvertCostTableLookup(Table: SSE2ConversionTbl, ISD,
3825 Dst: LTDest.second, Src: LTSrc.second))
3826 if (auto KindCost = Entry->Cost[CostKind])
3827 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3828
3829 // Fallback, for i8/i16 sitofp/uitofp cases we need to extend to i32 for
3830 // sitofp.
3831 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
3832 1 < Src->getScalarSizeInBits() && Src->getScalarSizeInBits() < 32) {
3833 Type *ExtSrc = Src->getWithNewBitWidth(NewBitWidth: 32);
3834 unsigned ExtOpc =
3835 (ISD == ISD::SINT_TO_FP) ? Instruction::SExt : Instruction::ZExt;
3836
3837 // For scalar loads the extend would be free.
3838 InstructionCost ExtCost = 0;
3839 if (!(Src->isIntegerTy() && I && isa<LoadInst>(Val: I->getOperand(i: 0))))
3840 ExtCost = getCastInstrCost(Opcode: ExtOpc, Dst: ExtSrc, Src, CCH, CostKind);
3841
3842 return ExtCost + getCastInstrCost(Opcode: Instruction::SIToFP, Dst, Src: ExtSrc,
3843 CCH: TTI::CastContextHint::None, CostKind);
3844 }
3845
3846 // Fallback for fptosi/fptoui i8/i16 cases we need to truncate from fptosi
3847 // i32.
3848 if ((ISD == ISD::FP_TO_SINT || ISD == ISD::FP_TO_UINT) &&
3849 1 < Dst->getScalarSizeInBits() && Dst->getScalarSizeInBits() < 32) {
3850 Type *TruncDst = Dst->getWithNewBitWidth(NewBitWidth: 32);
3851 return getCastInstrCost(Opcode: Instruction::FPToSI, Dst: TruncDst, Src, CCH, CostKind) +
3852 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: TruncDst,
3853 CCH: TTI::CastContextHint::None, CostKind);
3854 }
3855
3856 // TODO: Allow non-throughput costs that aren't binary.
3857 auto AdjustCost = [&CostKind](InstructionCost Cost,
3858 InstructionCost N = 1) -> InstructionCost {
3859 if (CostKind != TTI::TCK_RecipThroughput)
3860 return Cost == 0 ? 0 : N;
3861 return Cost * N;
3862 };
3863 return AdjustCost(
3864 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
3865}
3866
3867// Additive cost for a predicated op's "predicate fanout": a select, masked
3868// load/store or gather/scatter keeps a compact <N x i1> mask, but if its data
3869// legalizes into more parts than the mask, codegen builds a sub-mask per extra
3870// part (kshiftr on AVX-512, unpack/extend on SSE/AVX) the per-part tables miss.
3871// FanoutCostPerPart is a tie-break: it beats the vectorizer's widest-VF bias.
3872static InstructionCost getPredicateFanoutCost(const X86TTIImpl &TTI,
3873 Type *DataVTy, Type *MaskVTy) {
3874 constexpr unsigned FanoutCostPerPart = 4;
3875 unsigned DataParts = TTI.getNumberOfParts(Tp: DataVTy);
3876 unsigned MaskParts = TTI.getNumberOfParts(Tp: MaskVTy);
3877 if (DataParts > MaskParts && MaskParts > 0)
3878 return InstructionCost((DataParts - MaskParts) * FanoutCostPerPart);
3879 return InstructionCost(0);
3880}
3881
3882InstructionCost X86TTIImpl::getCmpSelInstrCost(
3883 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
3884 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
3885 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
3886 // Early out if this type isn't scalar/vector integer/float.
3887 if (!(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy()))
3888 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
3889 Op1Info, Op2Info, I);
3890
3891 // Legalize the type.
3892 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
3893
3894 MVT MTy = LT.second;
3895
3896 int ISD = TLI->InstructionOpcodeToISD(Opcode);
3897 assert(ISD && "Invalid opcode");
3898
3899 // Predicate fanout (see getPredicateFanoutCost), AVX-512 only: on SSE/AVX a
3900 // select blend is already priced per part, so charging it there shrinks VF.
3901 InstructionCost MaskExpCost = 0;
3902 if (Opcode == Instruction::Select && ST->hasAVX512() &&
3903 isa<VectorType>(Val: ValTy) && isa_and_nonnull<VectorType>(Val: CondTy))
3904 MaskExpCost = getPredicateFanoutCost(TTI: *this, DataVTy: ValTy, MaskVTy: CondTy);
3905
3906 InstructionCost ExtraCost = 0;
3907 if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
3908 // Some vector comparison predicates cost extra instructions.
3909 // TODO: Adjust ExtraCost based on CostKind?
3910 // TODO: Should we invert this and assume worst case cmp costs
3911 // and reduce for particular predicates?
3912 if (MTy.isVector() &&
3913 !((ST->hasXOP() && (!ST->hasAVX2() || MTy.is128BitVector())) ||
3914 (ST->hasAVX512() && 32 <= MTy.getScalarSizeInBits()) ||
3915 ST->hasBWI())) {
3916 // Fallback to I if a specific predicate wasn't specified.
3917 CmpInst::Predicate Pred = VecPred;
3918 if (I && (Pred == CmpInst::BAD_ICMP_PREDICATE ||
3919 Pred == CmpInst::BAD_FCMP_PREDICATE))
3920 Pred = cast<CmpInst>(Val: I)->getPredicate();
3921
3922 bool CmpWithConstant = false;
3923 if (auto *CmpInstr = dyn_cast_or_null<CmpInst>(Val: I))
3924 CmpWithConstant = isa<Constant>(Val: CmpInstr->getOperand(i_nocapture: 1));
3925
3926 switch (Pred) {
3927 case CmpInst::Predicate::ICMP_NE:
3928 // xor(cmpeq(x,y),-1)
3929 ExtraCost = CmpWithConstant ? 0 : 1;
3930 break;
3931 case CmpInst::Predicate::ICMP_SGE:
3932 case CmpInst::Predicate::ICMP_SLE:
3933 // xor(cmpgt(x,y),-1)
3934 ExtraCost = CmpWithConstant ? 0 : 1;
3935 break;
3936 case CmpInst::Predicate::ICMP_ULT:
3937 case CmpInst::Predicate::ICMP_UGT:
3938 // cmpgt(xor(x,signbit),xor(y,signbit))
3939 // xor(cmpeq(pmaxu(x,y),x),-1)
3940 ExtraCost = CmpWithConstant ? 1 : 2;
3941 break;
3942 case CmpInst::Predicate::ICMP_ULE:
3943 case CmpInst::Predicate::ICMP_UGE:
3944 if ((ST->hasSSE41() && MTy.getScalarSizeInBits() == 32) ||
3945 (ST->hasSSE2() && MTy.getScalarSizeInBits() < 32)) {
3946 // cmpeq(psubus(x,y),0)
3947 // cmpeq(pminu(x,y),x)
3948 ExtraCost = 1;
3949 } else {
3950 // xor(cmpgt(xor(x,signbit),xor(y,signbit)),-1)
3951 ExtraCost = CmpWithConstant ? 2 : 3;
3952 }
3953 break;
3954 case CmpInst::Predicate::FCMP_ONE:
3955 case CmpInst::Predicate::FCMP_UEQ:
3956 // Without AVX we need to expand FCMP_ONE/FCMP_UEQ cases.
3957 // Use FCMP_UEQ expansion - FCMP_ONE should be the same.
3958 if (CondTy && !ST->hasAVX())
3959 return getCmpSelInstrCost(Opcode, ValTy, CondTy,
3960 VecPred: CmpInst::Predicate::FCMP_UNO, CostKind,
3961 Op1Info, Op2Info) +
3962 getCmpSelInstrCost(Opcode, ValTy, CondTy,
3963 VecPred: CmpInst::Predicate::FCMP_OEQ, CostKind,
3964 Op1Info, Op2Info) +
3965 getArithmeticInstrCost(Opcode: Instruction::Or, Ty: CondTy, CostKind);
3966
3967 break;
3968 case CmpInst::Predicate::BAD_ICMP_PREDICATE:
3969 case CmpInst::Predicate::BAD_FCMP_PREDICATE:
3970 // Assume worst case scenario and add the maximum extra cost.
3971 ExtraCost = 3;
3972 break;
3973 default:
3974 break;
3975 }
3976 }
3977 }
3978
3979 static const CostKindTblEntry SLMCostTbl[] = {
3980 // slm pcmpeq/pcmpgt throughput is 2
3981 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
3982 // slm pblendvb/blendvpd/blendvps throughput is 4
3983 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vblendvpd
3984 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vblendvps
3985 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3986 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3987 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3988 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3989 };
3990
3991 static const CostKindTblEntry AVX512BWCostTbl[] = {
3992 { .ISD: ISD::SETCC, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3993 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3994 { .ISD: ISD::SETCC, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3995 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3996
3997 { .ISD: ISD::SELECT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3998 { .ISD: ISD::SELECT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3999 };
4000
4001 static const CostKindTblEntry AVX512CostTbl[] = {
4002 { .ISD: ISD::SETCC, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4003 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4004 { .ISD: ISD::SETCC, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4005 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4006
4007 { .ISD: ISD::SETCC, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4008 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4009 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4010 { .ISD: ISD::SETCC, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4011 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4012 { .ISD: ISD::SETCC, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4013 { .ISD: ISD::SETCC, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4014
4015 { .ISD: ISD::SELECT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4016 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4017 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4018 { .ISD: ISD::SELECT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4019 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4020 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4021 { .ISD: ISD::SELECT, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4022 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4023 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4024 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4025 { .ISD: ISD::SELECT, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4026 { .ISD: ISD::SELECT, .Type: MVT::v8f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4027 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4028 { .ISD: ISD::SELECT, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4029
4030 { .ISD: ISD::SELECT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4031 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4032 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4033 { .ISD: ISD::SELECT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4034 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4035 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4036 };
4037
4038 static const CostKindTblEntry AVX2CostTbl[] = {
4039 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4040 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4041 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4042 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4043 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4044 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4045
4046 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4047 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4048 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4049 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4050
4051 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4052 { .ISD: ISD::SELECT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4053 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4054 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4055 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4056 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4057 };
4058
4059 static const CostKindTblEntry XOPCostTbl[] = {
4060 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4061 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4062 };
4063
4064 static const CostKindTblEntry AVX1CostTbl[] = {
4065 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4066 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4067 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4068 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4069 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4070 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4071
4072 // AVX1 does not support 8-wide integer compare.
4073 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4074 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4075 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4076 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4077
4078 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4079 { .ISD: ISD::SELECT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4080 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4081 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4082 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // vandps + vandnps + vorps
4083 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // vandps + vandnps + vorps
4084 };
4085
4086 static const CostKindTblEntry SSE42CostTbl[] = {
4087 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4088 };
4089
4090 static const CostKindTblEntry SSE41CostTbl[] = {
4091 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4092 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4093
4094 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvpd
4095 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvpd
4096 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvps
4097 { .ISD: ISD::SELECT, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvps
4098 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4099 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4100 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4101 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4102 };
4103
4104 static const CostKindTblEntry SSE2CostTbl[] = {
4105 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4106 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4107
4108 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // pcmpeqd/pcmpgtd expansion
4109 { .ISD: ISD::SETCC, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4110 { .ISD: ISD::SETCC, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4111 { .ISD: ISD::SETCC, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4112
4113 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andpd + andnpd + orpd
4114 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andpd + andnpd + orpd
4115 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4116 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4117 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4118 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4119 };
4120
4121 static const CostKindTblEntry SSE1CostTbl[] = {
4122 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4123 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4124
4125 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andps + andnps + orps
4126 { .ISD: ISD::SELECT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andps + andnps + orps
4127 };
4128
4129 if (ST->useSLMArithCosts())
4130 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
4131 if (auto KindCost = Entry->Cost[CostKind])
4132 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4133
4134 if (ST->hasBWI())
4135 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
4136 if (auto KindCost = Entry->Cost[CostKind])
4137 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4138
4139 if (ST->hasAVX512())
4140 if (const auto *Entry = CostTableLookup(Table: AVX512CostTbl, ISD, Ty: MTy))
4141 if (auto KindCost = Entry->Cost[CostKind])
4142 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4143
4144 if (ST->hasAVX2())
4145 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
4146 if (auto KindCost = Entry->Cost[CostKind])
4147 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4148
4149 if (ST->hasXOP())
4150 if (const auto *Entry = CostTableLookup(Table: XOPCostTbl, ISD, Ty: MTy))
4151 if (auto KindCost = Entry->Cost[CostKind])
4152 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4153
4154 if (ST->hasAVX())
4155 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
4156 if (auto KindCost = Entry->Cost[CostKind])
4157 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4158
4159 if (ST->hasSSE42())
4160 if (const auto *Entry = CostTableLookup(Table: SSE42CostTbl, ISD, Ty: MTy))
4161 if (auto KindCost = Entry->Cost[CostKind])
4162 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4163
4164 if (ST->hasSSE41())
4165 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
4166 if (auto KindCost = Entry->Cost[CostKind])
4167 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4168
4169 if (ST->hasSSE2())
4170 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
4171 if (auto KindCost = Entry->Cost[CostKind])
4172 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4173
4174 if (ST->hasSSE1())
4175 if (const auto *Entry = CostTableLookup(Table: SSE1CostTbl, ISD, Ty: MTy))
4176 if (auto KindCost = Entry->Cost[CostKind])
4177 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4178
4179 // Assume a 3cy latency for fp select ops.
4180 if (CostKind == TTI::TCK_Latency && Opcode == Instruction::Select)
4181 if (ValTy->getScalarType()->isFloatingPointTy())
4182 return 3;
4183
4184 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
4185 Op1Info, Op2Info, I) +
4186 MaskExpCost;
4187}
4188
4189unsigned X86TTIImpl::getAtomicMemIntrinsicMaxElementSize() const { return 16; }
4190
4191/// Returns true if \p V is the square root, divided into with the reciprocal
4192/// allowed, and the target replaces the pair with the rsqrt* based estimate,
4193/// costed with the division. The estimate is formed for the legalized type
4194/// \p VT.
4195static bool isFoldedIntoRsqrtEstimate(const Value *V,
4196 const X86TargetLowering &TLI, MVT VT,
4197 TTI::TargetCostKind CostKind) {
4198 using namespace PatternMatch;
4199 if (CostKind != TTI::TCK_RecipThroughput && CostKind != TTI::TCK_Latency)
4200 return false;
4201 const auto *II = dyn_cast_if_present<IntrinsicInst>(Val: V);
4202 if (!II || II->getIntrinsicID() != Intrinsic::sqrt || !II->hasOneUse() ||
4203 !match(V: II->user_back(), P: m_FDiv(L: m_Value(), R: m_Specific(V: II))) ||
4204 !cast<FPMathOperator>(Val: II->user_back())->hasAllowReciprocal())
4205 return false;
4206 return TLI.hasSqrtEstimate(VT, /*Reciprocal=*/true) &&
4207 TLI.getRecipEstimateSqrtEnabled(VT, F: *II->getFunction()) !=
4208 TargetLoweringBase::ReciprocalEstimate::Disabled;
4209}
4210
4211InstructionCost
4212X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
4213 TTI::TargetCostKind CostKind) const {
4214 // Costs should match the codegen from:
4215 // BITREVERSE: llvm\test\CodeGen\X86\vector-bitreverse.ll
4216 // BSWAP: llvm\test\CodeGen\X86\bswap-vector.ll
4217 // CTLZ: llvm\test\CodeGen\X86\vector-lzcnt-*.ll
4218 // CTPOP: llvm\test\CodeGen\X86\vector-popcnt-*.ll
4219 // CTTZ: llvm\test\CodeGen\X86\vector-tzcnt-*.ll
4220
4221 // TODO: Overflow intrinsics (*ADDO, *SUBO, *MULO) with vector types are not
4222 // specialized in these tables yet.
4223 static const CostKindTblEntry AVX512VBMI2CostTbl[] = {
4224 { .ISD: ISD::FSHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4225 { .ISD: ISD::FSHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4226 { .ISD: ISD::FSHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4227 { .ISD: ISD::FSHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4228 { .ISD: ISD::FSHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4229 { .ISD: ISD::FSHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4230 { .ISD: ISD::FSHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4231 { .ISD: ISD::FSHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4232 { .ISD: ISD::FSHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4233 { .ISD: ISD::ROTL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4234 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4235 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4236 { .ISD: ISD::ROTR, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4237 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4238 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4239 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4240 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4241 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4242 };
4243 static const CostKindTblEntry AVX512BITALGCostTbl[] = {
4244 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4245 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4246 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4247 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4248 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4249 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4250 };
4251 static const CostKindTblEntry AVX512VPOPCNTDQCostTbl[] = {
4252 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4253 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4254 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4255 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4256 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4257 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4258 };
4259 static const CostKindTblEntry AVX512CDCostTbl[] = {
4260 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4261 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4262 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 27, .CodeSizeCost: 23, .SizeAndLatencyCost: 27 } },
4263 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 16, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4264 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4265 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4266 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 19, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4267 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4268 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4269 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4270 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 15, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
4271 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4272
4273 { .ISD: ISD::CTTZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4274 { .ISD: ISD::CTTZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4275 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4276 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4277 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4278 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4279 };
4280 static const CostKindTblEntry AVX512BWCostTbl[] = {
4281 { .ISD: ISD::ABS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4282 { .ISD: ISD::ABS, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4283 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4284 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4285 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4286 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4287 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4288 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4289 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4290 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4291 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4292 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4293 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4294 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 12 } },
4295 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4296 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4297 { .ISD: ISD::BSWAP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4298 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4299 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4300 { .ISD: ISD::BSWAP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4301 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4302 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4303 { .ISD: ISD::BSWAP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4304 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 22, .CodeSizeCost: 23, .SizeAndLatencyCost: 23 } },
4305 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 23, .CodeSizeCost: 25, .SizeAndLatencyCost: 25 } },
4306 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 15, .SizeAndLatencyCost: 16 } },
4307 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 9 } },
4308 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4309 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4310 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 10, .SizeAndLatencyCost: 12 } },
4311 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4312 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4313 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4314 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4315 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4316 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4317 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4318 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4319 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4320 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4321 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4322 { .ISD: ISD::CTTZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4323 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4324 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4325 { .ISD: ISD::CTTZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4326 { .ISD: ISD::MULHS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4327 { .ISD: ISD::MULHU, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4328 { .ISD: ISD::ROTL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } },
4329 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4330 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4331 { .ISD: ISD::ROTL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4332 { .ISD: ISD::ROTL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
4333 { .ISD: ISD::ROTL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
4334 { .ISD: ISD::ROTR, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } },
4335 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4336 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4337 { .ISD: ISD::ROTR, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 12, .SizeAndLatencyCost: 14 } },
4338 { .ISD: ISD::ROTR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 14, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
4339 { .ISD: ISD::ROTR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 14, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
4340 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4341 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4342 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4343 { .ISD: X86ISD::VROTLI, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4344 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4345 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4346 { .ISD: ISD::SADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4347 { .ISD: ISD::SADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4348 { .ISD: ISD::SMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4349 { .ISD: ISD::SMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4350 { .ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4351 { .ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4352 { .ISD: ISD::SMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4353 { .ISD: ISD::SMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 21, .CodeSizeCost: 17, .SizeAndLatencyCost: 18 } },
4354 { .ISD: ISD::UMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4355 { .ISD: ISD::UMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 15, .CodeSizeCost: 15, .SizeAndLatencyCost: 16 } },
4356 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4357 { .ISD: ISD::SSUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4358 { .ISD: ISD::UADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4359 { .ISD: ISD::UADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4360 { .ISD: ISD::UMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4361 { .ISD: ISD::UMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4362 { .ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4363 { .ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4364 { .ISD: ISD::USUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4365 { .ISD: ISD::USUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4366 };
4367 static const CostKindTblEntry AVX512CostTbl[] = {
4368 { .ISD: ISD::ABS, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4369 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4370 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4371 { .ISD: ISD::ABS, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4372 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4373 { .ISD: ISD::ABS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4374 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4375 { .ISD: ISD::ABS, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4376 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4377 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4378 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4379 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4380 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4381 { .ISD: ISD::BSWAP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4382 { .ISD: ISD::BSWAP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4383 { .ISD: ISD::BSWAP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4384 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 28, .CodeSizeCost: 32, .SizeAndLatencyCost: 32 } },
4385 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 30, .CodeSizeCost: 38, .SizeAndLatencyCost: 38 } },
4386 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 15, .CodeSizeCost: 29, .SizeAndLatencyCost: 29 } },
4387 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
4388 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
4389 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 19, .CodeSizeCost: 27, .SizeAndLatencyCost: 27 } },
4390 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 15, .CodeSizeCost: 22, .SizeAndLatencyCost: 22 } },
4391 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 11, .CodeSizeCost: 16, .SizeAndLatencyCost: 16 } },
4392 { .ISD: ISD::CTTZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4393 { .ISD: ISD::CTTZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4394 { .ISD: ISD::CTTZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 17, .CodeSizeCost: 27, .SizeAndLatencyCost: 27 } },
4395 { .ISD: ISD::CTTZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 21, .SizeAndLatencyCost: 21 } },
4396 { .ISD: ISD::MULHS, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4397 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4398 { .ISD: ISD::MULHS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4399 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4400 { .ISD: ISD::MULHU, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4401 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4402 { .ISD: ISD::MULHU, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4403 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4404 { .ISD: ISD::ROTL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4405 { .ISD: ISD::ROTL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4406 { .ISD: ISD::ROTL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4407 { .ISD: ISD::ROTL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4408 { .ISD: ISD::ROTL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4409 { .ISD: ISD::ROTL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4410 { .ISD: ISD::ROTR, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4411 { .ISD: ISD::ROTR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4412 { .ISD: ISD::ROTR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4413 { .ISD: ISD::ROTR, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4414 { .ISD: ISD::ROTR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4415 { .ISD: ISD::ROTR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4416 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4417 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4418 { .ISD: X86ISD::VROTLI, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4419 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4420 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4421 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4422 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4423 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4424 { .ISD: ISD::SADDSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4425 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4426 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4427 { .ISD: ISD::SADDSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4428 { .ISD: ISD::SADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4429 { .ISD: ISD::SADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4430 { .ISD: ISD::SMAX, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4431 { .ISD: ISD::SMAX, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4432 { .ISD: ISD::SMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4433 { .ISD: ISD::SMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4434 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4435 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4436 { .ISD: ISD::SMIN, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4437 { .ISD: ISD::SMIN, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4438 { .ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4439 { .ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4440 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4441 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4442 { .ISD: ISD::SMULO, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 44, .CodeSizeCost: 81, .SizeAndLatencyCost: 93 } },
4443 { .ISD: ISD::SMULO, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4444 { .ISD: ISD::SMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4445 { .ISD: ISD::SMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 28, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
4446 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4447 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4448 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4449 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4450 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4451 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4452 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4453 { .ISD: ISD::SSUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4454 { .ISD: ISD::UMAX, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4455 { .ISD: ISD::UMAX, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4456 { .ISD: ISD::UMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4457 { .ISD: ISD::UMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4458 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4459 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4460 { .ISD: ISD::UMIN, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4461 { .ISD: ISD::UMIN, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4462 { .ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4463 { .ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4464 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4465 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4466 { .ISD: ISD::UMULO, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 52, .LatencyCost: 52, .CodeSizeCost: 95, .SizeAndLatencyCost: 104} },
4467 { .ISD: ISD::UMULO, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4468 { .ISD: ISD::UMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 13, .CodeSizeCost: 16, .SizeAndLatencyCost: 16 } },
4469 { .ISD: ISD::UMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 30, .SizeAndLatencyCost: 30 } },
4470 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4471 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4472 { .ISD: ISD::UADDSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4473 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4474 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4475 { .ISD: ISD::UADDSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4476 { .ISD: ISD::UADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4477 { .ISD: ISD::UADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4478 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4479 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4480 { .ISD: ISD::USUBSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4481 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4482 { .ISD: ISD::USUBSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4483 { .ISD: ISD::USUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4484 { .ISD: ISD::USUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4485 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4486 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4487 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4488 { .ISD: ISD::FMAXNUM, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4489 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4490 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4491 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4492 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4493 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4494 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4495 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4496 { .ISD: ISD::FSQRT, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
4497 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4498 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4499 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4500 { .ISD: ISD::FSQRT, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
4501 };
4502 static const CostKindTblEntry XOPCostTbl[] = {
4503 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4504 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4505 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4506 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4507 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4508 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4509 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4510 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4511 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4512 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4513 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4514 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4515 // XOP: ROTL = VPROT(X,Y), ROTR = VPROT(X,SUB(0,Y))
4516 { .ISD: ISD::ROTL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4517 { .ISD: ISD::ROTL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4518 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4519 { .ISD: ISD::ROTL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4520 { .ISD: ISD::ROTL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4521 { .ISD: ISD::ROTL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4522 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4523 { .ISD: ISD::ROTL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4524 { .ISD: ISD::ROTR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4525 { .ISD: ISD::ROTR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4526 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4527 { .ISD: ISD::ROTR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4528 { .ISD: ISD::ROTR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4529 { .ISD: ISD::ROTR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4530 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4531 { .ISD: ISD::ROTR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4532 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4533 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4534 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4535 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4536 { .ISD: X86ISD::VROTLI, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4537 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4538 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4539 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4540 };
4541 static const CostKindTblEntry AVX2CostTbl[] = {
4542 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4543 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4544 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4545 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4546 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4547 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4548 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4549 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4550 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4551 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4552 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4553 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4554 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4555 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4556 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4557 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 15 } },
4558 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4559 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4560 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4561 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4562 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4563 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4564 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 18, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4565 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 18, .CodeSizeCost: 24, .SizeAndLatencyCost: 44 } },
4566 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 20 } },
4567 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 34 } },
4568 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } },
4569 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } },
4570 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4571 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 14 } },
4572 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4573 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4574 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4575 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4576 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4577 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 11, .SizeAndLatencyCost: 18 } },
4578 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4579 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4580 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 13 } },
4581 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 20 } },
4582 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 14, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4583 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 17, .SizeAndLatencyCost: 24 } },
4584 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4585 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } },
4586 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4587 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 18 } },
4588 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } },
4589 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4590 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } },
4591 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4592 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
4593 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4594 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } },
4595 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 13 } },
4596 { .ISD: ISD::SADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4597 { .ISD: ISD::SADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4598 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4599 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4600 { .ISD: ISD::SMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4601 { .ISD: ISD::SMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4602 { .ISD: ISD::SMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4603 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4604 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4605 { .ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4606 { .ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4607 { .ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4608 { .ISD: ISD::SMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 33, .SizeAndLatencyCost: 37 } },
4609 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 13, .SizeAndLatencyCost: 15 } },
4610 { .ISD: ISD::SMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 20, .CodeSizeCost: 13, .SizeAndLatencyCost: 24 } },
4611 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4612 { .ISD: ISD::SMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 8, .SizeAndLatencyCost: 14 } },
4613 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4614 { .ISD: ISD::SMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 15, .CodeSizeCost: 18, .SizeAndLatencyCost: 35 } },
4615 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 22, .CodeSizeCost: 14, .SizeAndLatencyCost: 21 } },
4616 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4617 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4618 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 14, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4619 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 9, .SizeAndLatencyCost: 16 } },
4620 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4621 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4622 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4623 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 10 } },
4624 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } },
4625 { .ISD: ISD::UADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4626 { .ISD: ISD::UADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4627 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4628 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
4629 { .ISD: ISD::UMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4630 { .ISD: ISD::UMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4631 { .ISD: ISD::UMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4632 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4633 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
4634 { .ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4635 { .ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4636 { .ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4637 { .ISD: ISD::UMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 24, .CodeSizeCost: 39, .SizeAndLatencyCost: 43 } },
4638 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 10, .CodeSizeCost: 15, .SizeAndLatencyCost: 19 } },
4639 { .ISD: ISD::UMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 23 } },
4640 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4641 { .ISD: ISD::UMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 8, .SizeAndLatencyCost: 13 } },
4642 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4643 { .ISD: ISD::UMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 17, .SizeAndLatencyCost: 33 } },
4644 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 19, .CodeSizeCost: 13, .SizeAndLatencyCost: 20 } },
4645 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4646 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 10 } },
4647 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
4648 { .ISD: ISD::USUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4649 { .ISD: ISD::USUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4650 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4651 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4652 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4653 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4654 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4655 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4656 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtss
4657 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtps
4658 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtps
4659 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtsd
4660 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtpd
4661 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtpd
4662 };
4663 static const CostKindTblEntry AVX1CostTbl[] = {
4664 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4665 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4666 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4667 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4668 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4669 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4670 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4671 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4672 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4673 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4674 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 15, .CodeSizeCost: 17, .SizeAndLatencyCost: 26 } }, // 2 x 128-bit Op + extract/insert
4675 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4676 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4677 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4678 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4679 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4680 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4681 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4682 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 29, .LatencyCost: 33, .CodeSizeCost: 49, .SizeAndLatencyCost: 58 } }, // 2 x 128-bit Op + extract/insert
4683 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 24, .CodeSizeCost: 24, .SizeAndLatencyCost: 28 } },
4684 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 28, .CodeSizeCost: 39, .SizeAndLatencyCost: 48 } }, // 2 x 128-bit Op + extract/insert
4685 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 20, .CodeSizeCost: 19, .SizeAndLatencyCost: 23 } },
4686 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 29, .SizeAndLatencyCost: 38 } }, // 2 x 128-bit Op + extract/insert
4687 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 16, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4688 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 15, .CodeSizeCost: 19, .SizeAndLatencyCost: 28 } }, // 2 x 128-bit Op + extract/insert
4689 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4690 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 18, .CodeSizeCost: 19, .SizeAndLatencyCost: 28 } }, // 2 x 128-bit Op + extract/insert
4691 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 14, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4692 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 27, .SizeAndLatencyCost: 36 } }, // 2 x 128-bit Op + extract/insert
4693 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 20, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4694 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 21, .CodeSizeCost: 22, .SizeAndLatencyCost: 31 } }, // 2 x 128-bit Op + extract/insert
4695 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 18, .CodeSizeCost: 11, .SizeAndLatencyCost: 15 } },
4696 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 15, .CodeSizeCost: 16, .SizeAndLatencyCost: 25 } }, // 2 x 128-bit Op + extract/insert
4697 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4698 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 22, .CodeSizeCost: 24, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4699 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 19, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4700 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 27, .CodeSizeCost: 32, .SizeAndLatencyCost: 41 } }, // 2 x 128-bit Op + extract/insert
4701 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 24, .CodeSizeCost: 17, .SizeAndLatencyCost: 21 } },
4702 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 27, .SizeAndLatencyCost: 36 } }, // 2 x 128-bit Op + extract/insert
4703 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 21, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4704 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 18, .CodeSizeCost: 21, .SizeAndLatencyCost: 30 } }, // 2 x 128-bit Op + extract/insert
4705 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 16, .CodeSizeCost: 11, .SizeAndLatencyCost: 15 } },
4706 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4707 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4708 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4709 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4710 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
4711 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 20, .CodeSizeCost: 15, .SizeAndLatencyCost: 25 } }, // 2 x 128-bit Op + extract/insert
4712 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 18, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } }, // 2 x 128-bit Op + extract/insert
4713 { .ISD: ISD::SADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4714 { .ISD: ISD::SADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4715 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // 2 x 128-bit Op + extract/insert
4716 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
4717 { .ISD: ISD::SMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4718 { .ISD: ISD::SMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4719 { .ISD: ISD::SMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4720 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // 2 x 128-bit Op + extract/insert
4721 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4722 { .ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4723 { .ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4724 { .ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4725 { .ISD: ISD::SMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 33, .SizeAndLatencyCost: 37 } },
4726 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4727 { .ISD: ISD::SMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 24, .SizeAndLatencyCost: 29 } },
4728 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4729 { .ISD: ISD::SMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } },
4730 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4731 { .ISD: ISD::SMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 37, .SizeAndLatencyCost: 39 } },
4732 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 22, .CodeSizeCost: 18, .SizeAndLatencyCost: 21 } },
4733 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4734 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 21, .CodeSizeCost: 18, .SizeAndLatencyCost: 29 } }, // 2 x 128-bit Op + extract/insert
4735 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19, .CodeSizeCost: 18, .SizeAndLatencyCost: 29 } }, // 2 x 128-bit Op + extract/insert
4736 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4737 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4738 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4739 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } }, // 2 x 128-bit Op + extract/insert
4740 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } }, // 2 x 128-bit Op + extract/insert
4741 { .ISD: ISD::UADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4742 { .ISD: ISD::UADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4743 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 11, .SizeAndLatencyCost: 17 } }, // 2 x 128-bit Op + extract/insert
4744 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } },
4745 { .ISD: ISD::UMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4746 { .ISD: ISD::UMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4747 { .ISD: ISD::UMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4748 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 11, .SizeAndLatencyCost: 17 } }, // 2 x 128-bit Op + extract/insert
4749 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } },
4750 { .ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4751 { .ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4752 { .ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4753 { .ISD: ISD::UMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 26, .CodeSizeCost: 39, .SizeAndLatencyCost: 45 } },
4754 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 12, .CodeSizeCost: 15, .SizeAndLatencyCost: 20 } },
4755 { .ISD: ISD::UMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 15, .CodeSizeCost: 23, .SizeAndLatencyCost: 28 } },
4756 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4757 { .ISD: ISD::UMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 14 } },
4758 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4759 { .ISD: ISD::UMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 19, .CodeSizeCost: 35, .SizeAndLatencyCost: 37 } },
4760 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 19, .CodeSizeCost: 17, .SizeAndLatencyCost: 20 } },
4761 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4762 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } }, // 2 x 128-bit Op + extract/insert
4763 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2 x 128-bit Op + extract/insert
4764 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4765 { .ISD: ISD::USUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4766 { .ISD: ISD::USUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4767 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4768 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4769 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 10 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4770 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4771 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4772 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 10 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4773 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtss
4774 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtps
4775 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtps
4776 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 27, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtsd
4777 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 27, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtpd
4778 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 54, .LatencyCost: 54, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtpd
4779 };
4780 static const CostKindTblEntry GFNICostTbl[] = {
4781 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4782 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // gf2p8affineqb
4783 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // gf2p8affineqb
4784 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // gf2p8affineqb
4785 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4786 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4787 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4788 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4789 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4790 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4791 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4792 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4793 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4794 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4795 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4796 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4797 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4798 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4799 { .ISD: X86ISD::VROTLI, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4800 };
4801 static const CostKindTblEntry GLMCostTbl[] = {
4802 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtss
4803 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 37, .LatencyCost: 41, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtps
4804 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 34, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtsd
4805 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 67, .LatencyCost: 71, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtpd
4806 };
4807 static const CostKindTblEntry SLMCostTbl[] = {
4808 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4809 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4810 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4811 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtss
4812 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 40, .LatencyCost: 41, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtps
4813 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 35, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtsd
4814 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 70, .LatencyCost: 71, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtpd
4815 };
4816 static const CostKindTblEntry SSE42CostTbl[] = {
4817 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4818 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4819 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4820 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4821 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4822 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4823 };
4824 static const CostKindTblEntry SSE41CostTbl[] = {
4825 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // BLENDVPD(X,PSUBQ(0,X),X)
4826 { .ISD: ISD::MULHS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4827 { .ISD: ISD::MULHU, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4828 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 14, .CodeSizeCost: 17, .SizeAndLatencyCost: 21 } },
4829 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4830 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 19, .CodeSizeCost: 25, .SizeAndLatencyCost: 29 } },
4831 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 10, .SizeAndLatencyCost: 12 } },
4832 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4833 { .ISD: ISD::SMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4834 { .ISD: ISD::SMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4835 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4836 { .ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4837 { .ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4838 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4839 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 24, .CodeSizeCost: 13, .SizeAndLatencyCost: 19 } },
4840 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 9, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4841 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 22, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4842 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4843 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4844 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4845 { .ISD: ISD::USUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4846 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4847 { .ISD: ISD::UMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4848 { .ISD: ISD::UMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4849 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4850 { .ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4851 { .ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4852 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 15, .SizeAndLatencyCost: 20 } },
4853 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 12, .SizeAndLatencyCost: 18 } },
4854 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4855 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 18, .SizeAndLatencyCost: 20 } },
4856 };
4857 static const CostKindTblEntry SSSE3CostTbl[] = {
4858 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4859 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4860 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4861 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4862 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4863 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4864 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4865 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4866 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4867 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4868 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 28, .CodeSizeCost: 28, .SizeAndLatencyCost: 35 } },
4869 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 22, .SizeAndLatencyCost: 28 } },
4870 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 17, .CodeSizeCost: 16, .SizeAndLatencyCost: 22 } },
4871 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 15, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4872 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 12, .SizeAndLatencyCost: 18 } },
4873 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 16, .SizeAndLatencyCost: 22 } },
4874 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 18, .CodeSizeCost: 14, .SizeAndLatencyCost: 20 } },
4875 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4876 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 25, .CodeSizeCost: 15, .SizeAndLatencyCost: 22 } },
4877 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 26, .CodeSizeCost: 19, .SizeAndLatencyCost: 25 } },
4878 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 20, .CodeSizeCost: 17, .SizeAndLatencyCost: 23 } },
4879 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16, .CodeSizeCost: 13, .SizeAndLatencyCost: 19 } }
4880 };
4881 static const CostKindTblEntry SSE2CostTbl[] = {
4882 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4883 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4884 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4885 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4886 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 32, .SizeAndLatencyCost: 32 } },
4887 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 30, .SizeAndLatencyCost: 30 } },
4888 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 25, .SizeAndLatencyCost: 25 } },
4889 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 21, .SizeAndLatencyCost: 21 } },
4890 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4891 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4892 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4893 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 45, .CodeSizeCost: 36, .SizeAndLatencyCost: 38 } },
4894 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 45, .CodeSizeCost: 38, .SizeAndLatencyCost: 40 } },
4895 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 38, .CodeSizeCost: 32, .SizeAndLatencyCost: 34 } },
4896 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 39, .CodeSizeCost: 29, .SizeAndLatencyCost: 32 } },
4897 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 26, .CodeSizeCost: 16, .SizeAndLatencyCost: 18 } },
4898 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 29, .CodeSizeCost: 21, .SizeAndLatencyCost: 23 } },
4899 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 25, .CodeSizeCost: 18, .SizeAndLatencyCost: 20 } },
4900 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 21, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4901 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 28, .CodeSizeCost: 19, .SizeAndLatencyCost: 21 } },
4902 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 31, .CodeSizeCost: 24, .SizeAndLatencyCost: 26 } },
4903 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 27, .CodeSizeCost: 21, .SizeAndLatencyCost: 23 } },
4904 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 23, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
4905 { .ISD: ISD::MULHS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4906 { .ISD: ISD::MULHS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4907 { .ISD: ISD::MULHU, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4908 { .ISD: ISD::MULHU, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4909 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 14, .CodeSizeCost: 24, .SizeAndLatencyCost: 24 } },
4910 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4911 { .ISD: ISD::SADDSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4912 { .ISD: ISD::SADDSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4913 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4914 { .ISD: ISD::SMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4915 { .ISD: ISD::SMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4916 { .ISD: ISD::SMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4917 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4918 { .ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4919 { .ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4920 { .ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4921 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 30, .LatencyCost: 33, .CodeSizeCost: 13, .SizeAndLatencyCost: 23 } },
4922 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 24, .CodeSizeCost: 23, .SizeAndLatencyCost: 23 } },
4923 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4924 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 23, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4925 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19, .CodeSizeCost: 31, .SizeAndLatencyCost: 31 } },
4926 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 12, .SizeAndLatencyCost: 13 } },
4927 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4928 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4929 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4930 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4931 { .ISD: ISD::UADDSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4932 { .ISD: ISD::UADDSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4933 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4934 { .ISD: ISD::UMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4935 { .ISD: ISD::UMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4936 { .ISD: ISD::UMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4937 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4938 { .ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4939 { .ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4940 { .ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4941 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 30, .LatencyCost: 33, .CodeSizeCost: 15, .SizeAndLatencyCost: 29 } },
4942 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4943 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4944 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4945 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4946 { .ISD: ISD::USUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4947 { .ISD: ISD::USUBSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4948 { .ISD: ISD::USUBSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4949 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4950 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4951 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4952 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4953 };
4954 static const CostKindTblEntry SSE1CostTbl[] = {
4955 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4956 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4957 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
4958 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 56, .LatencyCost: 56, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
4959 };
4960 static const CostKindTblEntry BMI64CostTbl[] = { // 64-bit targets
4961 { .ISD: ISD::CTTZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4962 };
4963 static const CostKindTblEntry BMI32CostTbl[] = { // 32 or 64-bit targets
4964 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4965 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4966 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4967 };
4968 static const CostKindTblEntry LZCNT64CostTbl[] = { // 64-bit targets
4969 { .ISD: ISD::CTLZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4970 };
4971 static const CostKindTblEntry LZCNT32CostTbl[] = { // 32 or 64-bit targets
4972 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4973 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4974 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4975 };
4976 static const CostKindTblEntry POPCNT64CostTbl[] = { // 64-bit targets
4977 { .ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // popcnt
4978 };
4979 static const CostKindTblEntry POPCNT32CostTbl[] = { // 32 or 64-bit targets
4980 { .ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // popcnt
4981 { .ISD: ISD::CTPOP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // popcnt(zext())
4982 { .ISD: ISD::CTPOP, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // popcnt(zext())
4983 };
4984 static const CostKindTblEntry PCLMULCostTbl[] = {
4985 { .ISD: ISD::CLMUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+2xPCLMUL+unpack
4986 { .ISD: ISD::CLMUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 18, .CodeSizeCost: 12, .SizeAndLatencyCost: 16 } }, // MOV+4xPCLMUL+unpack
4987 { .ISD: ISD::CLMUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4988 { .ISD: ISD::CLMUL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4989 { .ISD: ISD::CLMUL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4990 { .ISD: ISD::CLMUL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4991 };
4992 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
4993 { .ISD: ISD::ABS, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+CMOV
4994 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 12, .CodeSizeCost: 20, .SizeAndLatencyCost: 22 } },
4995 { .ISD: ISD::BSWAP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4996 { .ISD: ISD::CTLZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
4997 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
4998 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
4999 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
5000 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i64,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5001 { .ISD: ISD::CTTZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5002 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5003 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5004 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5005 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i64,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5006 { .ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 6, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
5007 { .ISD: ISD::ROTL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5008 { .ISD: ISD::ROTR, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5009 { .ISD: X86ISD::VROTLI, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5010 { .ISD: ISD::FSHL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 4 } },
5011 { .ISD: ISD::SADDSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5012 { .ISD: ISD::SSUBSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5013 { .ISD: ISD::UADDSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5014 { .ISD: ISD::USUBSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5015 { .ISD: ISD::SMAX, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5016 { .ISD: ISD::SMIN, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5017 { .ISD: ISD::UMAX, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5018 { .ISD: ISD::UMIN, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5019 { .ISD: ISD::SADDO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5020 { .ISD: ISD::UADDO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5021 { .ISD: ISD::SMULO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5022 { .ISD: ISD::UMULO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5023 };
5024 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
5025 { .ISD: ISD::ABS, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA or SUB+CMOV
5026 { .ISD: ISD::ABS, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA or SUB+CMOV
5027 { .ISD: ISD::ABS, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA
5028 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
5029 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
5030 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 9, .CodeSizeCost: 13, .SizeAndLatencyCost: 14 } },
5031 { .ISD: ISD::BSWAP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5032 { .ISD: ISD::BSWAP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // ROL
5033 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // BSR+XOR or BSR+XOR+CMOV
5034 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // BSR+XOR or BSR+XOR+CMOV
5035 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // BSR+XOR or BSR+XOR+CMOV
5036 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i32,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5037 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5038 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // BSR+XOR
5039 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5040 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5041 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5042 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i32,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5043 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5044 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5045 { .ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 7, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
5046 { .ISD: ISD::CTPOP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 8, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
5047 { .ISD: ISD::CTPOP, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
5048 { .ISD: ISD::ROTL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5049 { .ISD: ISD::ROTL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5050 { .ISD: ISD::ROTL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5051 { .ISD: ISD::ROTR, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5052 { .ISD: ISD::ROTR, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5053 { .ISD: ISD::ROTR, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5054 { .ISD: X86ISD::VROTLI, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5055 { .ISD: X86ISD::VROTLI, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5056 { .ISD: X86ISD::VROTLI, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5057 { .ISD: ISD::FSHL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 4 } },
5058 { .ISD: ISD::FSHL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 5 } },
5059 { .ISD: ISD::FSHL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 5 } },
5060 { .ISD: ISD::SADDSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
5061 { .ISD: ISD::SADDSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5062 { .ISD: ISD::SADDSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5063 { .ISD: ISD::SSUBSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5064 { .ISD: ISD::SSUBSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5065 { .ISD: ISD::SSUBSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5066 { .ISD: ISD::UADDSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5067 { .ISD: ISD::UADDSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5068 { .ISD: ISD::UADDSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
5069 { .ISD: ISD::USUBSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5070 { .ISD: ISD::USUBSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5071 { .ISD: ISD::USUBSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
5072 { .ISD: ISD::SMAX, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5073 { .ISD: ISD::SMAX, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5074 { .ISD: ISD::SMAX, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5075 { .ISD: ISD::SMIN, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5076 { .ISD: ISD::SMIN, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5077 { .ISD: ISD::SMIN, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5078 { .ISD: ISD::UMAX, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5079 { .ISD: ISD::UMAX, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5080 { .ISD: ISD::UMAX, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5081 { .ISD: ISD::UMIN, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5082 { .ISD: ISD::UMIN, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5083 { .ISD: ISD::UMIN, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5084 { .ISD: ISD::SADDO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5085 { .ISD: ISD::SADDO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5086 { .ISD: ISD::SADDO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5087 { .ISD: ISD::UADDO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5088 { .ISD: ISD::UADDO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5089 { .ISD: ISD::UADDO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5090 { .ISD: ISD::SMULO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5091 { .ISD: ISD::SMULO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5092 { .ISD: ISD::SMULO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5093 { .ISD: ISD::UMULO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } },
5094 { .ISD: ISD::UMULO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 9 } },
5095 { .ISD: ISD::UMULO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5096 };
5097
5098 Type *RetTy = ICA.getReturnType();
5099 Type *OpTy = RetTy;
5100 Intrinsic::ID IID = ICA.getID();
5101 unsigned ISD = ISD::DELETED_NODE;
5102 switch (IID) {
5103 default:
5104 break;
5105 case Intrinsic::abs:
5106 ISD = ISD::ABS;
5107 break;
5108 case Intrinsic::bitreverse:
5109 ISD = ISD::BITREVERSE;
5110 break;
5111 case Intrinsic::bswap:
5112 ISD = ISD::BSWAP;
5113 break;
5114 case Intrinsic::ctlz:
5115 ISD = ISD::CTLZ;
5116 break;
5117 case Intrinsic::ctpop:
5118 ISD = ISD::CTPOP;
5119 break;
5120 case Intrinsic::cttz:
5121 ISD = ISD::CTTZ;
5122 break;
5123 case Intrinsic::fshl:
5124 ISD = ISD::FSHL;
5125 if (!ICA.isTypeBasedOnly()) {
5126 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5127 if (Args[0] == Args[1]) {
5128 ISD = ISD::ROTL;
5129 // Handle uniform constant rotation amounts.
5130 // TODO: Handle funnel-shift cases.
5131 const APInt *Amt;
5132 if (Args[2] &&
5133 PatternMatch::match(V: Args[2], P: PatternMatch::m_APIntAllowPoison(Res&: Amt)))
5134 ISD = X86ISD::VROTLI;
5135 }
5136 }
5137 break;
5138 case Intrinsic::fshr:
5139 // FSHR has same costs so don't duplicate.
5140 ISD = ISD::FSHL;
5141 if (!ICA.isTypeBasedOnly()) {
5142 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5143 if (Args[0] == Args[1]) {
5144 ISD = ISD::ROTR;
5145 // Handle uniform constant rotation amount.
5146 // TODO: Handle funnel-shift cases.
5147 const APInt *Amt;
5148 if (Args[2] &&
5149 PatternMatch::match(V: Args[2], P: PatternMatch::m_APIntAllowPoison(Res&: Amt)))
5150 ISD = X86ISD::VROTLI;
5151 }
5152 }
5153 break;
5154 case Intrinsic::lrint:
5155 case Intrinsic::llrint: {
5156 // X86 can use the CVTP2SI instructions to lower lrint/llrint calls, which
5157 // have the same costs as the CVTTP2SI (fptosi) instructions
5158 const SmallVectorImpl<Type *> &ArgTys = ICA.getArgTypes();
5159 return getCastInstrCost(Opcode: Instruction::FPToSI, Dst: RetTy, Src: ArgTys[0],
5160 CCH: TTI::CastContextHint::None, CostKind);
5161 }
5162 case Intrinsic::maxnum:
5163 case Intrinsic::minnum:
5164 // FMINNUM has same costs so don't duplicate.
5165 ISD = ISD::FMAXNUM;
5166 break;
5167 case Intrinsic::sadd_sat:
5168 ISD = ISD::SADDSAT;
5169 break;
5170 case Intrinsic::smax:
5171 ISD = ISD::SMAX;
5172 break;
5173 case Intrinsic::smin:
5174 ISD = ISD::SMIN;
5175 break;
5176 case Intrinsic::smulh:
5177 ISD = ISD::MULHS;
5178 break;
5179 case Intrinsic::ssub_sat:
5180 ISD = ISD::SSUBSAT;
5181 break;
5182 case Intrinsic::uadd_sat:
5183 ISD = ISD::UADDSAT;
5184 break;
5185 case Intrinsic::umax:
5186 ISD = ISD::UMAX;
5187 break;
5188 case Intrinsic::umin:
5189 ISD = ISD::UMIN;
5190 break;
5191 case Intrinsic::usub_sat:
5192 ISD = ISD::USUBSAT;
5193 break;
5194 case Intrinsic::umulh:
5195 ISD = ISD::MULHU;
5196 break;
5197 case Intrinsic::sqrt:
5198 // The estimate sequence is costed with the division it is folded into.
5199 if (isFoldedIntoRsqrtEstimate(V: ICA.getInst(), TLI: *TLI,
5200 VT: getTypeLegalizationCost(Ty: RetTy).second,
5201 CostKind))
5202 return TTI::TCC_Free;
5203 ISD = ISD::FSQRT;
5204 break;
5205 case Intrinsic::sadd_with_overflow:
5206 case Intrinsic::ssub_with_overflow:
5207 // SSUBO has same costs so don't duplicate.
5208 ISD = ISD::SADDO;
5209 OpTy = RetTy->getContainedType(i: 0);
5210 break;
5211 case Intrinsic::uadd_with_overflow:
5212 case Intrinsic::usub_with_overflow:
5213 // USUBO has same costs so don't duplicate.
5214 ISD = ISD::UADDO;
5215 OpTy = RetTy->getContainedType(i: 0);
5216 break;
5217 case Intrinsic::smul_with_overflow:
5218 ISD = ISD::SMULO;
5219 OpTy = RetTy->getContainedType(i: 0);
5220 break;
5221 case Intrinsic::umul_with_overflow:
5222 ISD = ISD::UMULO;
5223 OpTy = RetTy->getContainedType(i: 0);
5224 break;
5225 case Intrinsic::clmul:
5226 ISD = ISD::CLMUL;
5227 break;
5228 }
5229
5230 if (ISD != ISD::DELETED_NODE) {
5231 auto adjustTableCost = [&](int ISD, unsigned Cost,
5232 std::pair<InstructionCost, MVT> LT,
5233 FastMathFlags FMF) -> InstructionCost {
5234 InstructionCost LegalizationCost = LT.first;
5235 MVT MTy = LT.second;
5236
5237 // If there are no NANs to deal with, then these are reduced to a
5238 // single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
5239 // assume is used in the non-fast case.
5240 if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
5241 if (FMF.noNaNs())
5242 return LegalizationCost * 1;
5243 }
5244
5245 // For cases where some ops can be folded into a load/store, assume free.
5246 if (MTy.isScalarInteger()) {
5247 if (ISD == ISD::BSWAP && ST->hasMOVBE() && ST->hasFastMOVBE()) {
5248 if (const Instruction *II = ICA.getInst()) {
5249 if (II->hasOneUse() && isa<StoreInst>(Val: II->user_back()))
5250 return TTI::TCC_Free;
5251 if (auto *LI = dyn_cast<LoadInst>(Val: II->getOperand(i: 0))) {
5252 if (LI->hasOneUse())
5253 return TTI::TCC_Free;
5254 }
5255 }
5256 }
5257 }
5258
5259 return LegalizationCost * (int)Cost;
5260 };
5261
5262 // Legalize the type.
5263 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: OpTy);
5264 MVT MTy = LT.second;
5265
5266 // Without BMI/LZCNT see if we're only looking for a *_ZERO_POISON cost.
5267 if (((ISD == ISD::CTTZ && !ST->hasBMI()) ||
5268 (ISD == ISD::CTLZ && !ST->hasLZCNT())) &&
5269 !MTy.isVector() && !ICA.isTypeBasedOnly()) {
5270 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5271 if (auto *Cst = dyn_cast<ConstantInt>(Val: Args[1]))
5272 if (Cst->isAllOnesValue())
5273 ISD =
5274 ISD == ISD::CTTZ ? ISD::CTTZ_ZERO_POISON : ISD::CTLZ_ZERO_POISON;
5275 }
5276
5277 // FSQRT is a single instruction.
5278 if (ISD == ISD::FSQRT && CostKind == TTI::TCK_CodeSize)
5279 return LT.first;
5280
5281 if (ST->useGLMDivSqrtCosts())
5282 if (const auto *Entry = CostTableLookup(Table: GLMCostTbl, ISD, Ty: MTy))
5283 if (auto KindCost = Entry->Cost[CostKind])
5284 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5285
5286 if (ST->useSLMArithCosts())
5287 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
5288 if (auto KindCost = Entry->Cost[CostKind])
5289 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5290
5291 if (ST->hasVBMI2())
5292 if (const auto *Entry = CostTableLookup(Table: AVX512VBMI2CostTbl, ISD, Ty: MTy))
5293 if (auto KindCost = Entry->Cost[CostKind])
5294 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5295
5296 if (ST->hasBITALG())
5297 if (const auto *Entry = CostTableLookup(Table: AVX512BITALGCostTbl, ISD, Ty: MTy))
5298 if (auto KindCost = Entry->Cost[CostKind])
5299 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5300
5301 if (ST->hasVPOPCNTDQ())
5302 if (const auto *Entry = CostTableLookup(Table: AVX512VPOPCNTDQCostTbl, ISD, Ty: MTy))
5303 if (auto KindCost = Entry->Cost[CostKind])
5304 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5305
5306 if (ST->hasGFNI())
5307 if (const auto *Entry = CostTableLookup(Table: GFNICostTbl, ISD, Ty: MTy))
5308 if (auto KindCost = Entry->Cost[CostKind])
5309 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5310
5311 if (ST->hasCDI())
5312 if (const auto *Entry = CostTableLookup(Table: AVX512CDCostTbl, ISD, Ty: MTy))
5313 if (auto KindCost = Entry->Cost[CostKind])
5314 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5315
5316 if (ST->hasBWI())
5317 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
5318 if (auto KindCost = Entry->Cost[CostKind])
5319 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5320
5321 if (ST->hasAVX512())
5322 if (const auto *Entry = CostTableLookup(Table: AVX512CostTbl, ISD, Ty: MTy))
5323 if (auto KindCost = Entry->Cost[CostKind])
5324 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5325
5326 if (ST->hasXOP())
5327 if (const auto *Entry = CostTableLookup(Table: XOPCostTbl, ISD, Ty: MTy))
5328 if (auto KindCost = Entry->Cost[CostKind])
5329 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5330
5331 if (ST->hasAVX2())
5332 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
5333 if (auto KindCost = Entry->Cost[CostKind])
5334 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5335
5336 if (ST->hasAVX())
5337 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
5338 if (auto KindCost = Entry->Cost[CostKind])
5339 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5340
5341 if (ST->hasSSE42())
5342 if (const auto *Entry = CostTableLookup(Table: SSE42CostTbl, ISD, Ty: MTy))
5343 if (auto KindCost = Entry->Cost[CostKind])
5344 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5345
5346 if (ST->hasSSE41())
5347 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
5348 if (auto KindCost = Entry->Cost[CostKind])
5349 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5350
5351 if (ST->hasSSSE3())
5352 if (const auto *Entry = CostTableLookup(Table: SSSE3CostTbl, ISD, Ty: MTy))
5353 if (auto KindCost = Entry->Cost[CostKind])
5354 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5355
5356 if (ST->hasSSE2())
5357 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
5358 if (auto KindCost = Entry->Cost[CostKind])
5359 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5360
5361 if (ST->hasSSE1())
5362 if (const auto *Entry = CostTableLookup(Table: SSE1CostTbl, ISD, Ty: MTy))
5363 if (auto KindCost = Entry->Cost[CostKind])
5364 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5365
5366 if (ST->hasBMI()) {
5367 if (ST->is64Bit())
5368 if (const auto *Entry = CostTableLookup(Table: BMI64CostTbl, ISD, Ty: MTy))
5369 if (auto KindCost = Entry->Cost[CostKind])
5370 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5371
5372 if (const auto *Entry = CostTableLookup(Table: BMI32CostTbl, ISD, Ty: MTy))
5373 if (auto KindCost = Entry->Cost[CostKind])
5374 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5375 }
5376
5377 if (ST->hasLZCNT()) {
5378 if (ST->is64Bit())
5379 if (const auto *Entry = CostTableLookup(Table: LZCNT64CostTbl, ISD, Ty: MTy))
5380 if (auto KindCost = Entry->Cost[CostKind])
5381 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5382
5383 if (const auto *Entry = CostTableLookup(Table: LZCNT32CostTbl, ISD, Ty: MTy))
5384 if (auto KindCost = Entry->Cost[CostKind])
5385 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5386 }
5387
5388 if (ST->hasPOPCNT()) {
5389 if (ST->is64Bit())
5390 if (const auto *Entry = CostTableLookup(Table: POPCNT64CostTbl, ISD, Ty: MTy))
5391 if (auto KindCost = Entry->Cost[CostKind])
5392 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5393
5394 if (const auto *Entry = CostTableLookup(Table: POPCNT32CostTbl, ISD, Ty: MTy))
5395 if (auto KindCost = Entry->Cost[CostKind])
5396 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5397 }
5398
5399 // FIXME: PCLMUL w/ AVX/AVX512 and VPCLMULQDQ are not handled properly.
5400 if (ST->hasPCLMUL())
5401 if (const auto *Entry = CostTableLookup(Table: PCLMULCostTbl, ISD, Ty: MTy))
5402 if (auto KindCost = Entry->Cost[CostKind])
5403 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5404
5405 if (ST->is64Bit())
5406 if (const auto *Entry = CostTableLookup(Table: X64CostTbl, ISD, Ty: MTy))
5407 if (auto KindCost = Entry->Cost[CostKind])
5408 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5409
5410 if (const auto *Entry = CostTableLookup(Table: X86CostTbl, ISD, Ty: MTy))
5411 if (auto KindCost = Entry->Cost[CostKind])
5412 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5413
5414 // Without arg data, we need to compute the expanded costs of custom lowered
5415 // intrinsics to prevent use of the (very low) default costs.
5416 if (ICA.isTypeBasedOnly() &&
5417 (IID == Intrinsic::fshl || IID == Intrinsic::fshr)) {
5418 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
5419 InstructionCost Cost = 0;
5420 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Or, Ty: RetTy, CostKind);
5421 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Sub, Ty: RetTy, CostKind);
5422 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Shl, Ty: RetTy, CostKind);
5423 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::LShr, Ty: RetTy, CostKind);
5424 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::And, Ty: RetTy, CostKind);
5425 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy,
5426 VecPred: CmpInst::ICMP_EQ, CostKind);
5427 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
5428 VecPred: CmpInst::ICMP_EQ, CostKind);
5429 return Cost;
5430 }
5431 }
5432
5433 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
5434}
5435
5436InstructionCost X86TTIImpl::getVectorInstrCost(
5437 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
5438 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
5439 static const CostTblEntry SLMCostTbl[] = {
5440 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i8, .Cost: 4 },
5441 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i16, .Cost: 4 },
5442 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i32, .Cost: 4 },
5443 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i64, .Cost: 7 }
5444 };
5445
5446 assert(Val->isVectorTy() && "This must be a vector type");
5447 auto *VT = cast<VectorType>(Val);
5448 if (VT->isScalableTy())
5449 return InstructionCost::getInvalid();
5450
5451 Type *ScalarType = Val->getScalarType();
5452 InstructionCost RegisterFileMoveCost = 0;
5453
5454 // Non-immediate extraction/insertion can be handled as a sequence of
5455 // aliased loads+stores via the stack.
5456 if (Index == -1U && (Opcode == Instruction::ExtractElement ||
5457 Opcode == Instruction::InsertElement)) {
5458 // TODO: On some SSE41+ targets, we expand to cmp+splat+select patterns:
5459 // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
5460
5461 // TODO: Move this to BasicTTIImpl.h? We'd need better gep + index handling.
5462 assert(isa<FixedVectorType>(Val) && "Fixed vector type expected");
5463 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
5464 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
5465
5466 // Extract - store vector to stack, load scalar.
5467 if (Opcode == Instruction::ExtractElement) {
5468 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
5469 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
5470 CostKind);
5471 }
5472 // Insert - store vector to stack, store scalar, load vector.
5473 if (Opcode == Instruction::InsertElement) {
5474 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
5475 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
5476 CostKind) +
5477 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind);
5478 }
5479 }
5480
5481 if (Index != -1U && (Opcode == Instruction::ExtractElement ||
5482 Opcode == Instruction::InsertElement)) {
5483 // Extraction of vXi1 elements are now efficiently handled by MOVMSK.
5484 if (Opcode == Instruction::ExtractElement &&
5485 ScalarType->getScalarSizeInBits() == 1 &&
5486 cast<FixedVectorType>(Val)->getNumElements() > 1)
5487 return 1;
5488
5489 // Legalize the type.
5490 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
5491
5492 // This type is legalized to a scalar type.
5493 if (!LT.second.isVector())
5494 return TTI::TCC_Free;
5495
5496 // The type may be split. Normalize the index to the new type.
5497 unsigned SizeInBits = LT.second.getSizeInBits();
5498 unsigned NumElts = LT.second.getVectorNumElements();
5499 unsigned SubNumElts = NumElts;
5500 Index = Index % NumElts;
5501
5502 // For >128-bit vectors, we need to extract higher 128-bit subvectors.
5503 // For inserts, we also need to insert the subvector back.
5504 if (SizeInBits > 128) {
5505 assert((SizeInBits % 128) == 0 && "Illegal vector");
5506 unsigned NumSubVecs = SizeInBits / 128;
5507 SubNumElts = NumElts / NumSubVecs;
5508 if (SubNumElts <= Index) {
5509 RegisterFileMoveCost += (Opcode == Instruction::InsertElement ? 2 : 1);
5510 Index %= SubNumElts;
5511 }
5512 }
5513
5514 MVT MScalarTy = LT.second.getScalarType();
5515 auto IsCheapPInsrPExtrInsertPS = [&]() {
5516 // Assume pinsr/pextr XMM <-> GPR is relatively cheap on all targets.
5517 // Inserting f32 into index0 is just movss.
5518 // Also, assume insertps is relatively cheap on all >= SSE41 targets.
5519 return (MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5520 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5521 (MScalarTy == MVT::f32 && ST->hasSSE1() && Index == 0 &&
5522 Opcode == Instruction::InsertElement) ||
5523 (MScalarTy == MVT::f32 && ST->hasSSE41() &&
5524 Opcode == Instruction::InsertElement);
5525 };
5526
5527 if (Index == 0) {
5528 // Floating point scalars are already located in index #0.
5529 // Many insertions to #0 can fold away for scalar fp-ops, so let's assume
5530 // true for all.
5531 if (ScalarType->isFloatingPointTy() &&
5532 (Opcode != Instruction::InsertElement || !Op0 ||
5533 isa<UndefValue>(Val: Op0)))
5534 return RegisterFileMoveCost;
5535
5536 if (Opcode == Instruction::InsertElement &&
5537 isa_and_nonnull<UndefValue>(Val: Op0)) {
5538 // Consider the gather cost to be cheap.
5539 if (isa_and_nonnull<LoadInst>(Val: Op1))
5540 return RegisterFileMoveCost;
5541 if (!IsCheapPInsrPExtrInsertPS()) {
5542 // mov constant-to-GPR + movd/movq GPR -> XMM.
5543 if (isa_and_nonnull<Constant>(Val: Op1) && Op1->getType()->isIntegerTy())
5544 return 2 + RegisterFileMoveCost;
5545 // Assume movd/movq GPR -> XMM is relatively cheap on all targets.
5546 return 1 + RegisterFileMoveCost;
5547 }
5548 }
5549
5550 // Assume movd/movq XMM -> GPR is relatively cheap on all targets.
5551 if (ScalarType->isIntegerTy() && Opcode == Instruction::ExtractElement)
5552 return 1 + RegisterFileMoveCost;
5553 }
5554
5555 int ISD = TLI->InstructionOpcodeToISD(Opcode);
5556 assert(ISD && "Unexpected vector opcode");
5557 if (ST->useSLMArithCosts())
5558 if (auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MScalarTy))
5559 return Entry->Cost + RegisterFileMoveCost;
5560
5561 // Consider cheap cases.
5562 if (IsCheapPInsrPExtrInsertPS())
5563 return 1 + RegisterFileMoveCost;
5564
5565 // For extractions we just need to shuffle the element to index 0, which
5566 // should be very cheap (assume cost = 1). For insertions we need to shuffle
5567 // the elements to its destination. In both cases we must handle the
5568 // subvector move(s).
5569 // If the vector type is already less than 128-bits then don't reduce it.
5570 // TODO: Under what circumstances should we shuffle using the full width?
5571 InstructionCost ShuffleCost = 1;
5572 if (Opcode == Instruction::InsertElement) {
5573 auto *SubTy = cast<VectorType>(Val);
5574 EVT VT = TLI->getValueType(DL, Ty: Val);
5575 if (VT.getScalarType() != MScalarTy || VT.getSizeInBits() >= 128)
5576 SubTy = FixedVectorType::get(ElementType: ScalarType, NumElts: SubNumElts);
5577 ShuffleCost = getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SubTy, SrcTy: SubTy,
5578 CostKind, Mask: {}, Index: 0, SubTp: SubTy);
5579 }
5580 int IntOrFpCost = ScalarType->isFloatingPointTy() ? 0 : 1;
5581 return ShuffleCost + IntOrFpCost + RegisterFileMoveCost;
5582 }
5583
5584 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
5585 VIC) +
5586 RegisterFileMoveCost;
5587}
5588
5589InstructionCost X86TTIImpl::getScalarizationOverhead(
5590 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
5591 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
5592 TTI::VectorInstrContext VIC) const {
5593 assert(DemandedElts.getBitWidth() ==
5594 cast<FixedVectorType>(Ty)->getNumElements() &&
5595 "Vector size mismatch");
5596
5597 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5598 MVT MScalarTy = LT.second.getScalarType();
5599 unsigned LegalVectorBitWidth = LT.second.getSizeInBits();
5600 InstructionCost Cost = 0;
5601 // The square roots, folded with the divisions into the estimate, are
5602 // computed separately to be inserted.
5603 if (Insert && !VL.empty())
5604 for (unsigned I : seq(Size: DemandedElts.getBitWidth()))
5605 if (DemandedElts[I] &&
5606 isFoldedIntoRsqrtEstimate(V: VL[I], TLI: *TLI, VT: MScalarTy, CostKind))
5607 Cost += getIntrinsicInstrCost(
5608 ICA: IntrinsicCostAttributes(Intrinsic::sqrt, VL[I]->getType(),
5609 {VL[I]->getType()}),
5610 CostKind);
5611
5612 constexpr unsigned LaneBitWidth = 128;
5613 assert((LegalVectorBitWidth < LaneBitWidth ||
5614 (LegalVectorBitWidth % LaneBitWidth) == 0) &&
5615 "Illegal vector");
5616
5617 const int NumLegalVectors = LT.first.getValue();
5618 assert(NumLegalVectors >= 0 && "Negative cost!");
5619
5620 // For insertions, a ISD::BUILD_VECTOR style vector initialization can be much
5621 // cheaper than an accumulation of ISD::INSERT_VECTOR_ELT. SLPVectorizer has
5622 // a special heuristic regarding poison input which is passed here in
5623 // ForPoisonSrc.
5624 if (Insert && !ForPoisonSrc) {
5625 // This is nearly identical to BaseT::getScalarizationOverhead(), except
5626 // it is passing nullptr to getVectorInstrCost() for Op0 (instead of
5627 // Constant::getNullValue()), which makes the X86TTIImpl
5628 // getVectorInstrCost() return 0 instead of 1.
5629 for (unsigned I : seq(Size: DemandedElts.getBitWidth())) {
5630 if (!DemandedElts[I])
5631 continue;
5632 Cost += getVectorInstrCost(Opcode: Instruction::InsertElement, Val: Ty, CostKind, Index: I,
5633 Op0: Constant::getNullValue(Ty),
5634 Op1: VL.empty() ? nullptr : VL[I],
5635 VIC: TTI::VectorInstrContext::None);
5636 }
5637 return Cost;
5638 }
5639
5640 if (Insert) {
5641 if ((MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5642 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5643 (MScalarTy == MVT::f32 && ST->hasSSE41())) {
5644 // For types we can insert directly, insertion into 128-bit sub vectors is
5645 // cheap, followed by a cheap chain of concatenations.
5646 if (LegalVectorBitWidth <= LaneBitWidth) {
5647 Cost += BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, Insert,
5648 /*Extract*/ false, CostKind);
5649 } else {
5650 // In each 128-lane, if at least one index is demanded but not all
5651 // indices are demanded and this 128-lane is not the first 128-lane of
5652 // the legalized-vector, then this 128-lane needs a extracti128; If in
5653 // each 128-lane, there is at least one demanded index, this 128-lane
5654 // needs a inserti128.
5655
5656 // The following cases will help you build a better understanding:
5657 // Assume we insert several elements into a v8i32 vector in avx2,
5658 // Case#1: inserting into 1th index needs vpinsrd + inserti128.
5659 // Case#2: inserting into 5th index needs extracti128 + vpinsrd +
5660 // inserti128.
5661 // Case#3: inserting into 4,5,6,7 index needs 4*vpinsrd + inserti128.
5662 assert((LegalVectorBitWidth % LaneBitWidth) == 0 && "Illegal vector");
5663 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5664 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5665 unsigned NumLegalElts =
5666 LT.second.getVectorNumElements() * NumLegalVectors;
5667 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5668 "Vector has been legalized to smaller element count");
5669 assert((NumLegalElts % NumLanesTotal) == 0 &&
5670 "Unexpected elts per lane");
5671 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5672
5673 APInt WidenedDemandedElts = DemandedElts.zext(width: NumLegalElts);
5674 auto *LaneTy =
5675 FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumEltsPerLane);
5676
5677 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5678 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5679 numBits: NumEltsPerLane, bitPosition: NumEltsPerLane * I);
5680 if (LaneEltMask.isZero())
5681 continue;
5682 // FIXME: we don't need to extract if all non-demanded elements
5683 // are legalization-inserted padding.
5684 if (!LaneEltMask.isAllOnes())
5685 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
5686 Mask: {}, Index: I * NumEltsPerLane, SubTp: LaneTy);
5687 Cost += BaseT::getScalarizationOverhead(InTy: LaneTy, DemandedElts: LaneEltMask, Insert,
5688 /*Extract*/ false, CostKind);
5689 }
5690
5691 APInt AffectedLanes =
5692 APIntOps::ScaleBitMask(A: WidenedDemandedElts, NewBitWidth: NumLanesTotal);
5693 APInt FullyAffectedLegalVectors = APIntOps::ScaleBitMask(
5694 A: AffectedLanes, NewBitWidth: NumLegalVectors, /*MatchAllBits=*/true);
5695 for (int LegalVec = 0; LegalVec != NumLegalVectors; ++LegalVec) {
5696 for (unsigned Lane = 0; Lane != NumLegalLanes; ++Lane) {
5697 unsigned I = NumLegalLanes * LegalVec + Lane;
5698 // No need to insert unaffected lane; or lane 0 of each legal vector
5699 // iff ALL lanes of that vector were affected and will be inserted.
5700 if (!AffectedLanes[I] ||
5701 (Lane == 0 && FullyAffectedLegalVectors[LegalVec]))
5702 continue;
5703 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
5704 Mask: {}, Index: I * NumEltsPerLane, SubTp: LaneTy);
5705 }
5706 }
5707 }
5708 } else if (LT.second.isVector()) {
5709 // Without fast insertion, we need to use MOVD/MOVQ to pass each demanded
5710 // integer element as a SCALAR_TO_VECTOR, then we build the vector as a
5711 // series of UNPCK followed by CONCAT_VECTORS - all of these can be
5712 // considered cheap.
5713 if (Ty->isIntOrIntVectorTy())
5714 Cost += DemandedElts.popcount();
5715
5716 // Get the smaller of the legalized or original pow2-extended number of
5717 // vector elements, which represents the number of unpacks we'll end up
5718 // performing.
5719 unsigned NumElts = LT.second.getVectorNumElements();
5720 unsigned Pow2Elts =
5721 PowerOf2Ceil(A: cast<FixedVectorType>(Val: Ty)->getNumElements());
5722 Cost += (std::min<unsigned>(a: NumElts, b: Pow2Elts) - 1) * LT.first;
5723 }
5724 }
5725
5726 if (Extract) {
5727 // vXi1 can be efficiently extracted with MOVMSK.
5728 // TODO: AVX512 predicate mask handling.
5729 // NOTE: This doesn't work well for roundtrip scalarization.
5730 if (!Insert && Ty->getScalarSizeInBits() == 1 && !ST->hasAVX512()) {
5731 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
5732 unsigned MaxElts = ST->hasAVX2() ? 32 : 16;
5733 unsigned MOVMSKCost = (NumElts + MaxElts - 1) / MaxElts;
5734 return MOVMSKCost;
5735 }
5736
5737 if (LT.second.isVector()) {
5738 unsigned NumLegalElts =
5739 LT.second.getVectorNumElements() * NumLegalVectors;
5740 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5741 "Vector has been legalized to smaller element count");
5742
5743 // If we're extracting elements from a 128-bit subvector lane,
5744 // we only need to extract each lane once, not for every element.
5745 if (LegalVectorBitWidth > LaneBitWidth) {
5746 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5747 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5748 assert((NumLegalElts % NumLanesTotal) == 0 &&
5749 "Unexpected elts per lane");
5750 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5751
5752 // Add cost for each demanded 128-bit subvector extraction.
5753 // Luckily this is a lot easier than for insertion.
5754 APInt WidenedDemandedElts = DemandedElts.zext(width: NumLegalElts);
5755 auto *LaneTy =
5756 FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumEltsPerLane);
5757
5758 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5759 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5760 numBits: NumEltsPerLane, bitPosition: I * NumEltsPerLane);
5761 if (LaneEltMask.isZero())
5762 continue;
5763 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind, Mask: {},
5764 Index: I * NumEltsPerLane, SubTp: LaneTy);
5765 Cost += BaseT::getScalarizationOverhead(
5766 InTy: LaneTy, DemandedElts: LaneEltMask, /*Insert*/ false, Extract, CostKind);
5767 }
5768
5769 return Cost;
5770 }
5771 }
5772
5773 // Fallback to default extraction.
5774 Cost += BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, /*Insert*/ false,
5775 Extract, CostKind);
5776 }
5777
5778 return Cost;
5779}
5780
5781InstructionCost
5782X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
5783 int VF, const APInt &DemandedDstElts,
5784 TTI::TargetCostKind CostKind) const {
5785 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: EltTy);
5786 // We don't differentiate element types here, only element bit width.
5787 EltTy = IntegerType::getIntNTy(C&: EltTy->getContext(), N: EltTyBits);
5788
5789 auto bailout = [&]() {
5790 return BaseT::getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
5791 DemandedDstElts, CostKind);
5792 };
5793
5794 // For now, only deal with AVX512 cases.
5795 if (!ST->hasAVX512())
5796 return bailout();
5797
5798 // Do we have a native shuffle for this element type, or should we promote?
5799 unsigned PromEltTyBits = EltTyBits;
5800 switch (EltTyBits) {
5801 case 32:
5802 case 64:
5803 break; // AVX512F.
5804 case 16:
5805 if (!ST->hasBWI())
5806 PromEltTyBits = 32; // promote to i32, AVX512F.
5807 break; // AVX512BW
5808 case 8:
5809 if (!ST->hasVBMI())
5810 PromEltTyBits = 32; // promote to i32, AVX512F.
5811 break; // AVX512VBMI
5812 case 1:
5813 // There is no support for shuffling i1 elements. We *must* promote.
5814 if (ST->hasBWI()) {
5815 if (ST->hasVBMI())
5816 PromEltTyBits = 8; // promote to i8, AVX512VBMI.
5817 else
5818 PromEltTyBits = 16; // promote to i16, AVX512BW.
5819 break;
5820 }
5821 PromEltTyBits = 32; // promote to i32, AVX512F.
5822 break;
5823 default:
5824 return bailout();
5825 }
5826 auto *PromEltTy = IntegerType::getIntNTy(C&: EltTy->getContext(), N: PromEltTyBits);
5827
5828 auto *SrcVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: VF);
5829 auto *PromSrcVecTy = FixedVectorType::get(ElementType: PromEltTy, NumElts: VF);
5830
5831 int NumDstElements = VF * ReplicationFactor;
5832 auto *PromDstVecTy = FixedVectorType::get(ElementType: PromEltTy, NumElts: NumDstElements);
5833 auto *DstVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumDstElements);
5834
5835 // Legalize the types.
5836 MVT LegalSrcVecTy = getTypeLegalizationCost(Ty: SrcVecTy).second;
5837 MVT LegalPromSrcVecTy = getTypeLegalizationCost(Ty: PromSrcVecTy).second;
5838 MVT LegalPromDstVecTy = getTypeLegalizationCost(Ty: PromDstVecTy).second;
5839 MVT LegalDstVecTy = getTypeLegalizationCost(Ty: DstVecTy).second;
5840 // They should have legalized into vector types.
5841 if (!LegalSrcVecTy.isVector() || !LegalPromSrcVecTy.isVector() ||
5842 !LegalPromDstVecTy.isVector() || !LegalDstVecTy.isVector())
5843 return bailout();
5844
5845 if (PromEltTyBits != EltTyBits) {
5846 // If we have to perform the shuffle with wider elt type than our data type,
5847 // then we will first need to anyext (we don't care about the new bits)
5848 // the source elements, and then truncate Dst elements.
5849 InstructionCost PromotionCost;
5850 PromotionCost += getCastInstrCost(
5851 Opcode: Instruction::SExt, /*Dst=*/PromSrcVecTy, /*Src=*/SrcVecTy,
5852 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5853 PromotionCost +=
5854 getCastInstrCost(Opcode: Instruction::Trunc, /*Dst=*/DstVecTy,
5855 /*Src=*/PromDstVecTy,
5856 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5857 return PromotionCost + getReplicationShuffleCost(EltTy: PromEltTy,
5858 ReplicationFactor, VF,
5859 DemandedDstElts, CostKind);
5860 }
5861
5862 assert(LegalSrcVecTy.getScalarSizeInBits() == EltTyBits &&
5863 LegalSrcVecTy.getScalarType() == LegalDstVecTy.getScalarType() &&
5864 "We expect that the legalization doesn't affect the element width, "
5865 "doesn't coalesce/split elements.");
5866
5867 unsigned NumEltsPerDstVec = LegalDstVecTy.getVectorNumElements();
5868 unsigned NumDstVectors =
5869 divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: NumEltsPerDstVec);
5870
5871 auto *SingleDstVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumEltsPerDstVec);
5872
5873 // Not all the produced Dst elements may be demanded. In our case,
5874 // given that a single Dst vector is formed by a single shuffle,
5875 // if all elements that will form a single Dst vector aren't demanded,
5876 // then we won't need to do that shuffle, so adjust the cost accordingly.
5877 APInt DemandedDstVectors = APIntOps::ScaleBitMask(
5878 A: DemandedDstElts.zext(width: NumDstVectors * NumEltsPerDstVec), NewBitWidth: NumDstVectors);
5879 unsigned NumDstVectorsDemanded = DemandedDstVectors.popcount();
5880
5881 InstructionCost SingleShuffleCost =
5882 getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleDstVecTy, SrcTy: SingleDstVecTy,
5883 CostKind, /*Mask=*/{},
5884 /*Index=*/0, /*SubTp=*/nullptr);
5885 return NumDstVectorsDemanded * SingleShuffleCost;
5886}
5887
5888InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
5889 Align Alignment,
5890 unsigned AddressSpace,
5891 TTI::TargetCostKind CostKind,
5892 TTI::OperandValueInfo OpInfo,
5893 const Instruction *I) const {
5894 // FIXME: Load latency isn't handled here
5895 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
5896 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5897 CostKind, OpInfo, I);
5898
5899 // TODO: Handle other cost kinds.
5900 if (CostKind != TTI::TCK_RecipThroughput) {
5901 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
5902 // Store instruction with index and scale costs 2 Uops.
5903 // Check the preceding GEP to identify non-const indices.
5904 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: SI->getPointerOperand())) {
5905 if (!all_of(Range: GEP->indices(), P: [](Value *V) { return isa<Constant>(Val: V); }))
5906 return TTI::TCC_Basic * 2;
5907 }
5908 }
5909 return TTI::TCC_Basic;
5910 }
5911
5912 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5913 "Invalid Opcode");
5914 // Type legalization can't handle structs
5915 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
5916 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5917 CostKind, OpInfo, I);
5918
5919 // Legalize the type.
5920 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
5921
5922 auto *VTy = dyn_cast<FixedVectorType>(Val: Src);
5923
5924 InstructionCost Cost = 0;
5925
5926 // Add a cost for constant load to vector.
5927 if (Opcode == Instruction::Store && OpInfo.isConstant())
5928 Cost += getMemoryOpCost(Opcode: Instruction::Load, Src, Alignment: DL.getABITypeAlign(Ty: Src),
5929 /*AddressSpace=*/0, CostKind, OpInfo);
5930
5931 // Handle the simple case of non-vectors.
5932 // NOTE: this assumes that legalization never creates vector from scalars!
5933 if (!VTy || !LT.second.isVector()) {
5934 // Each load/store unit costs 1.
5935 return (LT.second.isFloatingPoint() ? Cost : 0) + LT.first * 1;
5936 }
5937
5938 bool IsLoad = Opcode == Instruction::Load;
5939
5940 Type *EltTy = VTy->getElementType();
5941
5942 const int EltTyBits = DL.getTypeSizeInBits(Ty: EltTy);
5943
5944 // Source of truth: how many elements were there in the original IR vector?
5945 const unsigned SrcNumElt = VTy->getNumElements();
5946
5947 // How far have we gotten?
5948 int NumEltRemaining = SrcNumElt;
5949 // Note that we intentionally capture by-reference, NumEltRemaining changes.
5950 auto NumEltDone = [&]() { return SrcNumElt - NumEltRemaining; };
5951
5952 const int MaxLegalOpSizeBytes = divideCeil(Numerator: LT.second.getSizeInBits(), Denominator: 8);
5953
5954 // Note that even if we can store 64 bits of an XMM, we still operate on XMM.
5955 const unsigned XMMBits = 128;
5956 if (XMMBits % EltTyBits != 0)
5957 // Vector size must be a multiple of the element size. I.e. no padding.
5958 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5959 CostKind, OpInfo, I);
5960 const int NumEltPerXMM = XMMBits / EltTyBits;
5961
5962 auto *XMMVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumEltPerXMM);
5963
5964 for (int CurrOpSizeBytes = MaxLegalOpSizeBytes, SubVecEltsLeft = 0;
5965 NumEltRemaining > 0; CurrOpSizeBytes /= 2) {
5966 // How many elements would a single op deal with at once?
5967 if ((8 * CurrOpSizeBytes) % EltTyBits != 0)
5968 // Vector size must be a multiple of the element size. I.e. no padding.
5969 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5970 CostKind, OpInfo, I);
5971 int CurrNumEltPerOp = (8 * CurrOpSizeBytes) / EltTyBits;
5972
5973 assert(CurrOpSizeBytes > 0 && CurrNumEltPerOp > 0 && "How'd we get here?");
5974 assert((((NumEltRemaining * EltTyBits) < (2 * 8 * CurrOpSizeBytes)) ||
5975 (CurrOpSizeBytes == MaxLegalOpSizeBytes)) &&
5976 "Unless we haven't halved the op size yet, "
5977 "we have less than two op's sized units of work left.");
5978
5979 auto *CurrVecTy = CurrNumEltPerOp > NumEltPerXMM
5980 ? FixedVectorType::get(ElementType: EltTy, NumElts: CurrNumEltPerOp)
5981 : XMMVecTy;
5982
5983 assert(CurrVecTy->getNumElements() % CurrNumEltPerOp == 0 &&
5984 "After halving sizes, the vector elt count is no longer a multiple "
5985 "of number of elements per operation?");
5986 auto *CoalescedVecTy =
5987 CurrNumEltPerOp == 1
5988 ? CurrVecTy
5989 : FixedVectorType::get(
5990 ElementType: IntegerType::get(C&: Src->getContext(),
5991 NumBits: EltTyBits * CurrNumEltPerOp),
5992 NumElts: CurrVecTy->getNumElements() / CurrNumEltPerOp);
5993 assert(DL.getTypeSizeInBits(CoalescedVecTy) ==
5994 DL.getTypeSizeInBits(CurrVecTy) &&
5995 "coalesciing elements doesn't change vector width.");
5996
5997 while (NumEltRemaining > 0) {
5998 assert(SubVecEltsLeft >= 0 && "Subreg element count overconsumtion?");
5999
6000 // Can we use this vector size, as per the remaining element count?
6001 // Iff the vector is naturally aligned, we can do a wide load regardless.
6002 if (NumEltRemaining < CurrNumEltPerOp &&
6003 (!IsLoad || Alignment < CurrOpSizeBytes) && CurrOpSizeBytes != 1)
6004 break; // Try smalled vector size.
6005
6006 // This isn't exactly right. We're using slow unaligned 32-byte accesses
6007 // as a proxy for a double-pumped AVX memory interface such as on
6008 // Sandybridge.
6009 // Sub-32-bit loads/stores will be slower either with PINSR*/PEXTR* or
6010 // will be scalarized.
6011 //
6012 // For a vector load, each non-0th 1/2/4-byte in-lane remainder chunk is
6013 // materialized by a *single* folded PINSR*(mem) that both loads and
6014 // inserts the lane (1B->PINSRB, 2B->PINSRW, 4B->PINSRD; the byte and
6015 // dword folds need SSE4.1, the word fold only SSE2).
6016 // When that fold is available the chunk is one instruction, so it must be
6017 // priced once here (as a plain load) and the separate lane-insert charge
6018 // below must be skipped - otherwise the folded insert is double-counted.
6019 // Stores (the symmetric PEXTR*(mem) fold) are left unchanged here.
6020 bool Is0thSubVec = (NumEltDone() % LT.second.getVectorNumElements()) == 0;
6021 bool FoldedInLaneInsert = IsLoad && !Is0thSubVec &&
6022 ((CurrOpSizeBytes == 1 && ST->hasSSE41()) ||
6023 (CurrOpSizeBytes == 2 && ST->hasSSE2()) ||
6024 (CurrOpSizeBytes == 4 && ST->hasSSE41()));
6025 if (CurrOpSizeBytes == 32 && ST->isUnalignedMem32Slow())
6026 Cost += 2;
6027 else if (CurrOpSizeBytes < 4 && !FoldedInLaneInsert)
6028 Cost += 2;
6029 else
6030 Cost += 1;
6031
6032 // If we're loading a uniform value, then we don't need to split the load,
6033 // loading just a single (widest) vector can be reused by all splits.
6034 if (IsLoad && OpInfo.isUniform())
6035 return Cost;
6036
6037 // If we have fully processed the previous reg, we need to replenish it.
6038 if (SubVecEltsLeft == 0) {
6039 SubVecEltsLeft += CurrVecTy->getNumElements();
6040 // And that's free only for the 0'th subvector of a legalized vector.
6041 if (!Is0thSubVec)
6042 Cost +=
6043 getShuffleCost(Kind: IsLoad ? TTI::ShuffleKind::SK_InsertSubvector
6044 : TTI::ShuffleKind::SK_ExtractSubvector,
6045 DstTy: VTy, SrcTy: VTy, CostKind, Mask: {}, Index: NumEltDone(), SubTp: CurrVecTy);
6046 }
6047
6048 // While we can directly load/store ZMM, YMM, and 64-bit halves of XMM,
6049 // for smaller widths (32/16/8) we have to insert/extract them separately.
6050 // Again, it's free for the 0'th subreg (if op is 32/64 bit wide,
6051 // but let's pretend that it is also true for 16/8 bit wide ops...)
6052 if (CurrOpSizeBytes <= 32 / 8 && !Is0thSubVec && !FoldedInLaneInsert) {
6053 int NumEltDoneInCurrXMM = NumEltDone() % NumEltPerXMM;
6054 assert(NumEltDoneInCurrXMM % CurrNumEltPerOp == 0 && "");
6055 int CoalescedVecEltIdx = NumEltDoneInCurrXMM / CurrNumEltPerOp;
6056 APInt DemandedElts =
6057 APInt::getBitsSet(numBits: CoalescedVecTy->getNumElements(),
6058 loBit: CoalescedVecEltIdx, hiBit: CoalescedVecEltIdx + 1);
6059 assert(DemandedElts.popcount() == 1 && "Inserting single value");
6060 Cost += getScalarizationOverhead(Ty: CoalescedVecTy, DemandedElts, Insert: IsLoad,
6061 Extract: !IsLoad, CostKind);
6062 }
6063
6064 SubVecEltsLeft -= CurrNumEltPerOp;
6065 NumEltRemaining -= CurrNumEltPerOp;
6066 Alignment = commonAlignment(A: Alignment, Offset: CurrOpSizeBytes);
6067 }
6068 }
6069
6070 assert(NumEltRemaining <= 0 && "Should have processed all the elements.");
6071
6072 return Cost;
6073}
6074
6075InstructionCost
6076X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
6077 TTI::TargetCostKind CostKind) const {
6078 switch (MICA.getID()) {
6079 case Intrinsic::masked_scatter:
6080 case Intrinsic::masked_gather:
6081 return getGatherScatterOpCost(MICA, CostKind);
6082 case Intrinsic::masked_load:
6083 case Intrinsic::masked_store:
6084 return getMaskedMemoryOpCost(MICA, CostKind);
6085 case Intrinsic::masked_compressstore:
6086 // Fallback to scalarization for slow compressstore targets (e.g. znver4).
6087 if (ST->isVecCompressStoreSlow())
6088 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6089 break;
6090 }
6091
6092 static const CostKindTblEntry AVX512VBMI2CostTable[] = {
6093 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6094 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6095 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6096
6097 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6098 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6099 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6100
6101 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6102 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6103 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6104
6105 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6106 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6107 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6108 };
6109
6110 static const CostKindTblEntry AVX512CostTable[] = {
6111 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6112 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6113 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6114 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6115 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6116 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6117
6118 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6119 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6120 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6121 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6122 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6123 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6124
6125 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6126 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6127 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6128 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6129 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6130 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6131
6132 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6133 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6134 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6135 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6136 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6137 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6138 };
6139
6140 std::pair<InstructionCost, MVT> LT =
6141 getTypeLegalizationCost(Ty: MICA.getDataType());
6142
6143 if (ST->hasVBMI2())
6144 if (const auto *Entry =
6145 CostTableLookup(Table: AVX512VBMI2CostTable, ISD: MICA.getID(), Ty: LT.second))
6146 if (auto KindCost = Entry->Cost[CostKind])
6147 return LT.first * *KindCost;
6148
6149 if (ST->hasAVX512())
6150 if (const auto *Entry =
6151 CostTableLookup(Table: AVX512CostTable, ISD: MICA.getID(), Ty: LT.second))
6152 if (auto KindCost = Entry->Cost[CostKind])
6153 return LT.first * *KindCost;
6154
6155 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6156}
6157
6158InstructionCost
6159X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
6160 TTI::TargetCostKind CostKind) const {
6161 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
6162 : Instruction::Store;
6163 Type *SrcTy = MICA.getDataType();
6164 Align Alignment = MICA.getAlignment();
6165 unsigned AddressSpace = MICA.getAddressSpace();
6166
6167 bool IsLoad = (Instruction::Load == Opcode);
6168 bool IsStore = (Instruction::Store == Opcode);
6169
6170 auto *SrcVTy = dyn_cast<FixedVectorType>(Val: SrcTy);
6171 if (!SrcVTy)
6172 // To calculate scalar take the regular cost, without mask
6173 return getMemoryOpCost(Opcode, Src: SrcTy, Alignment, AddressSpace, CostKind);
6174
6175 unsigned NumElem = SrcVTy->getNumElements();
6176 auto *MaskTy =
6177 FixedVectorType::get(ElementType: Type::getInt8Ty(C&: SrcVTy->getContext()), NumElts: NumElem);
6178 if ((IsLoad && !isLegalMaskedLoad(DataType: SrcVTy, Alignment, AddressSpace)) ||
6179 (IsStore && !isLegalMaskedStore(DataType: SrcVTy, Alignment, AddressSpace))) {
6180 // Scalarization
6181 APInt DemandedElts = APInt::getAllOnes(numBits: NumElem);
6182 InstructionCost MaskSplitCost = getScalarizationOverhead(
6183 Ty: MaskTy, DemandedElts, /*Insert*/ false, /*Extract*/ true, CostKind);
6184 InstructionCost ScalarCompareCost = getCmpSelInstrCost(
6185 Opcode: Instruction::ICmp, ValTy: Type::getInt8Ty(C&: SrcVTy->getContext()), CondTy: nullptr,
6186 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
6187 InstructionCost BranchCost = getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
6188 InstructionCost MaskCmpCost = NumElem * (BranchCost + ScalarCompareCost);
6189 InstructionCost ValueSplitCost = getScalarizationOverhead(
6190 Ty: SrcVTy, DemandedElts, Insert: IsLoad, Extract: IsStore, CostKind);
6191 InstructionCost MemopCost =
6192 NumElem * BaseT::getMemoryOpCost(Opcode, Src: SrcVTy->getScalarType(),
6193 Alignment, AddressSpace, CostKind);
6194 return MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost;
6195 }
6196
6197 // Legalize the type.
6198 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcVTy);
6199 auto VT = TLI->getValueType(DL, Ty: SrcVTy);
6200 InstructionCost Cost = 0;
6201 MVT Ty = LT.second;
6202 if (Ty == MVT::i16 || Ty == MVT::i32 || Ty == MVT::i64)
6203 // APX masked load/store for scalar is cheap.
6204 return Cost + LT.first;
6205
6206 if (VT.isSimple() && Ty != VT.getSimpleVT() &&
6207 LT.second.getVectorNumElements() == NumElem)
6208 // Promotion requires extend/truncate for data and a shuffle for mask.
6209 Cost += getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SrcVTy, SrcTy: SrcVTy, CostKind, Mask: {},
6210 Index: 0, SubTp: nullptr) +
6211 getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: MaskTy, SrcTy: MaskTy, CostKind, Mask: {},
6212 Index: 0, SubTp: nullptr);
6213
6214 else if (LT.first * Ty.getVectorNumElements() > NumElem) {
6215 auto *NewMaskTy = FixedVectorType::get(ElementType: MaskTy->getElementType(),
6216 NumElts: (unsigned)LT.first.getValue() *
6217 Ty.getVectorNumElements());
6218 // Expanding requires fill mask with zeroes
6219 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: NewMaskTy, SrcTy: NewMaskTy,
6220 CostKind, Mask: {}, Index: 0, SubTp: MaskTy);
6221 }
6222
6223 // Predicate fanout (see getPredicateFanoutCost): only a variable mask needs
6224 // a per-part sub-mask; a constant mask is folded by codegen and pays nothing.
6225 InstructionCost MaskExpCost = 0;
6226 if (MICA.getVariableMask()) {
6227 auto *MaskVecTy =
6228 FixedVectorType::get(ElementType: Type::getInt1Ty(C&: SrcVTy->getContext()), NumElts: NumElem);
6229 MaskExpCost = getPredicateFanoutCost(TTI: *this, DataVTy: SrcVTy, MaskVTy: MaskVecTy);
6230 }
6231
6232 // Pre-AVX512 - each maskmov load costs 2 + store costs ~8.
6233 if (!ST->hasAVX512())
6234 return Cost + LT.first * (IsLoad ? 2 : 8) + MaskExpCost;
6235
6236 // AVX-512 masked load/store is cheaper.
6237 return Cost + LT.first + MaskExpCost;
6238}
6239
6240InstructionCost X86TTIImpl::getPointersChainCost(
6241 ArrayRef<const Value *> Ptrs, const Value *Base,
6242 const TTI::PointersChainInfo &Info, Type *AccessTy,
6243 const TTI::TargetCostKind CostKind) const {
6244 if (Info.isSameBase() && Info.isKnownStride()) {
6245 // If all the pointers have known stride all the differences are translated
6246 // into constants. X86 memory addressing allows encoding it into
6247 // displacement. So we just need to take the base GEP cost.
6248 if (const auto *BaseGEP = dyn_cast<GetElementPtrInst>(Val: Base)) {
6249 SmallVector<const Value *> Indices(BaseGEP->indices());
6250 return getGEPCost(PointeeType: BaseGEP->getSourceElementType(),
6251 Ptr: BaseGEP->getPointerOperand(), Operands: Indices, CostKind,
6252 AccessType: nullptr);
6253 }
6254 return TTI::TCC_Free;
6255 }
6256 return BaseT::getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
6257}
6258
6259InstructionCost
6260X86TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
6261 const SCEV *Ptr,
6262 TTI::TargetCostKind CostKind) const {
6263 // Address computations in vectorized code with non-consecutive addresses will
6264 // likely result in more instructions compared to scalar code where the
6265 // computation can more often be merged into the index mode. The resulting
6266 // extra micro-ops can significantly decrease throughput.
6267 const unsigned NumVectorInstToHideOverhead = 10;
6268
6269 // Cost modeling of Strided Access Computation is hidden by the indexing
6270 // modes of X86 regardless of the stride value. We dont believe that there
6271 // is a difference between constant strided access in gerenal and constant
6272 // strided value which is less than or equal to 64.
6273 // Even in the case of (loop invariant) stride whose value is not known at
6274 // compile time, the address computation will not incur more than one extra
6275 // ADD instruction.
6276 if (PtrTy->isVectorTy() && SE) {
6277 if (BaseT::isStridedAccess(Ptr) && !BaseT::getConstantStrideStep(SE, Ptr))
6278 return 1;
6279 if (!ST->hasAVX2()) {
6280 // TODO: AVX2 is the current cut-off because we don't have correct
6281 // interleaving costs for prior ISA's.
6282 if (!BaseT::isStridedAccess(Ptr))
6283 return NumVectorInstToHideOverhead;
6284 }
6285 }
6286
6287 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
6288}
6289
6290InstructionCost X86TTIImpl::getPartialReductionCost(
6291 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
6292 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
6293 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
6294 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
6295 auto ExpandCost = [&]() {
6296 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
6297 AccumType, VF, OpAExtend, OpBExtend,
6298 BinOp, CostKind, FMF);
6299 };
6300
6301 // The dot product instructions multiply-accumulate i8 x i8 -> i32,
6302 // i16 x i16 -> i32, bf16 x bf16 -> f32 or f16 x f16 -> f32. Partial
6303 // reductions may also multiply inputs extended from different types, which
6304 // they can't handle.
6305 if (VF.isScalable() || !BinOp || OpAExtend == TTI::PR_None ||
6306 OpBExtend == TTI::PR_None || InputTypeA != InputTypeB)
6307 return ExpandCost();
6308
6309 unsigned Opc;
6310 if (Opcode == Instruction::Add && *BinOp == Instruction::Mul &&
6311 AccumType->isIntegerTy(BitWidth: 32) &&
6312 (InputTypeA->isIntegerTy(BitWidth: 8) || InputTypeA->isIntegerTy(BitWidth: 16))) {
6313 if (OpAExtend != OpBExtend)
6314 Opc = ISD::PARTIAL_REDUCE_SUMLA;
6315 else if (OpAExtend == TTI::PR_SignExtend)
6316 Opc = ISD::PARTIAL_REDUCE_SMLA;
6317 else
6318 Opc = ISD::PARTIAL_REDUCE_UMLA;
6319 } else if (Opcode == Instruction::FAdd && *BinOp == Instruction::FMul &&
6320 AccumType->isFloatTy() &&
6321 (InputTypeA->isBFloatTy() || InputTypeA->isHalfTy())) {
6322 // VDPBF16PS and VDPPHPS, like the expansion, reassociate the additions and
6323 // fuse the multiplications.
6324 if (!FMF || !FMF->allowReassoc() || !FMF->allowContract())
6325 return InstructionCost::getInvalid();
6326 Opc = ISD::PARTIAL_REDUCE_FMLA;
6327 } else {
6328 return ExpandCost();
6329 }
6330
6331 unsigned Ratio =
6332 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
6333 if (!VF.isKnownMultipleOf(RHS: Ratio))
6334 return ExpandCost();
6335
6336 // One dot product per legal accumulator vector. Accumulators narrower than
6337 // a legal vector are widened by expanding the partial reduction instead.
6338 auto *AccVecTy = VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: Ratio));
6339 auto *InputVecTy = VectorType::get(ElementType: InputTypeA, EC: VF);
6340 std::pair<InstructionCost, MVT> AccLT = getTypeLegalizationCost(Ty: AccVecTy);
6341 std::pair<InstructionCost, MVT> InputLT = getTypeLegalizationCost(Ty: InputVecTy);
6342 if (AccLT.second.getFixedSizeInBits() >
6343 AccVecTy->getPrimitiveSizeInBits().getFixedValue() ||
6344 !TLI->isPartialReduceMLALegalOrCustom(Opc, AccVT: AccLT.second, InputVT: InputLT.second))
6345 return ExpandCost();
6346
6347 return AccLT.first;
6348}
6349
6350InstructionCost
6351X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
6352 std::optional<FastMathFlags> FMF,
6353 TTI::TargetCostKind CostKind) const {
6354 if (TTI::requiresOrderedReduction(FMF))
6355 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6356
6357 // We use llvm-mca across all supported CPUs to measure the logic cost stats.
6358 // We use the Intel Architecture Code Analyzer(IACA) to measure the throughput
6359 // and make it as the cost. TODO: Update old IACA numbers to llvm-mca.
6360
6361 static const CostKindTblEntry SLMCostTbl[] = {
6362 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6363 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6364 };
6365
6366 static const CostKindTblEntry SSE2CostTbl[] = {
6367 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6368 { .ISD: ISD::FADD, .Type: MVT::v2f32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6369 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6370 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // The data reported by the IACA tool is "1.6".
6371 { .ISD: ISD::ADD, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // FIXME: chosen to be less than v4i32
6372 { .ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} }, // The data reported by the IACA tool is "3.3".
6373 { .ISD: ISD::ADD, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // The data reported by the IACA tool is "4.3".
6374 { .ISD: ISD::ADD, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} }, // The data reported by the IACA tool is "4.3".
6375 { .ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // The data reported by the IACA tool is "4.3".
6376 { .ISD: ISD::ADD, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6377 { .ISD: ISD::ADD, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6378 { .ISD: ISD::ADD, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6379 { .ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6380
6381 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6382 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6383 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6384 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6385 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6386 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6387 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6388 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6389 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6390 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6391 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6392 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6393 };
6394
6395 static const CostKindTblEntry AVX1CostTbl[] = {
6396 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6397 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6398 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6399 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1} }, // The data reported by the IACA tool is "1.5".
6400 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6401 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6402 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6403 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6404
6405 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6406 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6407 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6408 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6409 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6410 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6411 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6412 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6413 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6414 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6415 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6416 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6417 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6418 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6419 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6420 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6421 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6422 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6423 };
6424
6425 static const CostKindTblEntry AVX2CostTbl[] = {
6426 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6427 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6428 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6429 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6430 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6431 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6432 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6433 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6434 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6435 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6436 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6437 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6438 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6439 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6440 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6441 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6442 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6443 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6444 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6445 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6446 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6447 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6448 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6449 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6450 };
6451
6452 static const CostKindTblEntry AVX512FCostTbl[] = {
6453 { .ISD: ISD::FADD, .Type: MVT::v8f64, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6454 { .ISD: ISD::FADD, .Type: MVT::v16f32, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6455 { .ISD: ISD::ADD, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6456 { .ISD: ISD::ADD, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6} },
6457
6458 { .ISD: ISD::AND, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6459 { .ISD: ISD::AND, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6460 { .ISD: ISD::AND, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6461 { .ISD: ISD::AND, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6462 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6463 { .ISD: ISD::OR, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6464 { .ISD: ISD::OR, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6465 { .ISD: ISD::OR, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6466 { .ISD: ISD::OR, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6467 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6468 { .ISD: ISD::XOR, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6469 { .ISD: ISD::XOR, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6470 { .ISD: ISD::XOR, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6471 { .ISD: ISD::XOR, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6472 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6473 };
6474
6475 static const CostKindTblEntry AVX512BWCostTbl[] = {
6476 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6477 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6478 };
6479
6480 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6481 assert(ISD && "Invalid opcode");
6482
6483 // Before legalizing the type, give a chance to look up illegal narrow types
6484 // in the table.
6485 // FIXME: Is there a better way to do this?
6486 EVT VT = TLI->getValueType(DL, Ty: ValTy);
6487 if (VT.isSimple()) {
6488 MVT MTy = VT.getSimpleVT();
6489 if (ST->useSLMArithCosts())
6490 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
6491 if (auto KindCost = Entry->Cost[CostKind])
6492 return *KindCost;
6493
6494 if (ST->hasBWI())
6495 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6496 if (auto KindCost = Entry->Cost[CostKind])
6497 return *KindCost;
6498
6499 if (ST->hasAVX512())
6500 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6501 if (auto KindCost = Entry->Cost[CostKind])
6502 return *KindCost;
6503
6504 if (ST->hasAVX2())
6505 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6506 if (auto KindCost = Entry->Cost[CostKind])
6507 return *KindCost;
6508
6509 if (ST->hasAVX())
6510 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6511 if (auto KindCost = Entry->Cost[CostKind])
6512 return *KindCost;
6513
6514 if (ST->hasSSE2())
6515 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6516 if (auto KindCost = Entry->Cost[CostKind])
6517 return *KindCost;
6518 }
6519
6520 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6521
6522 MVT MTy = LT.second;
6523
6524 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6525
6526 InstructionCost ArithmeticCost = 0;
6527 if (LT.first != 1 && MTy.isVector() &&
6528 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6529 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6530 auto *SingleOpTy = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6531 NumElts: MTy.getVectorNumElements());
6532 ArithmeticCost = getArithmeticInstrCost(Opcode, Ty: SingleOpTy, CostKind);
6533 ArithmeticCost *= LT.first - 1;
6534 }
6535
6536 // FIXME: These assume a naive kshift+binop lowering, which is probably
6537 // conservative in most cases.
6538 static const CostKindTblEntry AVX512BoolReduction[] = {
6539 { .ISD: ISD::AND, .Type: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6540 { .ISD: ISD::AND, .Type: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6541 { .ISD: ISD::AND, .Type: MVT::v8i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6542 { .ISD: ISD::AND, .Type: MVT::v16i1, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6543 { .ISD: ISD::AND, .Type: MVT::v32i1, .Cost: {.RecipThroughputCost: 11,.LatencyCost: 11,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6544 { .ISD: ISD::AND, .Type: MVT::v64i1, .Cost: {.RecipThroughputCost: 13,.LatencyCost: 13,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6545 { .ISD: ISD::OR, .Type: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6546 { .ISD: ISD::OR, .Type: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6547 { .ISD: ISD::OR, .Type: MVT::v8i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6548 { .ISD: ISD::OR, .Type: MVT::v16i1, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6549 { .ISD: ISD::OR, .Type: MVT::v32i1, .Cost: {.RecipThroughputCost: 11,.LatencyCost: 11,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6550 { .ISD: ISD::OR, .Type: MVT::v64i1, .Cost: {.RecipThroughputCost: 13,.LatencyCost: 13,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6551 };
6552
6553 static const CostKindTblEntry AVX2BoolReduction[] = {
6554 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6555 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6556 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6557 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6558 };
6559
6560 static const CostKindTblEntry AVX1BoolReduction[] = {
6561 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskpd + cmp
6562 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskps + cmp
6563 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6564 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6565 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskpd + cmp
6566 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskps + cmp
6567 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6568 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6569 };
6570
6571 static const CostKindTblEntry SSE2BoolReduction[] = {
6572 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskpd + cmp
6573 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskps + cmp
6574 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6575 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6576 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskpd + cmp
6577 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskps + cmp
6578 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6579 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6580 };
6581
6582 // Handle bool allof/anyof vXi1 patterns before we check legal types.
6583 if (ValVTy->getElementType()->isIntegerTy(BitWidth: 1)) {
6584 if (ISD == ISD::ADD) {
6585 // vXi1 addition reduction will bitcast to scalar and perform a popcount.
6586 auto *IntTy = IntegerType::getIntNTy(C&: ValVTy->getContext(),
6587 N: ValVTy->getNumElements());
6588 IntrinsicCostAttributes ICA(Intrinsic::ctpop, IntTy, {IntTy});
6589 return getCastInstrCost(Opcode: Instruction::BitCast, Dst: IntTy, Src: ValVTy,
6590 CCH: TargetTransformInfo::CastContextHint::None,
6591 CostKind) +
6592 getIntrinsicInstrCost(ICA, CostKind);
6593 }
6594
6595 if (ST->hasAVX512())
6596 if (const auto *Entry = CostTableLookup(Table: AVX512BoolReduction, ISD, Ty: MTy))
6597 if (auto KindCost = Entry->Cost[CostKind])
6598 return ArithmeticCost + *KindCost;
6599 if (ST->hasAVX2())
6600 if (const auto *Entry = CostTableLookup(Table: AVX2BoolReduction, ISD, Ty: MTy))
6601 if (auto KindCost = Entry->Cost[CostKind])
6602 return ArithmeticCost + *KindCost;
6603 if (ST->hasAVX())
6604 if (const auto *Entry = CostTableLookup(Table: AVX1BoolReduction, ISD, Ty: MTy))
6605 if (auto KindCost = Entry->Cost[CostKind])
6606 return ArithmeticCost + *KindCost;
6607 if (ST->hasSSE2())
6608 if (const auto *Entry = CostTableLookup(Table: SSE2BoolReduction, ISD, Ty: MTy))
6609 if (auto KindCost = Entry->Cost[CostKind])
6610 return ArithmeticCost + *KindCost;
6611
6612 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValVTy, FMF, CostKind);
6613 }
6614
6615 // Special case: vXi8 mul reductions are performed as vXi16.
6616 if (ISD == ISD::MUL && MTy.getScalarType() == MVT::i8) {
6617 auto *WideSclTy = IntegerType::get(C&: ValVTy->getContext(), NumBits: 16);
6618 auto *WideVecTy = FixedVectorType::get(ElementType: WideSclTy, NumElts: ValVTy->getNumElements());
6619 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideVecTy, Src: ValTy,
6620 CCH: TargetTransformInfo::CastContextHint::None,
6621 CostKind) +
6622 getArithmeticReductionCost(Opcode, ValTy: WideVecTy, FMF, CostKind);
6623 }
6624
6625 if (ST->useSLMArithCosts())
6626 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
6627 if (auto KindCost = Entry->Cost[CostKind])
6628 return ArithmeticCost + *KindCost;
6629
6630 if (ST->hasBWI())
6631 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6632 if (auto KindCost = Entry->Cost[CostKind])
6633 return ArithmeticCost + *KindCost;
6634
6635 if (ST->hasAVX512())
6636 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6637 if (auto KindCost = Entry->Cost[CostKind])
6638 return ArithmeticCost + *KindCost;
6639
6640 if (ST->hasAVX2())
6641 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6642 if (auto KindCost = Entry->Cost[CostKind])
6643 return ArithmeticCost + *KindCost;
6644
6645 if (ST->hasAVX())
6646 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6647 if (auto KindCost = Entry->Cost[CostKind])
6648 return ArithmeticCost + *KindCost;
6649
6650 if (ST->hasSSE2())
6651 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6652 if (auto KindCost = Entry->Cost[CostKind])
6653 return ArithmeticCost + *KindCost;
6654
6655 unsigned NumVecElts = ValVTy->getNumElements();
6656 unsigned ScalarSize = ValVTy->getScalarSizeInBits();
6657
6658 // Special case power of 2 reductions where the scalar type isn't changed
6659 // by type legalization.
6660 if (!isPowerOf2_32(Value: NumVecElts) || ScalarSize != MTy.getScalarSizeInBits())
6661 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValVTy, FMF, CostKind);
6662
6663 InstructionCost ReductionCost = 0;
6664
6665 auto *Ty = ValVTy;
6666 if (LT.first != 1 && MTy.isVector() &&
6667 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6668 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6669 Ty = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6670 NumElts: MTy.getVectorNumElements());
6671 ReductionCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
6672 ReductionCost *= LT.first - 1;
6673 NumVecElts = MTy.getVectorNumElements();
6674 }
6675
6676 // Now handle reduction with the legal type, taking into account size changes
6677 // at each level.
6678 while (NumVecElts > 1) {
6679 // Determine the size of the remaining vector we need to reduce.
6680 unsigned Size = NumVecElts * ScalarSize;
6681 NumVecElts /= 2;
6682 // If we're reducing from 256/512 bits, use an extract_subvector.
6683 if (Size > 128) {
6684 auto *SubTy = FixedVectorType::get(ElementType: ValVTy->getElementType(), NumElts: NumVecElts);
6685 ReductionCost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty,
6686 CostKind, Mask: {}, Index: NumVecElts, SubTp: SubTy);
6687 Ty = SubTy;
6688 } else if (Size == 128) {
6689 // Reducing from 128 bits is a permute of v2f64/v2i64.
6690 FixedVectorType *ShufTy;
6691 if (ValVTy->isFloatingPointTy())
6692 ShufTy =
6693 FixedVectorType::get(ElementType: Type::getDoubleTy(C&: ValVTy->getContext()), NumElts: 2);
6694 else
6695 ShufTy =
6696 FixedVectorType::get(ElementType: Type::getInt64Ty(C&: ValVTy->getContext()), NumElts: 2);
6697 ReductionCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6698 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6699 } else if (Size == 64) {
6700 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
6701 FixedVectorType *ShufTy;
6702 if (ValVTy->isFloatingPointTy())
6703 ShufTy =
6704 FixedVectorType::get(ElementType: Type::getFloatTy(C&: ValVTy->getContext()), NumElts: 4);
6705 else
6706 ShufTy =
6707 FixedVectorType::get(ElementType: Type::getInt32Ty(C&: ValVTy->getContext()), NumElts: 4);
6708 ReductionCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6709 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6710 } else {
6711 // Reducing from smaller size is a shift by immediate.
6712 auto *ShiftTy = FixedVectorType::get(
6713 ElementType: Type::getIntNTy(C&: ValVTy->getContext(), N: Size), NumElts: 128 / Size);
6714 ReductionCost += getArithmeticInstrCost(
6715 Opcode: Instruction::LShr, Ty: ShiftTy, CostKind,
6716 Op1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
6717 Op2Info: {.Kind: TargetTransformInfo::OK_UniformConstantValue, .Properties: TargetTransformInfo::OP_None});
6718 }
6719
6720 // Add the arithmetic op for this level.
6721 ReductionCost += getArithmeticInstrCost(Opcode, Ty, CostKind);
6722 }
6723
6724 // Add the final extract element to the cost.
6725 return ReductionCost + getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty,
6726 CostKind, Index: 0, Op0: nullptr, Op1: nullptr,
6727 VIC: TTI::VectorInstrContext::None);
6728}
6729
6730InstructionCost X86TTIImpl::getMinMaxCost(Intrinsic::ID IID, Type *Ty,
6731 TTI::TargetCostKind CostKind,
6732 FastMathFlags FMF) const {
6733 IntrinsicCostAttributes ICA(IID, Ty, {Ty, Ty}, FMF);
6734 return getIntrinsicInstrCost(ICA, CostKind);
6735}
6736
6737InstructionCost
6738X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
6739 FastMathFlags FMF,
6740 TTI::TargetCostKind CostKind) const {
6741 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6742
6743 MVT MTy = LT.second;
6744
6745 ISD::NodeType ISD;
6746 if (ValTy->isIntOrIntVectorTy()) {
6747 ISD = (IID == Intrinsic::umin || IID == Intrinsic::umax) ? ISD::UMIN
6748 : ISD::SMIN;
6749 } else {
6750 assert(ValTy->isFPOrFPVectorTy() &&
6751 "Expected float point or integer vector type.");
6752 ISD = (IID == Intrinsic::minnum || IID == Intrinsic::maxnum)
6753 ? ISD::FMINNUM
6754 : ISD::FMINIMUM;
6755 }
6756
6757 // We use llvm-mca across all supported CPUs to measure the cost stats.
6758 static const CostKindTblEntry SSE2CostTbl[] = {
6759 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6760 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6761 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6762 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6763 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 12}},
6764 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7,.CodeSizeCost: 14,.SizeAndLatencyCost: 15}},
6765 {.ISD: ISD::SMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6766 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6767 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6768 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 10}},
6769 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}},
6770 {.ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8,.CodeSizeCost: 12,.SizeAndLatencyCost: 14}},
6771 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6772 {.ISD: ISD::UMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6773 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 6,.CodeSizeCost: 12,.SizeAndLatencyCost: 13}},
6774 {.ISD: ISD::UMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6775 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 9,.CodeSizeCost: 18,.SizeAndLatencyCost: 19}},
6776 {.ISD: ISD::UMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6777 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 7,.LatencyCost: 13,.CodeSizeCost: 24,.SizeAndLatencyCost: 25}},
6778 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11}},
6779 };
6780
6781 static const CostKindTblEntry SSE41CostTbl[] = {
6782 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6783 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6784 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6785 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6786 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6787 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6788 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6789 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6790 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6791 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5}},
6792 {.ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}},
6793 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6794 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6795 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6796 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 8}},
6797 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6798 };
6799
6800 static const CostKindTblEntry AVX1CostTbl[] = {
6801 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10}},
6802 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 12,.CodeSizeCost: 10,.SizeAndLatencyCost: 13}},
6803 {.ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6804 {.ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6805 {.ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 15, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6806 {.ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6807 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6808 {.ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6809 };
6810
6811 static const CostKindTblEntry AVX2CostTbl[] = {
6812 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10}},
6813 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12,.CodeSizeCost: 10,.SizeAndLatencyCost: 13}},
6814 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6815 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6816 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6817 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6818 {.ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6819 {.ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6820 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6821 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6822 {.ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 15, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6823 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6824 {.ISD: ISD::UMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6825 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6826 };
6827
6828 static const CostKindTblEntry AVX512FCostTbl[] = {
6829 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6830 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6831 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6832 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6833 {.ISD: ISD::SMIN, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 16, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6834 {.ISD: ISD::UMIN, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 16, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6835 {.ISD: ISD::SMIN, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6836 {.ISD: ISD::UMIN, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6837 };
6838
6839 static const CostKindTblEntry AVX512BWCostTbl[] = {
6840 {.ISD: ISD::SMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6841 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6842 {.ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 19, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6843 {.ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 12, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6844 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6845 {.ISD: ISD::UMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6846 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6847 {.ISD: ISD::UMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6848 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6849 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6850 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6851 {.ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6852 {.ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 21,.CodeSizeCost: 10,.SizeAndLatencyCost: 11}},
6853 {.ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 14, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}},
6854 };
6855
6856 // Before legalizing the type, give a chance to look up illegal narrow types
6857 // in the table.
6858 // FIXME: Is there a better way to do this?
6859 EVT VT = TLI->getValueType(DL, Ty: ValTy);
6860 if (VT.isSimple()) {
6861 MVT MTy = VT.getSimpleVT();
6862 if (ST->hasBWI())
6863 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6864 if (auto KindCost = Entry->Cost[CostKind])
6865 return *KindCost;
6866
6867 if (ST->hasAVX512())
6868 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6869 if (auto KindCost = Entry->Cost[CostKind])
6870 return *KindCost;
6871
6872 if (ST->hasAVX2())
6873 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6874 if (auto KindCost = Entry->Cost[CostKind])
6875 return *KindCost;
6876
6877 if (ST->hasAVX())
6878 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6879 if (auto KindCost = Entry->Cost[CostKind])
6880 return *KindCost;
6881
6882 if (ST->hasSSE41())
6883 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
6884 if (auto KindCost = Entry->Cost[CostKind])
6885 return *KindCost;
6886
6887 if (ST->hasSSE2())
6888 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6889 if (auto KindCost = Entry->Cost[CostKind])
6890 return *KindCost;
6891 }
6892
6893 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6894 unsigned NumVecElts = ValVTy->getNumElements();
6895
6896 auto *Ty = ValVTy;
6897 InstructionCost MinMaxCost = 0;
6898 if (LT.first != 1 && MTy.isVector() &&
6899 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6900 // Type needs to be split. We need LT.first - 1 operations ops.
6901 Ty = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6902 NumElts: MTy.getVectorNumElements());
6903 MinMaxCost = getMinMaxCost(IID, Ty, CostKind, FMF);
6904 MinMaxCost *= LT.first - 1;
6905 NumVecElts = MTy.getVectorNumElements();
6906 }
6907
6908 if (ST->hasBWI())
6909 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6910 if (auto KindCost = Entry->Cost[CostKind])
6911 return MinMaxCost + *KindCost;
6912
6913 if (ST->hasAVX512())
6914 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6915 if (auto KindCost = Entry->Cost[CostKind])
6916 return MinMaxCost + *KindCost;
6917
6918 if (ST->hasAVX2())
6919 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6920 if (auto KindCost = Entry->Cost[CostKind])
6921 return MinMaxCost + *KindCost;
6922
6923 if (ST->hasAVX())
6924 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6925 if (auto KindCost = Entry->Cost[CostKind])
6926 return MinMaxCost + *KindCost;
6927
6928 if (ST->hasSSE41())
6929 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
6930 if (auto KindCost = Entry->Cost[CostKind])
6931 return MinMaxCost + *KindCost;
6932
6933 if (ST->hasSSE2())
6934 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6935 if (auto KindCost = Entry->Cost[CostKind])
6936 return MinMaxCost + *KindCost;
6937
6938 unsigned ScalarSize = ValTy->getScalarSizeInBits();
6939
6940 // Special case power of 2 reductions where the scalar type isn't changed
6941 // by type legalization.
6942 if (!isPowerOf2_32(Value: ValVTy->getNumElements()) ||
6943 ScalarSize != MTy.getScalarSizeInBits())
6944 return BaseT::getMinMaxReductionCost(IID, Ty: ValTy, FMF, CostKind);
6945
6946 // Now handle reduction with the legal type, taking into account size changes
6947 // at each level.
6948 while (NumVecElts > 1) {
6949 // Determine the size of the remaining vector we need to reduce.
6950 unsigned Size = NumVecElts * ScalarSize;
6951 NumVecElts /= 2;
6952 // If we're reducing from 256/512 bits, use an extract_subvector.
6953 if (Size > 128) {
6954 auto *SubTy = FixedVectorType::get(ElementType: ValVTy->getElementType(), NumElts: NumVecElts);
6955 MinMaxCost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
6956 Mask: {}, Index: NumVecElts, SubTp: SubTy);
6957 Ty = SubTy;
6958 } else if (Size == 128) {
6959 // Reducing from 128 bits is a permute of v2f64/v2i64.
6960 VectorType *ShufTy;
6961 if (ValTy->isFloatingPointTy())
6962 ShufTy =
6963 FixedVectorType::get(ElementType: Type::getDoubleTy(C&: ValTy->getContext()), NumElts: 2);
6964 else
6965 ShufTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C&: ValTy->getContext()), NumElts: 2);
6966 MinMaxCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6967 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6968 } else if (Size == 64) {
6969 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
6970 FixedVectorType *ShufTy;
6971 if (ValTy->isFloatingPointTy())
6972 ShufTy = FixedVectorType::get(ElementType: Type::getFloatTy(C&: ValTy->getContext()), NumElts: 4);
6973 else
6974 ShufTy = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: ValTy->getContext()), NumElts: 4);
6975 MinMaxCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6976 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6977 } else {
6978 // Reducing from smaller size is a shift by immediate.
6979 auto *ShiftTy = FixedVectorType::get(
6980 ElementType: Type::getIntNTy(C&: ValTy->getContext(), N: Size), NumElts: 128 / Size);
6981 MinMaxCost += getArithmeticInstrCost(
6982 Opcode: Instruction::LShr, Ty: ShiftTy, CostKind: TTI::TCK_RecipThroughput,
6983 Op1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
6984 Op2Info: {.Kind: TargetTransformInfo::OK_UniformConstantValue, .Properties: TargetTransformInfo::OP_None});
6985 }
6986
6987 // Add the arithmetic op for this level.
6988 MinMaxCost += getMinMaxCost(IID, Ty, CostKind, FMF);
6989 }
6990
6991 // Add the final extract element to the cost.
6992 return MinMaxCost + getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty,
6993 CostKind, Index: 0, Op0: nullptr, Op1: nullptr,
6994 VIC: TTI::VectorInstrContext::None);
6995}
6996
6997/// Calculate the cost of materializing a 64-bit value. This helper
6998/// method might only calculate a fraction of a larger immediate. Therefore it
6999/// is valid to return a cost of ZERO.
7000InstructionCost X86TTIImpl::getIntImmCost(int64_t Val) const {
7001 if (Val == 0)
7002 return TTI::TCC_Free;
7003
7004 if (isInt<32>(x: Val))
7005 return TTI::TCC_Basic;
7006
7007 return 2 * TTI::TCC_Basic;
7008}
7009
7010InstructionCost X86TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
7011 TTI::TargetCostKind CostKind) const {
7012 assert(Ty->isIntegerTy());
7013
7014 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7015 if (BitSize == 0)
7016 return ~0U;
7017
7018 // Never hoist constants larger than 128bit, because this might lead to
7019 // incorrect code generation or assertions in codegen.
7020 // Fixme: Create a cost model for types larger than i128 once the codegen
7021 // issues have been fixed.
7022 if (BitSize > 128)
7023 return TTI::TCC_Free;
7024
7025 if (Imm == 0)
7026 return TTI::TCC_Free;
7027
7028 // Sign-extend all constants to a multiple of 64-bit.
7029 APInt ImmVal = Imm;
7030 if (BitSize % 64 != 0)
7031 ImmVal = Imm.sext(width: alignTo(Value: BitSize, Align: 64));
7032
7033 // Split the constant into 64-bit chunks and calculate the cost for each
7034 // chunk.
7035 InstructionCost Cost = 0;
7036 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
7037 APInt Tmp = ImmVal.ashr(ShiftAmt: ShiftVal).sextOrTrunc(width: 64);
7038 int64_t Val = Tmp.getSExtValue();
7039 Cost += getIntImmCost(Val);
7040 }
7041 // We need at least one instruction to materialize the constant.
7042 return std::max<InstructionCost>(a: 1, b: Cost);
7043}
7044
7045InstructionCost X86TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
7046 const APInt &Imm, Type *Ty,
7047 TTI::TargetCostKind CostKind,
7048 Instruction *Inst) const {
7049 assert(Ty->isIntegerTy());
7050
7051 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7052 unsigned ImmBitWidth = Imm.getBitWidth();
7053
7054 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7055 // here, so that constant hoisting will ignore this constant.
7056 if (BitSize == 0)
7057 return TTI::TCC_Free;
7058
7059 unsigned ImmIdx = ~0U;
7060 switch (Opcode) {
7061 default:
7062 return TTI::TCC_Free;
7063 case Instruction::GetElementPtr:
7064 // Always hoist the base address of a GetElementPtr. This prevents the
7065 // creation of new constants for every base constant that gets constant
7066 // folded with the offset.
7067 if (Idx == 0)
7068 return 2 * TTI::TCC_Basic;
7069 return TTI::TCC_Free;
7070 case Instruction::Store:
7071 ImmIdx = 0;
7072 break;
7073 case Instruction::ICmp:
7074 // This is an imperfect hack to prevent constant hoisting of
7075 // compares that might be trying to check if a 64-bit value fits in
7076 // 32-bits. The backend can optimize these cases using a right shift by 32.
7077 // There are other predicates and immediates the backend can use shifts for.
7078 if (Idx == 1 && ImmBitWidth == 64) {
7079 uint64_t ImmVal = Imm.getZExtValue();
7080 if (ImmVal == 0x100000000ULL || ImmVal == 0xffffffff)
7081 return TTI::TCC_Free;
7082
7083 if (auto *Cmp = dyn_cast_or_null<CmpInst>(Val: Inst)) {
7084 if (Cmp->isEquality()) {
7085 KnownBits Known = computeKnownBits(V: Cmp->getOperand(i_nocapture: 0), DL);
7086 if (Known.countMinTrailingZeros() >= 32)
7087 return TTI::TCC_Free;
7088 }
7089 }
7090 }
7091 ImmIdx = 1;
7092 break;
7093 case Instruction::And:
7094 // We support 64-bit ANDs with immediates with 32-bits of leading zeroes
7095 // by using a 32-bit operation with implicit zero extension. Detect such
7096 // immediates here as the normal path expects bit 31 to be sign extended.
7097 if (Idx == 1 && ImmBitWidth == 64 && Imm.isIntN(N: 32))
7098 return TTI::TCC_Free;
7099 // If we have BMI then we can use BEXTR/BZHI to mask out upper i64 bits.
7100 if (Idx == 1 && ImmBitWidth == 64 && ST->is64Bit() && ST->hasBMI() &&
7101 Imm.isMask())
7102 return X86TTIImpl::getIntImmCost(Val: ST->hasBMI2() ? 255 : 65535);
7103 ImmIdx = 1;
7104 break;
7105 case Instruction::Add:
7106 case Instruction::Sub:
7107 // For add/sub, we can use the opposite instruction for INT32_MIN.
7108 if (Idx == 1 && ImmBitWidth == 64 && Imm.getZExtValue() == 0x80000000)
7109 return TTI::TCC_Free;
7110 ImmIdx = 1;
7111 break;
7112 case Instruction::UDiv:
7113 case Instruction::SDiv:
7114 case Instruction::URem:
7115 case Instruction::SRem:
7116 // Division by constant is typically expanded later into a different
7117 // instruction sequence. This completely changes the constants.
7118 // Report them as "free" to stop ConstantHoist from marking them as opaque.
7119 return TTI::TCC_Free;
7120 case Instruction::Mul:
7121 case Instruction::Or:
7122 case Instruction::Xor:
7123 ImmIdx = 1;
7124 break;
7125 // Always return TCC_Free for the shift value of a shift instruction.
7126 case Instruction::Shl:
7127 case Instruction::LShr:
7128 case Instruction::AShr:
7129 if (Idx == 1)
7130 return TTI::TCC_Free;
7131 break;
7132 case Instruction::Trunc:
7133 case Instruction::ZExt:
7134 case Instruction::SExt:
7135 case Instruction::IntToPtr:
7136 case Instruction::PtrToInt:
7137 case Instruction::BitCast:
7138 case Instruction::PHI:
7139 case Instruction::Call:
7140 case Instruction::Select:
7141 case Instruction::Ret:
7142 case Instruction::Load:
7143 break;
7144 }
7145
7146 if (Idx == ImmIdx) {
7147 uint64_t NumConstants = divideCeil(Numerator: BitSize, Denominator: 64);
7148 InstructionCost Cost = X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7149 return (Cost <= NumConstants * TTI::TCC_Basic)
7150 ? static_cast<int>(TTI::TCC_Free)
7151 : Cost;
7152 }
7153
7154 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7155}
7156
7157InstructionCost
7158X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
7159 const APInt &Imm, Type *Ty,
7160 TTI::TargetCostKind CostKind) const {
7161 assert(Ty->isIntegerTy());
7162
7163 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7164 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7165 // here, so that constant hoisting will ignore this constant.
7166 if (BitSize == 0)
7167 return TTI::TCC_Free;
7168
7169 switch (IID) {
7170 default:
7171 return TTI::TCC_Free;
7172 case Intrinsic::sadd_with_overflow:
7173 case Intrinsic::uadd_with_overflow:
7174 case Intrinsic::ssub_with_overflow:
7175 case Intrinsic::usub_with_overflow:
7176 case Intrinsic::smul_with_overflow:
7177 case Intrinsic::umul_with_overflow:
7178 if ((Idx == 1) && Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 32))
7179 return TTI::TCC_Free;
7180 break;
7181 case Intrinsic::experimental_stackmap:
7182 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 64)))
7183 return TTI::TCC_Free;
7184 break;
7185 case Intrinsic::experimental_patchpoint_void:
7186 case Intrinsic::experimental_patchpoint:
7187 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 64)))
7188 return TTI::TCC_Free;
7189 break;
7190 }
7191 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7192}
7193
7194InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
7195 TTI::TargetCostKind CostKind,
7196 const Instruction *I) const {
7197 if (CostKind != TTI::TCK_RecipThroughput)
7198 return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
7199 // Branches are assumed to be predicted.
7200 return TTI::TCC_Free;
7201}
7202
7203int X86TTIImpl::getGatherOverhead() const {
7204 // Some CPUs have more overhead for gather. The specified overhead is relative
7205 // to the Load operation. "2" is the number provided by Intel architects. This
7206 // parameter is used for cost estimation of Gather Op and comparison with
7207 // other alternatives.
7208 // TODO: Remove the explicit hasAVX512()?, That would mean we would only
7209 // enable gather with a -march.
7210 if (ST->hasAVX512() || (ST->hasAVX2() && ST->hasFastGather()))
7211 return 2;
7212
7213 return 1024;
7214}
7215
7216int X86TTIImpl::getScatterOverhead() const {
7217 if (ST->hasAVX512())
7218 return 2;
7219
7220 return 1024;
7221}
7222
7223// Return an average cost of Gather / Scatter instruction, maybe improved later.
7224InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
7225 TTI::TargetCostKind CostKind,
7226 Type *SrcVTy, const Value *Ptr,
7227 Align Alignment,
7228 unsigned AddressSpace) const {
7229
7230 assert(isa<VectorType>(SrcVTy) && "Unexpected type in getGSVectorCost");
7231 unsigned VF = cast<FixedVectorType>(Val: SrcVTy)->getNumElements();
7232
7233 // Try to reduce index size from 64 bit (default for GEP)
7234 // to 32. It is essential for VF 16. If the index can't be reduced to 32, the
7235 // operation will use 16 x 64 indices which do not fit in a zmm and needs
7236 // to split. Also check that the base pointer is the same for all lanes,
7237 // and that there's at most one variable index.
7238 auto getIndexSizeInBits = [](const Value *Ptr, const DataLayout &DL) {
7239 unsigned IndexSize = DL.getPointerSizeInBits();
7240 const GetElementPtrInst *GEP = dyn_cast_or_null<GetElementPtrInst>(Val: Ptr);
7241 if (IndexSize < 64 || !GEP)
7242 return IndexSize;
7243
7244 unsigned NumOfVarIndices = 0;
7245 const Value *Ptrs = GEP->getPointerOperand();
7246 if (Ptrs->getType()->isVectorTy() && !getSplatValue(V: Ptrs))
7247 return IndexSize;
7248 for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
7249 if (isa<Constant>(Val: GEP->getOperand(i_nocapture: I)))
7250 continue;
7251 Type *IndxTy = GEP->getOperand(i_nocapture: I)->getType();
7252 if (auto *IndexVTy = dyn_cast<VectorType>(Val: IndxTy))
7253 IndxTy = IndexVTy->getElementType();
7254 if ((IndxTy->getPrimitiveSizeInBits() == 64 &&
7255 !isa<SExtInst>(Val: GEP->getOperand(i_nocapture: I))) ||
7256 ++NumOfVarIndices > 1)
7257 return IndexSize; // 64
7258 }
7259 return (unsigned)32;
7260 };
7261
7262 // Trying to reduce IndexSize to 32 bits for vector 16.
7263 // By default the IndexSize is equal to pointer size.
7264 unsigned IndexSize = (ST->hasAVX512() && VF >= 16)
7265 ? getIndexSizeInBits(Ptr, DL)
7266 : DL.getPointerSizeInBits();
7267
7268 auto *IndexVTy = FixedVectorType::get(
7269 ElementType: IntegerType::get(C&: SrcVTy->getContext(), NumBits: IndexSize), NumElts: VF);
7270 std::pair<InstructionCost, MVT> IdxsLT = getTypeLegalizationCost(Ty: IndexVTy);
7271 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: SrcVTy);
7272 InstructionCost::CostType SplitFactor =
7273 std::max(a: IdxsLT.first, b: SrcLT.first).getValue();
7274 if (SplitFactor > 1) {
7275 // Handle splitting of vector of pointers
7276 auto *SplitSrcTy =
7277 FixedVectorType::get(ElementType: SrcVTy->getScalarType(), NumElts: VF / SplitFactor);
7278 return SplitFactor * getGSVectorCost(Opcode, CostKind, SrcVTy: SplitSrcTy, Ptr,
7279 Alignment, AddressSpace);
7280 }
7281
7282 // If we didn't split, this will be a single gather/scatter instruction.
7283 if (CostKind == TTI::TCK_CodeSize)
7284 return 1;
7285
7286 // The gather / scatter cost is given by Intel architects. It is a rough
7287 // number since we are looking at one instruction in a time.
7288 const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
7289 : getScatterOverhead();
7290 return GSOverhead + VF * getMemoryOpCost(Opcode, Src: SrcVTy->getScalarType(),
7291 Alignment, AddressSpace, CostKind);
7292}
7293
7294/// Calculate the cost of Gather / Scatter operation
7295InstructionCost
7296X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
7297 TTI::TargetCostKind CostKind) const {
7298 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
7299 MICA.getID() == Intrinsic::vp_gather;
7300 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
7301 Type *SrcVTy = MICA.getDataType();
7302 const Value *Ptr = MICA.getPointer();
7303 Align Alignment = MICA.getAlignment();
7304 if ((Opcode == Instruction::Load &&
7305 (!isLegalMaskedGather(DataType: SrcVTy, Alignment: Align(Alignment)) ||
7306 forceScalarizeMaskedGather(VTy: cast<VectorType>(Val: SrcVTy),
7307 Alignment: Align(Alignment)))) ||
7308 (Opcode == Instruction::Store &&
7309 (!isLegalMaskedScatter(DataType: SrcVTy, Alignment: Align(Alignment)) ||
7310 forceScalarizeMaskedScatter(VTy: cast<VectorType>(Val: SrcVTy),
7311 Alignment: Align(Alignment)))))
7312 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
7313
7314 assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
7315 unsigned AddressSpace = MICA.getAddressSpace();
7316 InstructionCost Cost =
7317 getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment, AddressSpace);
7318
7319 // Predicate fanout (see getPredicateFanoutCost); variable mask only.
7320 if (MICA.getVariableMask()) {
7321 auto *MaskVecTy =
7322 FixedVectorType::get(ElementType: Type::getInt1Ty(C&: SrcVTy->getContext()),
7323 NumElts: cast<FixedVectorType>(Val: SrcVTy)->getNumElements());
7324 Cost += getPredicateFanoutCost(TTI: *this, DataVTy: SrcVTy, MaskVTy: MaskVecTy);
7325 }
7326 return Cost;
7327}
7328
7329bool X86TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
7330 const TargetTransformInfo::LSRCost &C2) const {
7331 // X86 specific here are "instruction number 1st priority".
7332 return std::tie(args: C1.Insns, args: C1.NumRegs, args: C1.AddRecCost, args: C1.NumIVMuls,
7333 args: C1.NumBaseAdds, args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
7334 std::tie(args: C2.Insns, args: C2.NumRegs, args: C2.AddRecCost, args: C2.NumIVMuls,
7335 args: C2.NumBaseAdds, args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
7336}
7337
7338bool X86TTIImpl::canMacroFuseCmp() const {
7339 return ST->hasMacroFusion() || ST->hasBranchFusion();
7340}
7341
7342static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
7343 if (!ST->hasAVX())
7344 return false;
7345
7346 if (ScalarTy->isPointerTy())
7347 return true;
7348
7349 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7350 return true;
7351
7352 if ((ScalarTy->isHalfTy() || ScalarTy->isBFloatTy()) && ST->hasBWI())
7353 return true;
7354
7355 if (!ScalarTy->isIntegerTy())
7356 return false;
7357
7358 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7359 return IntWidth == 32 || IntWidth == 64 ||
7360 ((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
7361}
7362
7363bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
7364 unsigned AddressSpace,
7365 TTI::MaskKind MaskKind) const {
7366 Type *ScalarTy = DataTy->getScalarType();
7367
7368 // The backend can't handle a single element vector w/o CFCMOV.
7369 if (isa<VectorType>(Val: DataTy) &&
7370 cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7371 return ST->hasCF() &&
7372 hasConditionalLoadStoreForType(Ty: ScalarTy, /*IsStore=*/false);
7373
7374 return isLegalMaskedLoadStore(ScalarTy, ST);
7375}
7376
7377bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
7378 unsigned AddressSpace,
7379 TTI::MaskKind MaskKind) const {
7380 Type *ScalarTy = DataTy->getScalarType();
7381
7382 // The backend can't handle a single element vector w/o CFCMOV.
7383 if (isa<VectorType>(Val: DataTy) &&
7384 cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7385 return ST->hasCF() &&
7386 hasConditionalLoadStoreForType(Ty: ScalarTy, /*IsStore=*/true);
7387
7388 return isLegalMaskedLoadStore(ScalarTy, ST);
7389}
7390
7391bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) const {
7392 unsigned DataSize = DL.getTypeStoreSize(Ty: DataType);
7393 // The only supported nontemporal loads are for aligned vectors of 16 or 32
7394 // bytes. Note that 32-byte nontemporal vector loads are supported by AVX2
7395 // (the equivalent stores only require AVX).
7396 if (Alignment >= DataSize && (DataSize == 16 || DataSize == 32))
7397 return DataSize == 16 ? ST->hasSSE1() : ST->hasAVX2();
7398
7399 return false;
7400}
7401
7402bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) const {
7403 unsigned DataSize = DL.getTypeStoreSize(Ty: DataType);
7404
7405 // SSE4A supports nontemporal stores of float and double at arbitrary
7406 // alignment.
7407 if (ST->hasSSE4A() && (DataType->isFloatTy() || DataType->isDoubleTy()))
7408 return true;
7409
7410 // Besides the SSE4A subtarget exception above, only aligned stores are
7411 // available nontemporaly on any other subtarget. And only stores with a size
7412 // of 4..32 bytes (powers of 2, only) are permitted.
7413 if (Alignment < DataSize || DataSize < 4 || DataSize > 32 ||
7414 !isPowerOf2_32(Value: DataSize))
7415 return false;
7416
7417 // 32-byte vector nontemporal stores are supported by AVX (the equivalent
7418 // loads require AVX2).
7419 if (DataSize == 32)
7420 return ST->hasAVX();
7421 if (DataSize == 16)
7422 return ST->hasSSE1();
7423 return true;
7424}
7425
7426bool X86TTIImpl::isLegalBroadcastLoad(Type *ElementTy,
7427 ElementCount NumElements) const {
7428 // movddup
7429 return ST->hasSSE3() && !NumElements.isScalable() &&
7430 NumElements.getFixedValue() == 2 &&
7431 ElementTy == Type::getDoubleTy(C&: ElementTy->getContext());
7432}
7433
7434bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const {
7435 if (!isa<VectorType>(Val: DataTy))
7436 return false;
7437
7438 if (!ST->hasAVX512())
7439 return false;
7440
7441 // The backend can't handle a single element vector.
7442 if (cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7443 return false;
7444
7445 Type *ScalarTy = cast<VectorType>(Val: DataTy)->getElementType();
7446
7447 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7448 return true;
7449
7450 if (!ScalarTy->isIntegerTy())
7451 return false;
7452
7453 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7454 return IntWidth == 32 || IntWidth == 64 ||
7455 ((IntWidth == 8 || IntWidth == 16) && ST->hasVBMI2());
7456}
7457
7458bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy,
7459 Align Alignment) const {
7460 return isLegalMaskedExpandLoad(DataTy, Alignment);
7461}
7462
7463bool X86TTIImpl::supportsGather() const {
7464 // Some CPUs have better gather performance than others.
7465 // TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
7466 // enable gather with a -march.
7467 return ST->hasAVX512() || (ST->hasFastGather() && ST->hasAVX2());
7468}
7469
7470bool X86TTIImpl::forceScalarizeMaskedGather(VectorType *VTy,
7471 Align Alignment) const {
7472 // Gather / Scatter for vector 2 is not profitable on KNL / SKX
7473 // Vector-4 of gather/scatter instruction does not exist on KNL. We can extend
7474 // it to 8 elements, but zeroing upper bits of the mask vector will add more
7475 // instructions. Right now we give the scalar cost of vector-4 for KNL. TODO:
7476 // Check, maybe the gather/scatter instruction is better in the VariableMask
7477 // case.
7478 unsigned NumElts = cast<FixedVectorType>(Val: VTy)->getNumElements();
7479 return NumElts == 1 ||
7480 (ST->hasAVX512() && (NumElts == 2 || (NumElts == 4 && !ST->hasVLX())));
7481}
7482
7483bool X86TTIImpl::isLegalMaskedGatherScatter(Type *DataTy,
7484 Align Alignment) const {
7485 Type *ScalarTy = DataTy->getScalarType();
7486 if (ScalarTy->isPointerTy())
7487 return true;
7488
7489 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7490 return true;
7491
7492 if (!ScalarTy->isIntegerTy())
7493 return false;
7494
7495 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7496 return IntWidth == 32 || IntWidth == 64;
7497}
7498
7499bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) const {
7500 if (!supportsGather() || !ST->preferGather())
7501 return false;
7502 return isLegalMaskedGatherScatter(DataTy, Alignment);
7503}
7504
7505bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
7506 unsigned Opcode1,
7507 const SmallBitVector &OpcodeMask,
7508 ArrayRef<const Value *> Scalars) const {
7509 // ADDSUBPS 4xf32 SSE3
7510 // VADDSUBPS 4xf32 AVX
7511 // VADDSUBPS 8xf32 AVX2
7512 // ADDSUBPD 2xf64 SSE3
7513 // VADDSUBPD 2xf64 AVX
7514 // VADDSUBPD 4xf64 AVX2
7515
7516 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
7517 assert(OpcodeMask.size() == NumElements && "Mask and VecTy are incompatible");
7518 if (!isPowerOf2_32(Value: NumElements))
7519 return false;
7520 // Check the opcode pattern. We apply the mask on the opcode arguments and
7521 // then check if it is what we expect.
7522 auto IsLaneOrder = [&](unsigned EvenOpc, unsigned OddOpc) {
7523 return all_of(Range: seq<unsigned>(Begin: 0, End: NumElements), P: [&](unsigned Lane) {
7524 return (OpcodeMask.test(Idx: Lane) ? Opcode1 : Opcode0) ==
7525 (Lane % 2 == 0 ? EvenOpc : OddOpc);
7526 });
7527 };
7528 // We expect FSub for even lanes and FAdd for odd lanes.
7529 const bool IsAddSub = IsLaneOrder(Instruction::FSub, Instruction::FAdd);
7530 // Now check that the pattern is supported by the target ISA.
7531 Type *ElemTy = cast<VectorType>(Val: VecTy)->getElementType();
7532 if (IsAddSub && ST->hasSSE3() &&
7533 ((ElemTy->isFloatTy() && NumElements % 4 == 0) ||
7534 (ElemTy->isDoubleTy() && NumElements % 2 == 0)))
7535 return true;
7536 // The multiplication is fused into every lane: fmaddsub (the even lanes
7537 // subtract and the odd lanes add the product) or fmsubadd (the other way
7538 // round). The subtracted product is the first operand of the subtraction.
7539 using namespace PatternMatch;
7540 auto IsFusedWithFMul = [](const Value *V) {
7541 const auto *I = dyn_cast<Instruction>(Val: V);
7542 if (!I || !I->hasAllowContract())
7543 return false;
7544 auto IsFMul = [&](const Value *Op) {
7545 return match(V: Op,
7546 P: m_OneUse(SubPattern: m_AllowContract(SubPattern: m_FMul(L: m_Value(), R: m_Value())))) &&
7547 cast<Instruction>(Val: Op)->getParent() == I->getParent();
7548 };
7549 return IsFMul(I->getOperand(i: 0)) ||
7550 (I->getOpcode() == Instruction::FAdd && IsFMul(I->getOperand(i: 1)));
7551 };
7552 return ST->hasFMA() && !Scalars.empty() &&
7553 (ElemTy->isFloatTy() || ElemTy->isDoubleTy()) &&
7554 (IsAddSub || IsLaneOrder(Instruction::FAdd, Instruction::FSub)) &&
7555 all_of(Range&: Scalars, P: IsFusedWithFMul);
7556}
7557
7558bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
7559 // AVX2 doesn't support scatter
7560 if (!ST->hasAVX512() || !ST->preferScatter())
7561 return false;
7562 return isLegalMaskedGatherScatter(DataTy: DataType, Alignment);
7563}
7564
7565bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
7566 EVT VT = TLI->getValueType(DL, Ty: DataType);
7567 return TLI->isOperationLegal(Op: IsSigned ? ISD::SDIVREM : ISD::UDIVREM, VT);
7568}
7569
7570bool X86TTIImpl::isExpensiveToSpeculativelyExecute(const Instruction *I) const {
7571 // FDIV is always expensive, even if it has a very low uop count.
7572 // TODO: Still necessary for recent CPUs with low latency/throughput fdiv?
7573 if (I->getOpcode() == Instruction::FDiv)
7574 return true;
7575
7576 return BaseT::isExpensiveToSpeculativelyExecute(I);
7577}
7578
7579bool X86TTIImpl::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return false; }
7580
7581bool X86TTIImpl::areInlineCompatible(const Function *Caller,
7582 const Function *Callee) const {
7583 const TargetMachine &TM = getTLI()->getTargetMachine();
7584
7585 // Work this as a subsetting of subtarget features.
7586 const X86Subtarget &CallerSubtarget = TM.getSubtarget<X86Subtarget>(F: *Caller);
7587 const X86Subtarget &CalleeSubtarget = TM.getSubtarget<X86Subtarget>(F: *Callee);
7588 const FeatureBitset &CallerBits = CallerSubtarget.getFeatureBits();
7589 const FeatureBitset &CalleeBits = CalleeSubtarget.getFeatureBits();
7590
7591 // Check whether callee features are a subset of caller features
7592 // (apart from the ignore list).
7593 const FeatureBitset &InlineIgnoreFeatures =
7594 CallerSubtarget.getInlineIgnoreFeatures();
7595 FeatureBitset RealCallerBits = CallerBits & ~InlineIgnoreFeatures;
7596 FeatureBitset RealCalleeBits = CalleeBits & ~InlineIgnoreFeatures;
7597 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
7598 return false;
7599
7600 // If the features are not exactly the same (or there is a difference in
7601 // AVX512 register usage), we need to additionally check for calls
7602 // that may become ABI-incompatible as a result of inlining.
7603 if (RealCallerBits == RealCalleeBits &&
7604 CallerSubtarget.useAVX512Regs() == CalleeSubtarget.useAVX512Regs())
7605 return true;
7606
7607 for (const Instruction &I : instructions(F: Callee)) {
7608 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
7609 // Having more target features is fine for inline ASM and intrinsics.
7610 if (CB->isInlineAsm() || CB->getIntrinsicID() != Intrinsic::not_intrinsic)
7611 continue;
7612
7613 SmallVector<Type *, 8> Types;
7614 for (Value *Arg : CB->args())
7615 Types.push_back(Elt: Arg->getType());
7616 if (!CB->getType()->isVoidTy())
7617 Types.push_back(Elt: CB->getType());
7618
7619 // Simple types are always ABI compatible.
7620 auto IsSimpleTy = [](Type *Ty) {
7621 return !Ty->isVectorTy() && !Ty->isAggregateType();
7622 };
7623 if (all_of(Range&: Types, P: IsSimpleTy))
7624 continue;
7625
7626 // Do a precise compatibility check.
7627 if (!areTypesABICompatible(Caller, Callee, Type: Types))
7628 return false;
7629 }
7630 }
7631 return true;
7632}
7633
7634bool X86TTIImpl::areTypesABICompatible(const Function *Caller,
7635 const Function *Callee,
7636 ArrayRef<Type *> Types) const {
7637 const TargetMachine &TM = getTLI()->getTargetMachine();
7638 const TargetLowering *CallerTLI =
7639 TM.getSubtargetImpl(*Caller)->getTargetLowering();
7640 const TargetLowering *CalleeTLI =
7641 TM.getSubtargetImpl(*Callee)->getTargetLowering();
7642
7643 LLVMContext &Ctx = Caller->getContext();
7644 const DataLayout &DL = Caller->getDataLayout();
7645 CallingConv::ID CC = Callee->getCallingConv();
7646 return all_of(Range&: Types, P: [&](Type *Ty) {
7647 SmallVector<EVT> VTs;
7648 ComputeValueVTs(TLI: *CallerTLI, DL, Ty, ValueVTs&: VTs);
7649 return all_of(Range&: VTs, P: [&](EVT VT) {
7650 return CallerTLI->getRegisterTypeForCallingConv(Context&: Ctx, CC, VT) ==
7651 CalleeTLI->getRegisterTypeForCallingConv(Context&: Ctx, CC, VT);
7652 });
7653 });
7654}
7655
7656X86TTIImpl::TTI::MemCmpExpansionOptions
7657X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
7658 TTI::MemCmpExpansionOptions Options;
7659 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
7660 Options.NumLoadsPerBlock = IsZeroCmp ? 2 : 1;
7661 // All GPR and vector loads can be unaligned.
7662 Options.AllowOverlappingLoads = true;
7663 if (IsZeroCmp) {
7664 // Only enable vector loads for equality comparison. Right now the vector
7665 // version is not as fast for three way compare (see #33329).
7666 const unsigned PreferredWidth = ST->getPreferVectorWidth();
7667 if (PreferredWidth >= 512 && ST->hasAVX512())
7668 Options.LoadSizes.push_back(Elt: 64);
7669 if (PreferredWidth >= 256 && ST->hasAVX()) Options.LoadSizes.push_back(Elt: 32);
7670 if (PreferredWidth >= 128 && ST->hasSSE2()) Options.LoadSizes.push_back(Elt: 16);
7671 }
7672 if (ST->is64Bit()) {
7673 Options.LoadSizes.push_back(Elt: 8);
7674 }
7675 Options.LoadSizes.push_back(Elt: 4);
7676 Options.LoadSizes.push_back(Elt: 2);
7677 Options.LoadSizes.push_back(Elt: 1);
7678 return Options;
7679}
7680
7681bool X86TTIImpl::prefersVectorizedAddressing() const {
7682 return supportsGather();
7683}
7684
7685bool X86TTIImpl::supportsEfficientVectorElementLoadStore() const {
7686 return false;
7687}
7688
7689bool X86TTIImpl::enableInterleavedAccessVectorization() const {
7690 // TODO: We expect this to be beneficial regardless of arch,
7691 // but there are currently some unexplained performance artifacts on Atom.
7692 // As a temporary solution, disable on Atom.
7693 return !(ST->isAtom());
7694}
7695
7696bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
7697 switch (II->getIntrinsicID()) {
7698 default:
7699 return true;
7700 case Intrinsic::vector_reduce_and:
7701 case Intrinsic::vector_reduce_or:
7702 case Intrinsic::vector_reduce_xor:
7703 case Intrinsic::vector_reduce_mul:
7704 case Intrinsic::vector_reduce_smax:
7705 case Intrinsic::vector_reduce_smin:
7706 case Intrinsic::vector_reduce_umax:
7707 case Intrinsic::vector_reduce_umin:
7708 return false;
7709 }
7710}
7711
7712// Get estimation for interleaved load/store operations and strided load.
7713// \p Indices contains indices for strided load.
7714// \p Factor - the factor of interleaving.
7715// AVX-512 provides 3-src shuffles that significantly reduces the cost.
7716InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
7717 unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
7718 ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
7719 TTI::TargetCostKind CostKind, bool UseMaskForCond,
7720 bool UseMaskForGaps) const {
7721 // VecTy for interleave memop is <VF*Factor x Elt>.
7722 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7723 // VecTy = <12 x i32>.
7724
7725 // Calculate the number of memory operations (NumOfMemOps), required
7726 // for load/store the VecTy.
7727 MVT LegalVT = getTypeLegalizationCost(Ty: VecTy).second;
7728 unsigned VecTySize = DL.getTypeStoreSize(Ty: VecTy);
7729 unsigned LegalVTSize = LegalVT.getStoreSize();
7730 unsigned NumOfMemOps = (VecTySize + LegalVTSize - 1) / LegalVTSize;
7731
7732 // Get the cost of one memory operation.
7733 auto *SingleMemOpTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
7734 NumElts: LegalVT.getVectorNumElements());
7735 InstructionCost MemOpCost;
7736 bool UseMaskedMemOp = UseMaskForCond || UseMaskForGaps;
7737 if (UseMaskedMemOp) {
7738 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
7739 : Intrinsic::masked_store;
7740 MemOpCost = getMaskedMemoryOpCost(
7741 MICA: {IID, SingleMemOpTy, Alignment, AddressSpace}, CostKind);
7742 } else
7743 MemOpCost = getMemoryOpCost(Opcode, Src: SingleMemOpTy, Alignment, AddressSpace,
7744 CostKind);
7745
7746 unsigned VF = VecTy->getNumElements() / Factor;
7747 MVT VT =
7748 MVT::getVectorVT(VT: TLI->getSimpleValueType(DL, Ty: VecTy->getScalarType()), NumElements: VF);
7749
7750 InstructionCost MaskCost;
7751 if (UseMaskedMemOp) {
7752 APInt DemandedLoadStoreElts = APInt::getZero(numBits: VecTy->getNumElements());
7753 for (unsigned Index : Indices) {
7754 assert(Index < Factor && "Invalid index for interleaved memory op");
7755 for (unsigned Elm = 0; Elm < VF; Elm++)
7756 DemandedLoadStoreElts.setBit(Index + Elm * Factor);
7757 }
7758
7759 Type *I1Type = Type::getInt1Ty(C&: VecTy->getContext());
7760
7761 MaskCost = getReplicationShuffleCost(
7762 EltTy: I1Type, ReplicationFactor: Factor, VF,
7763 DemandedDstElts: UseMaskForGaps ? DemandedLoadStoreElts
7764 : APInt::getAllOnes(numBits: VecTy->getNumElements()),
7765 CostKind);
7766
7767 // The Gaps mask is invariant and created outside the loop, therefore the
7768 // cost of creating it is not accounted for here. However if we have both
7769 // a MaskForGaps and some other mask that guards the execution of the
7770 // memory access, we need to account for the cost of And-ing the two masks
7771 // inside the loop.
7772 if (UseMaskForGaps) {
7773 auto *MaskVT = FixedVectorType::get(ElementType: I1Type, NumElts: VecTy->getNumElements());
7774 MaskCost += getArithmeticInstrCost(Opcode: BinaryOperator::And, Ty: MaskVT, CostKind);
7775 }
7776 }
7777
7778 if (Opcode == Instruction::Load) {
7779 // The tables (AVX512InterleavedLoadTbl and AVX512InterleavedStoreTbl)
7780 // contain the cost of the optimized shuffle sequence that the
7781 // X86InterleavedAccess pass will generate.
7782 // The cost of loads and stores are computed separately from the table.
7783
7784 // X86InterleavedAccess support only the following interleaved-access group.
7785 static const CostTblEntry AVX512InterleavedLoadTbl[] = {
7786 {.ISD: 3, .Type: MVT::v16i8, .Cost: 12}, //(load 48i8 and) deinterleave into 3 x 16i8
7787 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, //(load 96i8 and) deinterleave into 3 x 32i8
7788 {.ISD: 3, .Type: MVT::v64i8, .Cost: 22}, //(load 96i8 and) deinterleave into 3 x 32i8
7789 };
7790
7791 if (const auto *Entry =
7792 CostTableLookup(Table: AVX512InterleavedLoadTbl, ISD: Factor, Ty: VT))
7793 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7794 //If an entry does not exist, fallback to the default implementation.
7795
7796 // Kind of shuffle depends on number of loaded values.
7797 // If we load the entire data in one register, we can use a 1-src shuffle.
7798 // Otherwise, we'll merge 2 sources in each operation.
7799 TTI::ShuffleKind ShuffleKind =
7800 (NumOfMemOps > 1) ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc;
7801
7802 InstructionCost ShuffleCost = getShuffleCost(
7803 Kind: ShuffleKind, DstTy: SingleMemOpTy, SrcTy: SingleMemOpTy, CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7804
7805 unsigned NumOfLoadsInInterleaveGrp =
7806 Indices.size() ? Indices.size() : Factor;
7807 auto *ResultTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
7808 NumElts: VecTy->getNumElements() / Factor);
7809 InstructionCost NumOfResults =
7810 getTypeLegalizationCost(Ty: ResultTy).first * NumOfLoadsInInterleaveGrp;
7811
7812 // About a half of the loads may be folded in shuffles when we have only
7813 // one result. If we have more than one result, or the loads are masked,
7814 // we do not fold loads at all.
7815 unsigned NumOfUnfoldedLoads =
7816 UseMaskedMemOp || NumOfResults > 1 ? NumOfMemOps : NumOfMemOps / 2;
7817
7818 // Get a number of shuffle operations per result.
7819 unsigned NumOfShufflesPerResult =
7820 std::max(a: (unsigned)1, b: (unsigned)(NumOfMemOps - 1));
7821
7822 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7823 // When we have more than one destination, we need additional instructions
7824 // to keep sources.
7825 InstructionCost NumOfMoves = 0;
7826 if (NumOfResults > 1 && ShuffleKind == TTI::SK_PermuteTwoSrc)
7827 NumOfMoves = NumOfResults * NumOfShufflesPerResult / 2;
7828
7829 InstructionCost Cost = NumOfResults * NumOfShufflesPerResult * ShuffleCost +
7830 MaskCost + NumOfUnfoldedLoads * MemOpCost +
7831 NumOfMoves;
7832
7833 return Cost;
7834 }
7835
7836 // Store.
7837 assert(Opcode == Instruction::Store &&
7838 "Expected Store Instruction at this point");
7839 // X86InterleavedAccess support only the following interleaved-access group.
7840 static const CostTblEntry AVX512InterleavedStoreTbl[] = {
7841 {.ISD: 3, .Type: MVT::v16i8, .Cost: 12}, // interleave 3 x 16i8 into 48i8 (and store)
7842 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, // interleave 3 x 32i8 into 96i8 (and store)
7843 {.ISD: 3, .Type: MVT::v64i8, .Cost: 26}, // interleave 3 x 64i8 into 96i8 (and store)
7844
7845 {.ISD: 4, .Type: MVT::v8i8, .Cost: 10}, // interleave 4 x 8i8 into 32i8 (and store)
7846 {.ISD: 4, .Type: MVT::v16i8, .Cost: 11}, // interleave 4 x 16i8 into 64i8 (and store)
7847 {.ISD: 4, .Type: MVT::v32i8, .Cost: 14}, // interleave 4 x 32i8 into 128i8 (and store)
7848 {.ISD: 4, .Type: MVT::v64i8, .Cost: 24} // interleave 4 x 32i8 into 256i8 (and store)
7849 };
7850
7851 if (const auto *Entry =
7852 CostTableLookup(Table: AVX512InterleavedStoreTbl, ISD: Factor, Ty: VT))
7853 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7854 //If an entry does not exist, fallback to the default implementation.
7855
7856 // There is no strided stores meanwhile. And store can't be folded in
7857 // shuffle.
7858 unsigned NumOfSources = Factor; // The number of values to be merged.
7859 InstructionCost ShuffleCost =
7860 getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleMemOpTy, SrcTy: SingleMemOpTy,
7861 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7862 unsigned NumOfShufflesPerStore = NumOfSources - 1;
7863
7864 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7865 // We need additional instructions to keep sources.
7866 unsigned NumOfMoves = NumOfMemOps * NumOfShufflesPerStore / 2;
7867 InstructionCost Cost =
7868 MaskCost +
7869 NumOfMemOps * (MemOpCost + NumOfShufflesPerStore * ShuffleCost) +
7870 NumOfMoves;
7871 return Cost;
7872}
7873
7874InstructionCost X86TTIImpl::getInterleavedMemoryOpCost(
7875 unsigned Opcode, Type *BaseTy, unsigned Factor, ArrayRef<unsigned> Indices,
7876 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
7877 bool UseMaskForCond, bool UseMaskForGaps) const {
7878 auto *VecTy = cast<FixedVectorType>(Val: BaseTy);
7879
7880 auto isSupportedOnAVX512 = [&](Type *VecTy) {
7881 Type *EltTy = cast<VectorType>(Val: VecTy)->getElementType();
7882 if (EltTy->isFloatTy() || EltTy->isDoubleTy() || EltTy->isIntegerTy(BitWidth: 64) ||
7883 EltTy->isIntegerTy(BitWidth: 32) || EltTy->isPointerTy())
7884 return true;
7885 if (EltTy->isIntegerTy(BitWidth: 16) || EltTy->isIntegerTy(BitWidth: 8) || EltTy->isHalfTy())
7886 return ST->hasBWI();
7887 if (EltTy->isBFloatTy())
7888 return ST->hasBF16();
7889 return false;
7890 };
7891 if (ST->hasAVX512() && isSupportedOnAVX512(VecTy))
7892 return getInterleavedMemoryOpCostAVX512(
7893 Opcode, VecTy, Factor, Indices, Alignment,
7894 AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps);
7895
7896 if (UseMaskForCond || UseMaskForGaps)
7897 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7898 Alignment, AddressSpace, CostKind,
7899 UseMaskForCond, UseMaskForGaps);
7900
7901 // Get estimation for interleaved load/store operations for SSE-AVX2.
7902 // As opposed to AVX-512, SSE-AVX2 do not have generic shuffles that allow
7903 // computing the cost using a generic formula as a function of generic
7904 // shuffles. We therefore use a lookup table instead, filled according to
7905 // the instruction sequences that codegen currently generates.
7906
7907 // VecTy for interleave memop is <VF*Factor x Elt>.
7908 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7909 // VecTy = <12 x i32>.
7910 MVT LegalVT = getTypeLegalizationCost(Ty: VecTy).second;
7911
7912 // This function can be called with VecTy=<6xi128>, Factor=3, in which case
7913 // the VF=2, while v2i128 is an unsupported MVT vector type
7914 // (see MachineValueType.h::getVectorVT()).
7915 if (!LegalVT.isVector())
7916 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7917 Alignment, AddressSpace, CostKind);
7918
7919 unsigned VF = VecTy->getNumElements() / Factor;
7920 Type *ScalarTy = VecTy->getElementType();
7921 // Deduplicate entries, model floats/pointers as appropriately-sized integers.
7922 if (!ScalarTy->isIntegerTy())
7923 ScalarTy =
7924 Type::getIntNTy(C&: ScalarTy->getContext(), N: DL.getTypeSizeInBits(Ty: ScalarTy));
7925
7926 // Get the cost of all the memory operations.
7927 // FIXME: discount dead loads.
7928 InstructionCost MemOpCosts =
7929 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
7930
7931 auto *VT = FixedVectorType::get(ElementType: ScalarTy, NumElts: VF);
7932 EVT ETy = TLI->getValueType(DL, Ty: VT);
7933 if (!ETy.isSimple())
7934 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7935 Alignment, AddressSpace, CostKind);
7936
7937 // TODO: Complete for other data-types and strides.
7938 // Each combination of Stride, element bit width and VF results in a different
7939 // sequence; The cost tables are therefore accessed with:
7940 // Factor (stride) and VectorType=VFxiN.
7941 // The Cost accounts only for the shuffle sequence;
7942 // The cost of the loads/stores is accounted for separately.
7943 //
7944 static const CostTblEntry AVX2InterleavedLoadTbl[] = {
7945 {.ISD: 2, .Type: MVT::v2i8, .Cost: 2}, // (load 4i8 and) deinterleave into 2 x 2i8
7946 {.ISD: 2, .Type: MVT::v4i8, .Cost: 2}, // (load 8i8 and) deinterleave into 2 x 4i8
7947 {.ISD: 2, .Type: MVT::v8i8, .Cost: 2}, // (load 16i8 and) deinterleave into 2 x 8i8
7948 {.ISD: 2, .Type: MVT::v16i8, .Cost: 4}, // (load 32i8 and) deinterleave into 2 x 16i8
7949 {.ISD: 2, .Type: MVT::v32i8, .Cost: 6}, // (load 64i8 and) deinterleave into 2 x 32i8
7950
7951 {.ISD: 2, .Type: MVT::v8i16, .Cost: 6}, // (load 16i16 and) deinterleave into 2 x 8i16
7952 {.ISD: 2, .Type: MVT::v16i16, .Cost: 9}, // (load 32i16 and) deinterleave into 2 x 16i16
7953 {.ISD: 2, .Type: MVT::v32i16, .Cost: 18}, // (load 64i16 and) deinterleave into 2 x 32i16
7954
7955 {.ISD: 2, .Type: MVT::v8i32, .Cost: 4}, // (load 16i32 and) deinterleave into 2 x 8i32
7956 {.ISD: 2, .Type: MVT::v16i32, .Cost: 8}, // (load 32i32 and) deinterleave into 2 x 16i32
7957 {.ISD: 2, .Type: MVT::v32i32, .Cost: 16}, // (load 64i32 and) deinterleave into 2 x 32i32
7958
7959 {.ISD: 2, .Type: MVT::v4i64, .Cost: 4}, // (load 8i64 and) deinterleave into 2 x 4i64
7960 {.ISD: 2, .Type: MVT::v8i64, .Cost: 8}, // (load 16i64 and) deinterleave into 2 x 8i64
7961 {.ISD: 2, .Type: MVT::v16i64, .Cost: 16}, // (load 32i64 and) deinterleave into 2 x 16i64
7962 {.ISD: 2, .Type: MVT::v32i64, .Cost: 32}, // (load 64i64 and) deinterleave into 2 x 32i64
7963
7964 {.ISD: 3, .Type: MVT::v2i8, .Cost: 3}, // (load 6i8 and) deinterleave into 3 x 2i8
7965 {.ISD: 3, .Type: MVT::v4i8, .Cost: 3}, // (load 12i8 and) deinterleave into 3 x 4i8
7966 {.ISD: 3, .Type: MVT::v8i8, .Cost: 6}, // (load 24i8 and) deinterleave into 3 x 8i8
7967 {.ISD: 3, .Type: MVT::v16i8, .Cost: 11}, // (load 48i8 and) deinterleave into 3 x 16i8
7968 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, // (load 96i8 and) deinterleave into 3 x 32i8
7969
7970 {.ISD: 3, .Type: MVT::v2i16, .Cost: 5}, // (load 6i16 and) deinterleave into 3 x 2i16
7971 {.ISD: 3, .Type: MVT::v4i16, .Cost: 7}, // (load 12i16 and) deinterleave into 3 x 4i16
7972 {.ISD: 3, .Type: MVT::v8i16, .Cost: 9}, // (load 24i16 and) deinterleave into 3 x 8i16
7973 {.ISD: 3, .Type: MVT::v16i16, .Cost: 28}, // (load 48i16 and) deinterleave into 3 x 16i16
7974 {.ISD: 3, .Type: MVT::v32i16, .Cost: 56}, // (load 96i16 and) deinterleave into 3 x 32i16
7975
7976 {.ISD: 3, .Type: MVT::v2i32, .Cost: 3}, // (load 6i32 and) deinterleave into 3 x 2i32
7977 {.ISD: 3, .Type: MVT::v4i32, .Cost: 3}, // (load 12i32 and) deinterleave into 3 x 4i32
7978 {.ISD: 3, .Type: MVT::v8i32, .Cost: 7}, // (load 24i32 and) deinterleave into 3 x 8i32
7979 {.ISD: 3, .Type: MVT::v16i32, .Cost: 14}, // (load 48i32 and) deinterleave into 3 x 16i32
7980 {.ISD: 3, .Type: MVT::v32i32, .Cost: 32}, // (load 96i32 and) deinterleave into 3 x 32i32
7981
7982 {.ISD: 3, .Type: MVT::v2i64, .Cost: 1}, // (load 6i64 and) deinterleave into 3 x 2i64
7983 {.ISD: 3, .Type: MVT::v4i64, .Cost: 5}, // (load 12i64 and) deinterleave into 3 x 4i64
7984 {.ISD: 3, .Type: MVT::v8i64, .Cost: 10}, // (load 24i64 and) deinterleave into 3 x 8i64
7985 {.ISD: 3, .Type: MVT::v16i64, .Cost: 20}, // (load 48i64 and) deinterleave into 3 x 16i64
7986
7987 {.ISD: 4, .Type: MVT::v2i8, .Cost: 4}, // (load 8i8 and) deinterleave into 4 x 2i8
7988 {.ISD: 4, .Type: MVT::v4i8, .Cost: 4}, // (load 16i8 and) deinterleave into 4 x 4i8
7989 {.ISD: 4, .Type: MVT::v8i8, .Cost: 12}, // (load 32i8 and) deinterleave into 4 x 8i8
7990 {.ISD: 4, .Type: MVT::v16i8, .Cost: 24}, // (load 64i8 and) deinterleave into 4 x 16i8
7991 {.ISD: 4, .Type: MVT::v32i8, .Cost: 56}, // (load 128i8 and) deinterleave into 4 x 32i8
7992
7993 {.ISD: 4, .Type: MVT::v2i16, .Cost: 6}, // (load 8i16 and) deinterleave into 4 x 2i16
7994 {.ISD: 4, .Type: MVT::v4i16, .Cost: 17}, // (load 16i16 and) deinterleave into 4 x 4i16
7995 {.ISD: 4, .Type: MVT::v8i16, .Cost: 33}, // (load 32i16 and) deinterleave into 4 x 8i16
7996 {.ISD: 4, .Type: MVT::v16i16, .Cost: 75}, // (load 64i16 and) deinterleave into 4 x 16i16
7997 {.ISD: 4, .Type: MVT::v32i16, .Cost: 150}, // (load 128i16 and) deinterleave into 4 x 32i16
7998
7999 {.ISD: 4, .Type: MVT::v2i32, .Cost: 4}, // (load 8i32 and) deinterleave into 4 x 2i32
8000 {.ISD: 4, .Type: MVT::v4i32, .Cost: 8}, // (load 16i32 and) deinterleave into 4 x 4i32
8001 {.ISD: 4, .Type: MVT::v8i32, .Cost: 16}, // (load 32i32 and) deinterleave into 4 x 8i32
8002 {.ISD: 4, .Type: MVT::v16i32, .Cost: 32}, // (load 64i32 and) deinterleave into 4 x 16i32
8003 {.ISD: 4, .Type: MVT::v32i32, .Cost: 68}, // (load 128i32 and) deinterleave into 4 x 32i32
8004
8005 {.ISD: 4, .Type: MVT::v2i64, .Cost: 6}, // (load 8i64 and) deinterleave into 4 x 2i64
8006 {.ISD: 4, .Type: MVT::v4i64, .Cost: 8}, // (load 16i64 and) deinterleave into 4 x 4i64
8007 {.ISD: 4, .Type: MVT::v8i64, .Cost: 20}, // (load 32i64 and) deinterleave into 4 x 8i64
8008 {.ISD: 4, .Type: MVT::v16i64, .Cost: 40}, // (load 64i64 and) deinterleave into 4 x 16i64
8009
8010 {.ISD: 6, .Type: MVT::v2i8, .Cost: 6}, // (load 12i8 and) deinterleave into 6 x 2i8
8011 {.ISD: 6, .Type: MVT::v4i8, .Cost: 14}, // (load 24i8 and) deinterleave into 6 x 4i8
8012 {.ISD: 6, .Type: MVT::v8i8, .Cost: 18}, // (load 48i8 and) deinterleave into 6 x 8i8
8013 {.ISD: 6, .Type: MVT::v16i8, .Cost: 43}, // (load 96i8 and) deinterleave into 6 x 16i8
8014 {.ISD: 6, .Type: MVT::v32i8, .Cost: 82}, // (load 192i8 and) deinterleave into 6 x 32i8
8015
8016 {.ISD: 6, .Type: MVT::v2i16, .Cost: 13}, // (load 12i16 and) deinterleave into 6 x 2i16
8017 {.ISD: 6, .Type: MVT::v4i16, .Cost: 9}, // (load 24i16 and) deinterleave into 6 x 4i16
8018 {.ISD: 6, .Type: MVT::v8i16, .Cost: 39}, // (load 48i16 and) deinterleave into 6 x 8i16
8019 {.ISD: 6, .Type: MVT::v16i16, .Cost: 106}, // (load 96i16 and) deinterleave into 6 x 16i16
8020 {.ISD: 6, .Type: MVT::v32i16, .Cost: 212}, // (load 192i16 and) deinterleave into 6 x 32i16
8021
8022 {.ISD: 6, .Type: MVT::v2i32, .Cost: 6}, // (load 12i32 and) deinterleave into 6 x 2i32
8023 {.ISD: 6, .Type: MVT::v4i32, .Cost: 15}, // (load 24i32 and) deinterleave into 6 x 4i32
8024 {.ISD: 6, .Type: MVT::v8i32, .Cost: 31}, // (load 48i32 and) deinterleave into 6 x 8i32
8025 {.ISD: 6, .Type: MVT::v16i32, .Cost: 64}, // (load 96i32 and) deinterleave into 6 x 16i32
8026
8027 {.ISD: 6, .Type: MVT::v2i64, .Cost: 6}, // (load 12i64 and) deinterleave into 6 x 2i64
8028 {.ISD: 6, .Type: MVT::v4i64, .Cost: 18}, // (load 24i64 and) deinterleave into 6 x 4i64
8029 {.ISD: 6, .Type: MVT::v8i64, .Cost: 36}, // (load 48i64 and) deinterleave into 6 x 8i64
8030
8031 {.ISD: 8, .Type: MVT::v8i32, .Cost: 40} // (load 64i32 and) deinterleave into 8 x 8i32
8032 };
8033
8034 static const CostTblEntry SSSE3InterleavedLoadTbl[] = {
8035 {.ISD: 2, .Type: MVT::v4i16, .Cost: 2}, // (load 8i16 and) deinterleave into 2 x 4i16
8036 };
8037
8038 static const CostTblEntry SSE2InterleavedLoadTbl[] = {
8039 {.ISD: 2, .Type: MVT::v2i16, .Cost: 2}, // (load 4i16 and) deinterleave into 2 x 2i16
8040 {.ISD: 2, .Type: MVT::v4i16, .Cost: 7}, // (load 8i16 and) deinterleave into 2 x 4i16
8041
8042 {.ISD: 2, .Type: MVT::v2i32, .Cost: 2}, // (load 4i32 and) deinterleave into 2 x 2i32
8043 {.ISD: 2, .Type: MVT::v4i32, .Cost: 2}, // (load 8i32 and) deinterleave into 2 x 4i32
8044
8045 {.ISD: 2, .Type: MVT::v2i64, .Cost: 2}, // (load 4i64 and) deinterleave into 2 x 2i64
8046 };
8047
8048 static const CostTblEntry AVX2InterleavedStoreTbl[] = {
8049 {.ISD: 2, .Type: MVT::v16i8, .Cost: 3}, // interleave 2 x 16i8 into 32i8 (and store)
8050 {.ISD: 2, .Type: MVT::v32i8, .Cost: 4}, // interleave 2 x 32i8 into 64i8 (and store)
8051
8052 {.ISD: 2, .Type: MVT::v8i16, .Cost: 3}, // interleave 2 x 8i16 into 16i16 (and store)
8053 {.ISD: 2, .Type: MVT::v16i16, .Cost: 4}, // interleave 2 x 16i16 into 32i16 (and store)
8054 {.ISD: 2, .Type: MVT::v32i16, .Cost: 8}, // interleave 2 x 32i16 into 64i16 (and store)
8055
8056 {.ISD: 2, .Type: MVT::v4i32, .Cost: 2}, // interleave 2 x 4i32 into 8i32 (and store)
8057 {.ISD: 2, .Type: MVT::v8i32, .Cost: 4}, // interleave 2 x 8i32 into 16i32 (and store)
8058 {.ISD: 2, .Type: MVT::v16i32, .Cost: 8}, // interleave 2 x 16i32 into 32i32 (and store)
8059 {.ISD: 2, .Type: MVT::v32i32, .Cost: 16}, // interleave 2 x 32i32 into 64i32 (and store)
8060
8061 {.ISD: 2, .Type: MVT::v2i64, .Cost: 2}, // interleave 2 x 2i64 into 4i64 (and store)
8062 {.ISD: 2, .Type: MVT::v4i64, .Cost: 4}, // interleave 2 x 4i64 into 8i64 (and store)
8063 {.ISD: 2, .Type: MVT::v8i64, .Cost: 8}, // interleave 2 x 8i64 into 16i64 (and store)
8064 {.ISD: 2, .Type: MVT::v16i64, .Cost: 16}, // interleave 2 x 16i64 into 32i64 (and store)
8065 {.ISD: 2, .Type: MVT::v32i64, .Cost: 32}, // interleave 2 x 32i64 into 64i64 (and store)
8066
8067 {.ISD: 3, .Type: MVT::v2i8, .Cost: 4}, // interleave 3 x 2i8 into 6i8 (and store)
8068 {.ISD: 3, .Type: MVT::v4i8, .Cost: 4}, // interleave 3 x 4i8 into 12i8 (and store)
8069 {.ISD: 3, .Type: MVT::v8i8, .Cost: 6}, // interleave 3 x 8i8 into 24i8 (and store)
8070 {.ISD: 3, .Type: MVT::v16i8, .Cost: 11}, // interleave 3 x 16i8 into 48i8 (and store)
8071 {.ISD: 3, .Type: MVT::v32i8, .Cost: 13}, // interleave 3 x 32i8 into 96i8 (and store)
8072
8073 {.ISD: 3, .Type: MVT::v2i16, .Cost: 4}, // interleave 3 x 2i16 into 6i16 (and store)
8074 {.ISD: 3, .Type: MVT::v4i16, .Cost: 6}, // interleave 3 x 4i16 into 12i16 (and store)
8075 {.ISD: 3, .Type: MVT::v8i16, .Cost: 12}, // interleave 3 x 8i16 into 24i16 (and store)
8076 {.ISD: 3, .Type: MVT::v16i16, .Cost: 27}, // interleave 3 x 16i16 into 48i16 (and store)
8077 {.ISD: 3, .Type: MVT::v32i16, .Cost: 54}, // interleave 3 x 32i16 into 96i16 (and store)
8078
8079 {.ISD: 3, .Type: MVT::v2i32, .Cost: 4}, // interleave 3 x 2i32 into 6i32 (and store)
8080 {.ISD: 3, .Type: MVT::v4i32, .Cost: 5}, // interleave 3 x 4i32 into 12i32 (and store)
8081 {.ISD: 3, .Type: MVT::v8i32, .Cost: 11}, // interleave 3 x 8i32 into 24i32 (and store)
8082 {.ISD: 3, .Type: MVT::v16i32, .Cost: 22}, // interleave 3 x 16i32 into 48i32 (and store)
8083 {.ISD: 3, .Type: MVT::v32i32, .Cost: 48}, // interleave 3 x 32i32 into 96i32 (and store)
8084
8085 {.ISD: 3, .Type: MVT::v2i64, .Cost: 4}, // interleave 3 x 2i64 into 6i64 (and store)
8086 {.ISD: 3, .Type: MVT::v4i64, .Cost: 6}, // interleave 3 x 4i64 into 12i64 (and store)
8087 {.ISD: 3, .Type: MVT::v8i64, .Cost: 12}, // interleave 3 x 8i64 into 24i64 (and store)
8088 {.ISD: 3, .Type: MVT::v16i64, .Cost: 24}, // interleave 3 x 16i64 into 48i64 (and store)
8089
8090 {.ISD: 4, .Type: MVT::v2i8, .Cost: 4}, // interleave 4 x 2i8 into 8i8 (and store)
8091 {.ISD: 4, .Type: MVT::v4i8, .Cost: 4}, // interleave 4 x 4i8 into 16i8 (and store)
8092 {.ISD: 4, .Type: MVT::v8i8, .Cost: 4}, // interleave 4 x 8i8 into 32i8 (and store)
8093 {.ISD: 4, .Type: MVT::v16i8, .Cost: 8}, // interleave 4 x 16i8 into 64i8 (and store)
8094 {.ISD: 4, .Type: MVT::v32i8, .Cost: 12}, // interleave 4 x 32i8 into 128i8 (and store)
8095
8096 {.ISD: 4, .Type: MVT::v2i16, .Cost: 2}, // interleave 4 x 2i16 into 8i16 (and store)
8097 {.ISD: 4, .Type: MVT::v4i16, .Cost: 6}, // interleave 4 x 4i16 into 16i16 (and store)
8098 {.ISD: 4, .Type: MVT::v8i16, .Cost: 10}, // interleave 4 x 8i16 into 32i16 (and store)
8099 {.ISD: 4, .Type: MVT::v16i16, .Cost: 32}, // interleave 4 x 16i16 into 64i16 (and store)
8100 {.ISD: 4, .Type: MVT::v32i16, .Cost: 64}, // interleave 4 x 32i16 into 128i16 (and store)
8101
8102 {.ISD: 4, .Type: MVT::v2i32, .Cost: 5}, // interleave 4 x 2i32 into 8i32 (and store)
8103 {.ISD: 4, .Type: MVT::v4i32, .Cost: 6}, // interleave 4 x 4i32 into 16i32 (and store)
8104 {.ISD: 4, .Type: MVT::v8i32, .Cost: 16}, // interleave 4 x 8i32 into 32i32 (and store)
8105 {.ISD: 4, .Type: MVT::v16i32, .Cost: 32}, // interleave 4 x 16i32 into 64i32 (and store)
8106 {.ISD: 4, .Type: MVT::v32i32, .Cost: 64}, // interleave 4 x 32i32 into 128i32 (and store)
8107
8108 {.ISD: 4, .Type: MVT::v2i64, .Cost: 6}, // interleave 4 x 2i64 into 8i64 (and store)
8109 {.ISD: 4, .Type: MVT::v4i64, .Cost: 8}, // interleave 4 x 4i64 into 16i64 (and store)
8110 {.ISD: 4, .Type: MVT::v8i64, .Cost: 20}, // interleave 4 x 8i64 into 32i64 (and store)
8111 {.ISD: 4, .Type: MVT::v16i64, .Cost: 40}, // interleave 4 x 16i64 into 64i64 (and store)
8112
8113 {.ISD: 6, .Type: MVT::v2i8, .Cost: 7}, // interleave 6 x 2i8 into 12i8 (and store)
8114 {.ISD: 6, .Type: MVT::v4i8, .Cost: 9}, // interleave 6 x 4i8 into 24i8 (and store)
8115 {.ISD: 6, .Type: MVT::v8i8, .Cost: 16}, // interleave 6 x 8i8 into 48i8 (and store)
8116 {.ISD: 6, .Type: MVT::v16i8, .Cost: 27}, // interleave 6 x 16i8 into 96i8 (and store)
8117 {.ISD: 6, .Type: MVT::v32i8, .Cost: 90}, // interleave 6 x 32i8 into 192i8 (and store)
8118
8119 {.ISD: 6, .Type: MVT::v2i16, .Cost: 10}, // interleave 6 x 2i16 into 12i16 (and store)
8120 {.ISD: 6, .Type: MVT::v4i16, .Cost: 15}, // interleave 6 x 4i16 into 24i16 (and store)
8121 {.ISD: 6, .Type: MVT::v8i16, .Cost: 21}, // interleave 6 x 8i16 into 48i16 (and store)
8122 {.ISD: 6, .Type: MVT::v16i16, .Cost: 58}, // interleave 6 x 16i16 into 96i16 (and store)
8123 {.ISD: 6, .Type: MVT::v32i16, .Cost: 90}, // interleave 6 x 32i16 into 192i16 (and store)
8124
8125 {.ISD: 6, .Type: MVT::v2i32, .Cost: 9}, // interleave 6 x 2i32 into 12i32 (and store)
8126 {.ISD: 6, .Type: MVT::v4i32, .Cost: 12}, // interleave 6 x 4i32 into 24i32 (and store)
8127 {.ISD: 6, .Type: MVT::v8i32, .Cost: 33}, // interleave 6 x 8i32 into 48i32 (and store)
8128 {.ISD: 6, .Type: MVT::v16i32, .Cost: 66}, // interleave 6 x 16i32 into 96i32 (and store)
8129
8130 {.ISD: 6, .Type: MVT::v2i64, .Cost: 8}, // interleave 6 x 2i64 into 12i64 (and store)
8131 {.ISD: 6, .Type: MVT::v4i64, .Cost: 15}, // interleave 6 x 4i64 into 24i64 (and store)
8132 {.ISD: 6, .Type: MVT::v8i64, .Cost: 30}, // interleave 6 x 8i64 into 48i64 (and store)
8133 };
8134
8135 static const CostTblEntry SSE2InterleavedStoreTbl[] = {
8136 {.ISD: 2, .Type: MVT::v2i8, .Cost: 1}, // interleave 2 x 2i8 into 4i8 (and store)
8137 {.ISD: 2, .Type: MVT::v4i8, .Cost: 1}, // interleave 2 x 4i8 into 8i8 (and store)
8138 {.ISD: 2, .Type: MVT::v8i8, .Cost: 1}, // interleave 2 x 8i8 into 16i8 (and store)
8139
8140 {.ISD: 2, .Type: MVT::v2i16, .Cost: 1}, // interleave 2 x 2i16 into 4i16 (and store)
8141 {.ISD: 2, .Type: MVT::v4i16, .Cost: 1}, // interleave 2 x 4i16 into 8i16 (and store)
8142
8143 {.ISD: 2, .Type: MVT::v2i32, .Cost: 1}, // interleave 2 x 2i32 into 4i32 (and store)
8144 };
8145
8146 if (Opcode == Instruction::Load) {
8147 auto GetDiscountedCost = [Factor, NumMembers = Indices.size(),
8148 MemOpCosts](const CostTblEntry *Entry) {
8149 // NOTE: this is just an approximation!
8150 // It can over/under -estimate the cost!
8151 return MemOpCosts + divideCeil(Numerator: NumMembers * Entry->Cost, Denominator: Factor);
8152 };
8153
8154 if (ST->hasAVX2())
8155 if (const auto *Entry = CostTableLookup(Table: AVX2InterleavedLoadTbl, ISD: Factor,
8156 Ty: ETy.getSimpleVT()))
8157 return GetDiscountedCost(Entry);
8158
8159 if (ST->hasSSSE3())
8160 if (const auto *Entry = CostTableLookup(Table: SSSE3InterleavedLoadTbl, ISD: Factor,
8161 Ty: ETy.getSimpleVT()))
8162 return GetDiscountedCost(Entry);
8163
8164 if (ST->hasSSE2())
8165 if (const auto *Entry = CostTableLookup(Table: SSE2InterleavedLoadTbl, ISD: Factor,
8166 Ty: ETy.getSimpleVT()))
8167 return GetDiscountedCost(Entry);
8168 } else {
8169 assert(Opcode == Instruction::Store &&
8170 "Expected Store Instruction at this point");
8171 assert((!Indices.size() || Indices.size() == Factor) &&
8172 "Interleaved store only supports fully-interleaved groups.");
8173 if (ST->hasAVX2())
8174 if (const auto *Entry = CostTableLookup(Table: AVX2InterleavedStoreTbl, ISD: Factor,
8175 Ty: ETy.getSimpleVT()))
8176 return MemOpCosts + Entry->Cost;
8177
8178 if (ST->hasSSE2())
8179 if (const auto *Entry = CostTableLookup(Table: SSE2InterleavedStoreTbl, ISD: Factor,
8180 Ty: ETy.getSimpleVT()))
8181 return MemOpCosts + Entry->Cost;
8182 }
8183
8184 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
8185 Alignment, AddressSpace, CostKind,
8186 UseMaskForCond, UseMaskForGaps);
8187}
8188
8189InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
8190 StackOffset BaseOffset,
8191 bool HasBaseReg, int64_t Scale,
8192 unsigned AddrSpace) const {
8193 // Scaling factors are not free at all.
8194 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
8195 // will take 2 allocations in the out of order engine instead of 1
8196 // for plain addressing mode, i.e. inst (reg1).
8197 // E.g.,
8198 // vaddps (%rsi,%rdx), %ymm0, %ymm1
8199 // Requires two allocations (one for the load, one for the computation)
8200 // whereas:
8201 // vaddps (%rsi), %ymm0, %ymm1
8202 // Requires just 1 allocation, i.e., freeing allocations for other operations
8203 // and having less micro operations to execute.
8204 //
8205 // For some X86 architectures, this is even worse because for instance for
8206 // stores, the complex addressing mode forces the instruction to use the
8207 // "load" ports instead of the dedicated "store" port.
8208 // E.g., on Haswell:
8209 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
8210 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
8211 TargetLoweringBase::AddrMode AM;
8212 AM.BaseGV = BaseGV;
8213 AM.BaseOffs = BaseOffset.getFixed();
8214 AM.HasBaseReg = HasBaseReg;
8215 AM.Scale = Scale;
8216 AM.ScalableOffset = BaseOffset.getScalable();
8217 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace))
8218 // Scale represents reg2 * scale, thus account for 1
8219 // as soon as we use a second register.
8220 return AM.Scale != 0;
8221 return InstructionCost::getInvalid();
8222}
8223
8224InstructionCost X86TTIImpl::getBranchMispredictPenalty() const {
8225 // TODO: Hook MispredictPenalty of SchedMachineModel into this.
8226 return 14;
8227}
8228
8229bool X86TTIImpl::isVectorShiftByScalarCheap(Type *Ty) const {
8230 unsigned Bits = Ty->getScalarSizeInBits();
8231
8232 // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
8233 // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
8234 if (ST->hasXOP() && (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
8235 return false;
8236
8237 // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
8238 // shifts just as cheap as scalar ones.
8239 if (ST->hasAVX2() && (Bits == 32 || Bits == 64))
8240 return false;
8241
8242 // AVX512BW has shifts such as vpsllvw.
8243 if (ST->hasBWI() && Bits == 16)
8244 return false;
8245
8246 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
8247 // fully general vector.
8248 return true;
8249}
8250
8251unsigned X86TTIImpl::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
8252 Type *ScalarValTy, Align Alignment,
8253 unsigned AddrSpace) const {
8254 if (ST->hasF16C() && ScalarMemTy->isHalfTy()) {
8255 return 4;
8256 }
8257 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy, Alignment,
8258 AddrSpace);
8259}
8260
8261bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
8262 SmallVectorImpl<Use *> &Ops) const {
8263 using namespace llvm::PatternMatch;
8264
8265 if (I->getOpcode() == Instruction::And &&
8266 (ST->hasBMI() || (I->getType()->isVectorTy() && ST->hasSSE2()))) {
8267 for (auto &Op : I->operands()) {
8268 // (and X, (not Y)) -> (andn X, Y)
8269 if (match(V: Op.get(), P: m_Not(V: m_Value())) && !I->getType()->isIntegerTy(BitWidth: 8)) {
8270 Ops.push_back(Elt: &Op);
8271 return true;
8272 }
8273 // (and X, (splat (not Y))) -> (andn X, (splat Y))
8274 if (match(V: Op.get(),
8275 P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()), Idx: m_ZeroInt()),
8276 v2: m_Value(), mask: m_ZeroMask()))) {
8277 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
8278 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
8279 Ops.push_back(Elt: &Not);
8280 Ops.push_back(Elt: &InsertElt);
8281 Ops.push_back(Elt: &Op);
8282 return true;
8283 }
8284 }
8285 }
8286
8287 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: I->getType());
8288 if (!VTy)
8289 return false;
8290
8291 if (I->getOpcode() == Instruction::Mul &&
8292 VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
8293 for (auto &Op : I->operands()) {
8294 // Make sure we are not already sinking this operand
8295 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
8296 continue;
8297
8298 // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
8299 // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
8300 if (ST->hasSSE41() &&
8301 match(V: Op.get(), P: m_AShr(L: m_Shl(L: m_Value(), R: m_SpecificInt(V: 32)),
8302 R: m_SpecificInt(V: 32)))) {
8303 Ops.push_back(Elt: &cast<Instruction>(Val&: Op)->getOperandUse(i: 0));
8304 Ops.push_back(Elt: &Op);
8305 } else if (ST->hasSSE2() &&
8306 match(V: Op.get(),
8307 P: m_And(L: m_Value(), R: m_SpecificInt(UINT64_C(0xffffffff))))) {
8308 Ops.push_back(Elt: &Op);
8309 }
8310 }
8311
8312 return !Ops.empty();
8313 }
8314
8315 // A uniform shift amount in a vector shift or funnel shift may be much
8316 // cheaper than a generic variable vector shift, so make that pattern visible
8317 // to SDAG by sinking the shuffle instruction next to the shift.
8318 int ShiftAmountOpNum = -1;
8319 if (I->isShift())
8320 ShiftAmountOpNum = 1;
8321 else if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
8322 if (II->getIntrinsicID() == Intrinsic::fshl ||
8323 II->getIntrinsicID() == Intrinsic::fshr)
8324 ShiftAmountOpNum = 2;
8325 }
8326
8327 if (ShiftAmountOpNum == -1)
8328 return false;
8329
8330 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I->getOperand(i: ShiftAmountOpNum));
8331 if (Shuf && getSplatIndex(Mask: Shuf->getShuffleMask()) >= 0 &&
8332 isVectorShiftByScalarCheap(Ty: I->getType())) {
8333 Ops.push_back(Elt: &I->getOperandUse(i: ShiftAmountOpNum));
8334 return true;
8335 }
8336
8337 return false;
8338}
8339
8340bool X86TTIImpl::useFastCCForInternalCall(Function &F) const {
8341 bool HasEGPR = ST->hasEGPR();
8342 const TargetMachine &TM = getTLI()->getTargetMachine();
8343
8344 for (User *U : F.users()) {
8345 CallBase *CB = dyn_cast<CallBase>(Val: U);
8346 if (!CB || CB->getCalledOperand() != &F)
8347 continue;
8348 Function *CallerFunc = CB->getFunction();
8349 if (TM.getSubtarget<X86Subtarget>(F: *CallerFunc).hasEGPR() != HasEGPR)
8350 return false;
8351 }
8352
8353 return true;
8354}
8355