1//===-- X86TargetTransformInfo.cpp - X86 specific TTI pass ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements a TargetTransformInfo analysis pass specific to the
10/// X86 target machine. It uses the target's detailed information to provide
11/// more precise answers to certain TTI queries, while letting the target
12/// independent and default TTI implementations handle the rest.
13///
14//===----------------------------------------------------------------------===//
15/// About Cost Model numbers used below it's necessary to say the following:
16/// the numbers correspond to some "generic" X86 CPU instead of usage of a
17/// specific CPU model. Usually the numbers correspond to the CPU where the
18/// feature first appeared. For example, if we do Subtarget.hasSSE42() in
19/// the lookups below the cost is based on Nehalem as that was the first CPU
20/// to support that feature level and thus has most likely the worst case cost,
21/// although we may discard an outlying worst cost from one CPU (e.g. Atom).
22///
23/// Some examples of other technologies/CPUs:
24/// SSE 3 - Pentium4 / Athlon64
25/// SSE 4.1 - Penryn
26/// SSE 4.2 - Nehalem / Silvermont
27/// AVX - Sandy Bridge / Jaguar / Bulldozer
28/// AVX2 - Haswell / Ryzen
29/// AVX-512 - Xeon Phi / Skylake
30///
31/// And some examples of instruction target dependent costs (latency)
32/// divss sqrtss rsqrtss
33/// AMD K7 11-16 19 3
34/// Piledriver 9-24 13-15 5
35/// Jaguar 14 16 2
36/// Pentium II,III 18 30 2
37/// Nehalem 7-14 7-18 3
38/// Haswell 10-13 11 5
39///
40/// Interpreting the 4 TargetCostKind types:
41/// TCK_RecipThroughput and TCK_Latency should try to match the worst case
42/// values reported by the CPU scheduler models (and llvm-mca).
43/// TCK_CodeSize should match the instruction count (e.g. divss = 1), NOT the
44/// actual encoding size of the instruction.
45/// TCK_SizeAndLatency should match the worst case micro-op counts reported by
46/// by the CPU scheduler models (and llvm-mca), to ensure that they are
47/// compatible with the MicroOpBufferSize and LoopMicroOpBufferSize values which are
48/// often used as the cost thresholds where TCK_SizeAndLatency is requested.
49//===----------------------------------------------------------------------===//
50
51#include "X86TargetTransformInfo.h"
52#include "llvm/ADT/SmallBitVector.h"
53#include "llvm/Analysis/TargetTransformInfo.h"
54#include "llvm/CodeGen/Analysis.h"
55#include "llvm/CodeGen/BasicTTIImpl.h"
56#include "llvm/CodeGen/CostTable.h"
57#include "llvm/CodeGen/TargetLowering.h"
58#include "llvm/IR/InstIterator.h"
59#include "llvm/IR/IntrinsicInst.h"
60#include <optional>
61
62using namespace llvm;
63
64#define DEBUG_TYPE "x86tti"
65
66//===----------------------------------------------------------------------===//
67//
68// X86 cost model.
69//
70//===----------------------------------------------------------------------===//
71
72using CostKindTblEntry = CostTblEntryT<CostKindCosts>;
73using TypeConversionCostKindTblEntry = TypeConversionCostTblEntryT<CostKindCosts>;
74
75TargetTransformInfo::PopcntSupportKind
76X86TTIImpl::getPopcntSupport(unsigned TyWidth) const {
77 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
78 // TODO: Currently the __builtin_popcount() implementation using SSE3
79 // instructions is inefficient. Once the problem is fixed, we should
80 // call ST->hasSSE3() instead of ST->hasPOPCNT().
81 return ST->hasPOPCNT() ? TTI::PSK_FastHardware : TTI::PSK_Software;
82}
83
84std::optional<unsigned> X86TTIImpl::getCacheSize(
85 TargetTransformInfo::CacheLevel Level) const {
86 switch (Level) {
87 case TargetTransformInfo::CacheLevel::L1D:
88 // - Penryn
89 // - Nehalem
90 // - Westmere
91 // - Sandy Bridge
92 // - Ivy Bridge
93 // - Haswell
94 // - Broadwell
95 // - Skylake
96 // - Kabylake
97 return 32 * 1024; // 32 KiB
98 case TargetTransformInfo::CacheLevel::L2D:
99 // - Penryn
100 // - Nehalem
101 // - Westmere
102 // - Sandy Bridge
103 // - Ivy Bridge
104 // - Haswell
105 // - Broadwell
106 // - Skylake
107 // - Kabylake
108 return 256 * 1024; // 256 KiB
109 }
110
111 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
112}
113
114std::optional<unsigned> X86TTIImpl::getCacheAssociativity(
115 TargetTransformInfo::CacheLevel Level) const {
116 // - Penryn
117 // - Nehalem
118 // - Westmere
119 // - Sandy Bridge
120 // - Ivy Bridge
121 // - Haswell
122 // - Broadwell
123 // - Skylake
124 // - Kabylake
125 switch (Level) {
126 case TargetTransformInfo::CacheLevel::L1D:
127 [[fallthrough]];
128 case TargetTransformInfo::CacheLevel::L2D:
129 return 8;
130 }
131
132 llvm_unreachable("Unknown TargetTransformInfo::CacheLevel");
133}
134
135enum ClassIDEnum { GPRClass = 0, VectorClass = 1, ScalarFPClass = 2 };
136
137unsigned X86TTIImpl::getRegisterClassForType(bool Vector, Type *Ty) const {
138 return Vector ? VectorClass
139 : Ty && Ty->isFloatingPointTy() ? ScalarFPClass
140 : GPRClass;
141}
142
143unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const {
144 if (ClassID == VectorClass && !ST->hasSSE1())
145 return 0;
146
147 if (!ST->is64Bit())
148 return 8;
149
150 if ((ClassID == GPRClass && ST->hasEGPR()) ||
151 (ClassID != GPRClass && ST->hasAVX512()))
152 return 32;
153
154 return 16;
155}
156
157bool X86TTIImpl::hasConditionalLoadStoreForType(Type *Ty, bool IsStore) const {
158 if (!ST->hasCF())
159 return false;
160 if (!Ty)
161 return true;
162 // Conditional faulting is supported by CFCMOV, which only accepts
163 // 16/32/64-bit operands.
164 // TODO: Support f32/f64 with VMOVSS/VMOVSD with zero mask when it's
165 // profitable.
166 auto *VTy = dyn_cast<FixedVectorType>(Val: Ty);
167 if (!Ty->isIntegerTy() && (!VTy || VTy->getNumElements() != 1))
168 return false;
169 auto *ScalarTy = Ty->getScalarType();
170 switch (cast<IntegerType>(Val: ScalarTy)->getBitWidth()) {
171 default:
172 return false;
173 case 16:
174 case 32:
175 case 64:
176 return true;
177 }
178}
179
180TypeSize
181X86TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
182 unsigned PreferVectorWidth = ST->getPreferVectorWidth();
183 switch (K) {
184 case TargetTransformInfo::RGK_Scalar:
185 return TypeSize::getFixed(ExactSize: ST->is64Bit() ? 64 : 32);
186 case TargetTransformInfo::RGK_FixedWidthVector:
187 if (ST->hasAVX512() && PreferVectorWidth >= 512)
188 return TypeSize::getFixed(ExactSize: 512);
189 if (ST->hasAVX() && PreferVectorWidth >= 256)
190 return TypeSize::getFixed(ExactSize: 256);
191 if (ST->hasSSE1() && PreferVectorWidth >= 128)
192 return TypeSize::getFixed(ExactSize: 128);
193 return TypeSize::getFixed(ExactSize: 0);
194 case TargetTransformInfo::RGK_ScalableVector:
195 return TypeSize::getScalable(MinimumSize: 0);
196 }
197
198 llvm_unreachable("Unsupported register kind");
199}
200
201unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
202 return getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector)
203 .getFixedValue();
204}
205
206unsigned X86TTIImpl::getMaxInterleaveFactor(ElementCount VF,
207 bool HasUnorderedReductions) const {
208 // If the loop will not be vectorized, don't interleave the loop.
209 // Let regular unroll to unroll the loop, which saves the overflow
210 // check and memory check cost.
211 if (VF.isScalar())
212 return 1;
213
214 if (ST->isAtom())
215 return 1;
216
217 // Sandybridge and Haswell have multiple execution ports and pipelined
218 // vector units.
219 if (ST->hasAVX())
220 return 4;
221
222 return 2;
223}
224
225InstructionCost X86TTIImpl::getArithmeticInstrCost(
226 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
227 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
228 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
229
230 // vXi8 multiplications are always promoted to vXi16.
231 // Sub-128-bit types can be extended/packed more efficiently.
232 if (Opcode == Instruction::Mul && Ty->isVectorTy() &&
233 Ty->getPrimitiveSizeInBits() <= 64 && Ty->getScalarSizeInBits() == 8) {
234 Type *WideVecTy =
235 VectorType::getExtendedElementVectorType(VTy: cast<VectorType>(Val: Ty));
236 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideVecTy, Src: Ty,
237 CCH: TargetTransformInfo::CastContextHint::None,
238 CostKind) +
239 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
240 CCH: TargetTransformInfo::CastContextHint::None,
241 CostKind) +
242 getArithmeticInstrCost(Opcode, Ty: WideVecTy, CostKind, Op1Info, Op2Info);
243 }
244
245 // Legalize the type.
246 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
247
248 int ISD = TLI->InstructionOpcodeToISD(Opcode);
249 assert(ISD && "Invalid opcode");
250
251 if (ISD == ISD::MUL && Args.size() == 2 && LT.second.isVector() &&
252 (LT.second.getScalarType() == MVT::i32 ||
253 LT.second.getScalarType() == MVT::i64)) {
254 // Check if the operands can be represented as a smaller datatype.
255 bool Op1Signed = false, Op2Signed = false;
256 unsigned Op1MinSize = BaseT::minRequiredElementSize(Val: Args[0], isSigned&: Op1Signed);
257 unsigned Op2MinSize = BaseT::minRequiredElementSize(Val: Args[1], isSigned&: Op2Signed);
258 unsigned OpMinSize = std::max(a: Op1MinSize, b: Op2MinSize);
259 bool SignedMode = Op1Signed || Op2Signed;
260
261 // If both vXi32 are representable as i15 and at least one is constant,
262 // zero-extended, or sign-extended from vXi16 (or less pre-SSE41) then we
263 // can treat this as PMADDWD which has the same costs as a vXi16 multiply.
264 if (OpMinSize <= 15 && !ST->isPMADDWDSlow() &&
265 LT.second.getScalarType() == MVT::i32) {
266 bool Op1Constant =
267 isa<ConstantDataVector>(Val: Args[0]) || isa<ConstantVector>(Val: Args[0]);
268 bool Op2Constant =
269 isa<ConstantDataVector>(Val: Args[1]) || isa<ConstantVector>(Val: Args[1]);
270 bool Op1Sext = isa<SExtInst>(Val: Args[0]) &&
271 (Op1MinSize == 15 || (Op1MinSize < 15 && !ST->hasSSE41()));
272 bool Op2Sext = isa<SExtInst>(Val: Args[1]) &&
273 (Op2MinSize == 15 || (Op2MinSize < 15 && !ST->hasSSE41()));
274
275 bool IsZeroExtended = !Op1Signed || !Op2Signed;
276 bool IsConstant = Op1Constant || Op2Constant;
277 bool IsSext = Op1Sext || Op2Sext;
278 if (IsConstant || IsZeroExtended || IsSext)
279 LT.second =
280 MVT::getVectorVT(VT: MVT::i16, NumElements: 2 * LT.second.getVectorNumElements());
281 }
282
283 // Check if the vXi32 operands can be shrunk into a smaller datatype.
284 // This should match the codegen from reduceVMULWidth.
285 // TODO: Make this generic (!ST->SSE41 || ST->isPMULLDSlow()).
286 if (ST->useSLMArithCosts() && LT.second == MVT::v4i32) {
287 if (OpMinSize <= 7)
288 return LT.first * 3; // pmullw/sext
289 if (!SignedMode && OpMinSize <= 8)
290 return LT.first * 3; // pmullw/zext
291 if (OpMinSize <= 15)
292 return LT.first * 5; // pmullw/pmulhw/pshuf
293 if (!SignedMode && OpMinSize <= 16)
294 return LT.first * 5; // pmullw/pmulhw/pshuf
295 }
296
297 // If both vXi64 are representable as (unsigned) i32, then we can perform
298 // the multiple with a single PMULUDQ instruction.
299 // TODO: Add (SSE41+) PMULDQ handling for signed extensions.
300 if (!SignedMode && OpMinSize <= 32 && LT.second.getScalarType() == MVT::i64)
301 ISD = X86ISD::PMULUDQ;
302 }
303
304 // Vector multiply by pow2 will be simplified to shifts.
305 // Vector multiply by -pow2 will be simplified to shifts/negates.
306 if (ISD == ISD::MUL && Op2Info.isConstant() &&
307 (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2())) {
308 InstructionCost Cost =
309 getArithmeticInstrCost(Opcode: Instruction::Shl, Ty, CostKind,
310 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
311 if (Op2Info.isNegatedPowerOf2())
312 Cost += getArithmeticInstrCost(Opcode: Instruction::Sub, Ty, CostKind);
313 return Cost;
314 }
315
316 // On X86, vector signed division by constants power-of-two are
317 // normally expanded to the sequence SRA + SRL + ADD + SRA.
318 // The OperandValue properties may not be the same as that of the previous
319 // operation; conservatively assume OP_None.
320 if ((ISD == ISD::SDIV || ISD == ISD::SREM) &&
321 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
322 InstructionCost Cost =
323 2 * getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
324 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
325 Cost += getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
326 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
327 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
328 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
329
330 if (ISD == ISD::SREM) {
331 // For SREM: (X % C) is the equivalent of (X - (X/C)*C)
332 Cost += getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind, Op1Info: Op1Info.getNoProps(),
333 Op2Info: Op2Info.getNoProps());
334 Cost += getArithmeticInstrCost(Opcode: Instruction::Sub, Ty, CostKind, Op1Info: Op1Info.getNoProps(),
335 Op2Info: Op2Info.getNoProps());
336 }
337
338 return Cost;
339 }
340
341 // Vector unsigned division/remainder will be simplified to shifts/masks.
342 if ((ISD == ISD::UDIV || ISD == ISD::UREM) &&
343 Op2Info.isConstant() && Op2Info.isPowerOf2()) {
344 if (ISD == ISD::UDIV)
345 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
346 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
347 // UREM
348 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind,
349 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
350 }
351
352 // A rem paired with a div on the same operands in the same block is free:
353 // the pair costs the div alone. Power-of-two divisors are handled above.
354 if ((ISD == ISD::UREM || ISD == ISD::SREM) && CostKind != TTI::TCK_Latency &&
355 CtxI && CtxI->getOpcode() == Opcode) {
356 using namespace PatternMatch;
357 // Same use limit as SLP, bounds the walk over the operand users.
358 constexpr unsigned MaxUses = 64;
359 unsigned DivOpcode =
360 ISD == ISD::SREM ? Instruction::SDiv : Instruction::UDiv;
361 const Value *X = CtxI->getOperand(i: 0);
362 const Value *Y = CtxI->getOperand(i: 1);
363 // Constants have context-wide use lists, walk the variable operand.
364 const Value *V = isa<Constant>(Val: Y) ? X : Y;
365 if (!isa<Constant>(Val: V) && !V->hasNUsesOrMore(N: MaxUses) &&
366 any_of(Range: V->users(), P: [&](const User *U) {
367 return match(V: U, P: m_BinOp(Opcode: DivOpcode, L: m_Specific(V: X), R: m_Specific(V: Y))) &&
368 cast<Instruction>(Val: U)->getParent() == CtxI->getParent();
369 }))
370 return TTI::TCC_Free;
371 }
372
373 // A scalar integer divide/remainder by a constant is not a hardware divide;
374 // it lowers to a magic-number multiply-high plus a few fixup ops. Cost it as
375 // that sequence rather than the generic single-instruction divide, so the
376 // vectorizers do not compare against an artificially cheap scalar lane. The
377 // power-of-two cases are handled above; negated powers of two are left to the
378 // generic handling.
379 if (!Ty->isVectorTy() && Op2Info.isConstant() && !Op2Info.isNegatedPowerOf2() &&
380 (ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
381 ISD == ISD::SREM)) {
382 unsigned Cost = ISD == ISD::UREM || ISD == ISD::SREM ? 6 : 5;
383 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
384 Cost += 2;
385 return LT.first * Cost;
386 }
387
388 static const CostKindTblEntry GFNIUniformConstCostTable[] = {
389 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
390 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
391 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
392 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
393 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
394 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
395 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
396 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
397 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
398 };
399
400 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasGFNI())
401 if (const auto *Entry =
402 CostTableLookup(Table: GFNIUniformConstCostTable, ISD, Ty: LT.second))
403 if (auto KindCost = Entry->Cost[CostKind])
404 return LT.first * *KindCost;
405
406 static const CostKindTblEntry AVX512BWUniformConstCostTable[] = {
407 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
408 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
409 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw, pand, pxor, psubb.
410 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
411 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
412 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw, pand, pxor, psubb.
413 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
414 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
415 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
416
417 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
418 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
419 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
420 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
421 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
422 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
423 };
424
425 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasBWI())
426 if (const auto *Entry =
427 CostTableLookup(Table: AVX512BWUniformConstCostTable, ISD, Ty: LT.second))
428 if (auto KindCost = Entry->Cost[CostKind])
429 return LT.first * *KindCost;
430
431 static const CostKindTblEntry AVX512DQUniformConstCostTable[] = {
432 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15 } }, // vpmullq-based MULHS sequence
433 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17 } }, // vpmullq-based MULHS+mul+sub sequence
434 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 15 } }, // vpmullq-based MULHS sequence
435 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 17 } }, // vpmullq-based MULHS+mul+sub sequence
436 // The remainder's multiply-back is a single vpmullq with DQ, just like the
437 // pmulld the vXi32 entries above rely on. Without DQ it is another
438 // vpmuludq schoolbook, so the AVX512/AVX2 tables charge more.
439 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17 } }, // MULHU + vpmullq + sub sequence
440 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 17 } }, // MULHU + vpmullq + sub sequence
441 };
442
443 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasDQI())
444 if (const auto *Entry =
445 CostTableLookup(Table: AVX512DQUniformConstCostTable, ISD, Ty: LT.second))
446 if (auto KindCost = Entry->Cost[CostKind])
447 return LT.first * *KindCost;
448
449 static const CostKindTblEntry AVX512UniformConstCostTable[] = {
450 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 12, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psllw + pand.
451 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 12, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw + pand.
452 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 12, .SizeAndLatencyCost: 12 } }, // psrlw, pand, pxor, psubb.
453
454 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psllw + split.
455 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psrlw + split.
456 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // psraw + split.
457
458 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
459 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
460 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
461 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
462 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
463 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
464
465 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
466 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
467 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
468 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
469 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
470 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
471 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraq
472
473 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
474 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
475 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
476 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
477
478 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq-based MULHU sequence
479 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 21 } }, // pmuludq-based MULHU+mul+sub sequence
480 };
481
482 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX512())
483 if (const auto *Entry =
484 CostTableLookup(Table: AVX512UniformConstCostTable, ISD, Ty: LT.second))
485 if (auto KindCost = Entry->Cost[CostKind])
486 return LT.first * *KindCost;
487
488 static const CostKindTblEntry AVX2UniformConstCostTable[] = {
489 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
490 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
491 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
492 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // psllw + pand.
493 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // psrlw + pand.
494 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // psrlw, pand, pxor, psubb.
495
496 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw
497 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw
498 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw
499 { .ISD: ISD::SHL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw
500 { .ISD: ISD::SRL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw
501 { .ISD: ISD::SRA, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw
502
503 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
504 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld
505 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad
506 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
507 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld
508 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad
509
510 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq
511 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq
512 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // psrad + shuffle.
513 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
514 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
515 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // psrad + shuffle + split.
516
517 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
518 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
519 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
520 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
521
522 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq-based MULHU sequence
523 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 21 } }, // pmuludq-based MULHU+mul+sub sequence
524 };
525
526 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX2())
527 if (const auto *Entry =
528 CostTableLookup(Table: AVX2UniformConstCostTable, ISD, Ty: LT.second))
529 if (auto KindCost = Entry->Cost[CostKind])
530 return LT.first * *KindCost;
531
532 static const CostKindTblEntry AVXUniformConstCostTable[] = {
533 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
534 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
535 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
536 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2*(psllw + pand) + split.
537 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2*(psrlw + pand) + split.
538 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 12, .SizeAndLatencyCost: 13 } }, // 2*(psrlw, pand, pxor, psubb) + split.
539
540 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw.
541 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw.
542 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw.
543 { .ISD: ISD::SHL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psllw + split.
544 { .ISD: ISD::SRL, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrlw + split.
545 { .ISD: ISD::SRA, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psraw + split.
546
547 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld.
548 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld.
549 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad.
550 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // pslld + split.
551 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrld + split.
552 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // psrad + split.
553
554 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq.
555 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq.
556 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // psrad + shuffle.
557 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // 2 x psllq + split.
558 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // 2 x psllq + split.
559 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } }, // 2 x psrad + shuffle + split.
560
561 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmuludq sequence + split.
562 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmuludq+mul+sub sequence + split.
563 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 12 } }, // 2*pmuludq sequence + split.
564 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } }, // 2*pmuludq+mul+sub sequence + split.
565 };
566
567 // XOP has faster vXi8 shifts.
568 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX() &&
569 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
570 if (const auto *Entry =
571 CostTableLookup(Table: AVXUniformConstCostTable, ISD, Ty: LT.second))
572 if (auto KindCost = Entry->Cost[CostKind])
573 return LT.first * *KindCost;
574
575 static const CostKindTblEntry SSE2UniformConstCostTable[] = {
576 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw + pand.
577 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw + pand.
578 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psrlw, pand, pxor, psubb.
579
580 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllw.
581 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlw.
582 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psraw.
583
584 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pslld
585 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrld.
586 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrad.
587
588 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psllq.
589 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psrlq.
590 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } }, // 2 x psrad + shuffle.
591
592 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6 } }, // pmuludq sequence
593 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } }, // pmuludq+mul+sub sequence
594 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5 } }, // pmuludq sequence
595 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } }, // pmuludq+mul+sub sequence
596 };
597
598 // XOP has faster vXi8 shifts.
599 if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasSSE2() &&
600 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
601 if (const auto *Entry =
602 CostTableLookup(Table: SSE2UniformConstCostTable, ISD, Ty: LT.second))
603 if (auto KindCost = Entry->Cost[CostKind])
604 return LT.first * *KindCost;
605
606 static const CostKindTblEntry AVX512BWConstCostTable[] = {
607 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
608 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
609 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
610 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
611
612 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhw sequence
613 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhw+mul+sub sequence
614 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhuw sequence
615 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhuw+mul+sub sequence
616 };
617
618 if (Op2Info.isConstant() && ST->hasBWI())
619 if (const auto *Entry =
620 CostTableLookup(Table: AVX512BWConstCostTable, ISD, Ty: LT.second))
621 if (auto KindCost = Entry->Cost[CostKind])
622 return LT.first * *KindCost;
623
624 static const CostKindTblEntry AVX512DQConstCostTable[] = {
625 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 19 } }, // vpmullq-based MULHS sequence
626 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 21 } }, // vpmullq-based MULHS+mul+sub sequence
627 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 19 } }, // vpmullq-based MULHS sequence
628 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 21 } }, // vpmullq-based MULHS+mul+sub sequence
629 // The remainder's multiply-back is a single vpmullq with DQ, whereas the
630 // AVX512/AVX2 tables have to charge for another vpmuludq schoolbook.
631 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24 } }, // MULHU + vpmullq + sub sequence
632 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 24 } }, // MULHU + vpmullq + sub sequence
633 };
634
635 if (Op2Info.isConstant() && ST->hasDQI())
636 if (const auto *Entry =
637 CostTableLookup(Table: AVX512DQConstCostTable, ISD, Ty: LT.second))
638 if (auto KindCost = Entry->Cost[CostKind])
639 return LT.first * *KindCost;
640
641 static const CostKindTblEntry AVX512ConstCostTable[] = {
642 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 28 } }, // 4*ext+4*pmulhw sequence
643 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
644 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 28 } }, // 4*ext+4*pmulhw sequence
645 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
646
647 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 12 } }, // 2*vpmulhw sequence
648 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 16 } }, // 2*vpmulhw+mul+sub sequence
649 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 12 } }, // 2*vpmulhuw sequence
650 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 16 } }, // 2*vpmulhuw+mul+sub sequence
651
652 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuldq sequence
653 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 17 } }, // vpmuldq+mul+sub sequence
654 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
655 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 17 } }, // vpmuludq+mul+sub sequence
656
657 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 22 } }, // vpmuludq-based MULHU sequence
658 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 28 } }, // vpmuludq-based MULHU+mul+sub sequence
659 };
660
661 if (Op2Info.isConstant() && ST->hasAVX512())
662 if (const auto *Entry =
663 CostTableLookup(Table: AVX512ConstCostTable, ISD, Ty: LT.second))
664 if (auto KindCost = Entry->Cost[CostKind])
665 return LT.first * *KindCost;
666
667 static const CostKindTblEntry AVX2ConstCostTable[] = {
668 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
669 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
670 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
671 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
672
673 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhw sequence
674 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhw+mul+sub sequence
675 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6 } }, // vpmulhuw sequence
676 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8 } }, // vpmulhuw+mul+sub sequence
677
678 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuldq sequence
679 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 19 } }, // vpmuldq+mul+sub sequence
680 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
681 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 19 } }, // vpmuludq+mul+sub sequence
682
683 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 22 } }, // vpmuludq-based MULHU sequence
684 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 28 } }, // vpmuludq-based MULHU+mul+sub sequence
685 };
686
687 if (Op2Info.isConstant() && ST->hasAVX2())
688 if (const auto *Entry = CostTableLookup(Table: AVX2ConstCostTable, ISD, Ty: LT.second))
689 if (auto KindCost = Entry->Cost[CostKind])
690 return LT.first * *KindCost;
691
692 static const CostKindTblEntry AVXConstCostTable[] = {
693 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 30 } }, // 4*ext+4*pmulhw sequence + split.
694 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
695 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 30 } }, // 4*ext+4*pmulhw sequence + split.
696 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 34 } }, // 4*ext+4*pmulhw+mul+sub sequence + split.
697
698 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmulhw sequence + split.
699 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmulhw+mul+sub sequence + split.
700 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } }, // 2*pmulhuw sequence + split.
701 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18 } }, // 2*pmulhuw+mul+sub sequence + split.
702
703 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 32 } }, // vpmuludq sequence
704 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 38 } }, // vpmuludq+mul+sub sequence
705 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 32 } }, // 2*pmuludq sequence + split.
706 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 42 } }, // 2*pmuludq+mul+sub sequence + split.
707 };
708
709 if (Op2Info.isConstant() && ST->hasAVX())
710 if (const auto *Entry = CostTableLookup(Table: AVXConstCostTable, ISD, Ty: LT.second))
711 if (auto KindCost = Entry->Cost[CostKind])
712 return LT.first * *KindCost;
713
714 static const CostKindTblEntry SSE41ConstCostTable[] = {
715 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15 } }, // vpmuludq sequence
716 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20 } }, // vpmuludq+mul+sub sequence
717 };
718
719 if (Op2Info.isConstant() && ST->hasSSE41())
720 if (const auto *Entry =
721 CostTableLookup(Table: SSE41ConstCostTable, ISD, Ty: LT.second))
722 if (auto KindCost = Entry->Cost[CostKind])
723 return LT.first * *KindCost;
724
725 static const CostKindTblEntry SSE2ConstCostTable[] = {
726 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
727 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
728 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // 2*ext+2*pmulhw sequence
729 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16 } }, // 2*ext+2*pmulhw+mul+sub sequence
730
731 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6 } }, // pmulhw sequence
732 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8 } }, // pmulhw+mul+sub sequence
733 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6 } }, // pmulhuw sequence
734 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8 } }, // pmulhuw+mul+sub sequence
735
736 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19 } }, // pmuludq sequence
737 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 24 } }, // pmuludq+mul+sub sequence
738 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15 } }, // pmuludq sequence
739 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20 } }, // pmuludq+mul+sub sequence
740 };
741
742 if (Op2Info.isConstant() && ST->hasSSE2())
743 if (const auto *Entry = CostTableLookup(Table: SSE2ConstCostTable, ISD, Ty: LT.second))
744 if (auto KindCost = Entry->Cost[CostKind])
745 return LT.first * *KindCost;
746
747 // rem matches div because the divider returns the remainder for free.
748 static const CostKindTblEntry ScalarVarDivCostTable[] = {
749 { .ISD: ISD::SDIV, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
750 { .ISD: ISD::UDIV, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
751 { .ISD: ISD::SREM, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
752 { .ISD: ISD::UREM, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
753 { .ISD: ISD::SDIV, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
754 { .ISD: ISD::UDIV, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
755 { .ISD: ISD::SREM, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
756 { .ISD: ISD::UREM, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
757 { .ISD: ISD::SDIV, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
758 { .ISD: ISD::UDIV, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
759 { .ISD: ISD::SREM, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
760 { .ISD: ISD::UREM, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 25, .LatencyCost: 22, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
761 { .ISD: ISD::SDIV, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
762 { .ISD: ISD::UDIV, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
763 { .ISD: ISD::SREM, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
764 { .ISD: ISD::UREM, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 41, .LatencyCost: 24, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
765 };
766
767 if (!LT.second.isVector() && !Op2Info.isConstant())
768 if (const auto *Entry =
769 CostTableLookup(Table: ScalarVarDivCostTable, ISD, Ty: LT.second))
770 if (auto KindCost = Entry->Cost[CostKind])
771 return LT.first * *KindCost;
772
773 // Variable divisors lower through a float divide. strictfp needs SAE
774 // rounding which is 512-bit only.
775 bool IsStrictFP =
776 CtxI && CtxI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP);
777 bool IsDivRem = ISD == ISD::UDIV || ISD == ISD::SDIV || ISD == ISD::UREM ||
778 ISD == ISD::SREM;
779 bool VarDivToFP = IsDivRem && !Op2Info.isConstant() &&
780 (!IsStrictFP || ST->useAVX512Regs());
781
782 // i64 needs the qq converts, which are AVX512DQ only. Two tables because the
783 // lowering picks by operand value and not by type.
784 static const CostKindTblEntry AVX512DQExactVarDivCostTable[] = {
785 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } }, // cvt+divpd sequence
786 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
787 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
788 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5 } },
789 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
790 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
791 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
792 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8 } },
793 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
794 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
795 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
796 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
797 };
798
799 static const CostKindTblEntry AVX512DQVarDivCostTable[] = {
800 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
801 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
802 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
803 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16 } },
804 { .ISD: ISD::UDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
805 { .ISD: ISD::SDIV, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
806 { .ISD: ISD::UREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
807 { .ISD: ISD::SREM, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 16 } },
808 { .ISD: ISD::UDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
809 { .ISD: ISD::SDIV, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 18 } },
810 { .ISD: ISD::UREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16 } },
811 { .ISD: ISD::SREM, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 18 } },
812 };
813
814 // The DAG combine picks between the two sequences with these same two
815 // queries, so the cost cannot disagree with what codegen emits.
816 bool IsSignedDiv = ISD == ISD::SDIV || ISD == ISD::SREM;
817 auto OperandsFit = [&](unsigned Mantissa) {
818 if (Args.size() != 2 || !CtxI)
819 return false;
820 unsigned EltBits = LT.second.getScalarSizeInBits();
821 const DataLayout &DL = CtxI->getDataLayout();
822 auto Fits = [&](const Value *V) {
823 if (IsSignedDiv)
824 return ComputeNumSignBits(Op: V, DL, /*AC=*/nullptr, CtxI) + Mantissa >
825 EltBits;
826 return computeKnownBits(V, DL, /*AC=*/nullptr, CtxI)
827 .countMaxActiveBits() <= Mantissa;
828 };
829 return Fits(Args[0]) && Fits(Args[1]);
830 };
831
832 // An i32 divide goes through f32 when both operands fit in 24 bits, and
833 // through f64 at twice the vector width when they do not.
834 bool ExactI32 =
835 VarDivToFP && LT.second.getScalarType() == MVT::i32 &&
836 OperandsFit(APFloat::semanticsPrecision(APFloat::IEEEsingle()));
837
838 if (VarDivToFP && ST->hasDQI() && ST->useAVX512Regs() &&
839 LT.second.getScalarType() == MVT::i64) {
840 bool ExactFPDiv =
841 OperandsFit(APFloat::semanticsPrecision(APFloat::IEEEdouble()));
842
843 // Only the reciprocal chain multiplies, so only it is vpmullq gated.
844 bool SlowMultiply =
845 !ExactFPDiv && LT.second == MVT::v2i64 && ST->isPMULLQSlow();
846
847 if (!SlowMultiply) {
848 ArrayRef<CostKindTblEntry> Tbl =
849 ExactFPDiv ? AVX512DQExactVarDivCostTable : AVX512DQVarDivCostTable;
850 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
851 if (auto KindCost = Entry->Cost[CostKind])
852 return LT.first * *KindCost;
853 }
854 }
855
856 static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
857 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } }, // unpack+cvt+divps sequence
858 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
859 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
860 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10 } },
861 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
862 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
863 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
864 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20 } },
865 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
866 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
867 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
868 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 40 } },
869 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
870 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
871 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
872 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5 } },
873 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
874 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
875 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
876 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 10 } },
877 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
878 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
879 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
880 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 20 } },
881 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } }, // cvt+divpd sequence
882 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
883 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
884 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
885 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
886 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
887 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
888 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 16 } },
889 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
890 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
891 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
892 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 32 } },
893 };
894
895 static const CostKindTblEntry AVX512BWExactVarDivCostTable[] = {
896 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } }, // cvt+divps sequence
897 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
898 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
899 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3 } },
900 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
901 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
902 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
903 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5 } },
904 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
905 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
906 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
907 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 10 } },
908 };
909
910 if (VarDivToFP && ST->hasBWI()) {
911 ArrayRef<CostKindTblEntry> Tbl = AVX512BWVarDivCostTable;
912 if (ExactI32)
913 Tbl = AVX512BWExactVarDivCostTable;
914 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
915 if (auto KindCost = Entry->Cost[CostKind])
916 return LT.first * *KindCost;
917 }
918
919 static const CostKindTblEntry AVX512VarDivCostTable[] = {
920 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } }, // unpack+cvt+divps sequence
921 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
922 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
923 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14 } },
924 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
925 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
926 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
927 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 28 } },
928 { .ISD: ISD::UDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
929 { .ISD: ISD::SDIV, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
930 { .ISD: ISD::UREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
931 { .ISD: ISD::SREM, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 56 } },
932 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
933 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
934 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
935 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
936 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
937 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
938 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
939 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 14 } },
940 { .ISD: ISD::UDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
941 { .ISD: ISD::SDIV, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
942 { .ISD: ISD::UREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
943 { .ISD: ISD::SREM, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 28 } },
944 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } }, // cvt+divpd sequence
945 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
946 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
947 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
948 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
949 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
950 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
951 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
952 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
953 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
954 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
955 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 56 } },
956 };
957
958 static const CostKindTblEntry AVX512ExactVarDivCostTable[] = {
959 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } }, // cvt+divps sequence
960 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
961 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
962 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7 } },
963 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
964 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
965 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
966 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
967 { .ISD: ISD::UDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
968 { .ISD: ISD::SDIV, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
969 { .ISD: ISD::UREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
970 { .ISD: ISD::SREM, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 14 } },
971 };
972
973 if (VarDivToFP && ST->hasAVX512()) {
974 ArrayRef<CostKindTblEntry> Tbl = AVX512VarDivCostTable;
975 if (ExactI32)
976 Tbl = AVX512ExactVarDivCostTable;
977 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
978 if (auto KindCost = Entry->Cost[CostKind])
979 return LT.first * *KindCost;
980 }
981
982 static const CostKindTblEntry AVX2VarDivCostTable[] = {
983 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } }, // unpack+cvt+divps sequence
984 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
985 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
986 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 28 } },
987 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
988 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
989 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
990 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 56 } },
991 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
992 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
993 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
994 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 14 } },
995 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
996 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
997 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
998 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28 } },
999 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } }, // cvt+divpd sequence
1000 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1001 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1002 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 28 } },
1003 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1004 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1005 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1006 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 56 } },
1007 };
1008
1009 static const CostKindTblEntry AVX2ExactVarDivCostTable[] = {
1010 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9 } }, // cvt+divps sequence
1011 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
1012 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9 } },
1013 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8 } },
1014 { .ISD: ISD::UDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1015 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1016 { .ISD: ISD::UREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1017 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14 } },
1018 };
1019
1020 if (VarDivToFP && ST->hasAVX2()) {
1021 ArrayRef<CostKindTblEntry> Tbl = AVX2VarDivCostTable;
1022 if (ExactI32)
1023 Tbl = AVX2ExactVarDivCostTable;
1024 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1025 if (auto KindCost = Entry->Cost[CostKind])
1026 return LT.first * *KindCost;
1027 }
1028
1029 // No unsigned i32 entries below AVX2, where the u32 to f64 converts are
1030 // emulated and the fold stays off.
1031 static const CostKindTblEntry AVX1VarDivCostTable[] = {
1032 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } }, // unpack+cvt+divps sequence
1033 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1034 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1035 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1036 { .ISD: ISD::UDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1037 { .ISD: ISD::SDIV, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1038 { .ISD: ISD::UREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1039 { .ISD: ISD::SREM, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 112 } },
1040 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1041 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1042 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1043 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1044 { .ISD: ISD::UDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1045 { .ISD: ISD::SDIV, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1046 { .ISD: ISD::UREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1047 { .ISD: ISD::SREM, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 56 } },
1048 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } }, // cvt+divpd sequence
1049 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } },
1050 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 88 } },
1051 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 88 } },
1052 };
1053
1054 static const CostKindTblEntry AVX1ExactVarDivCostTable[] = {
1055 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } }, // cvt+divps sequence
1056 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } },
1057 { .ISD: ISD::SDIV, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
1058 { .ISD: ISD::SREM, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 28 } },
1059 };
1060
1061 if (VarDivToFP && ST->hasAVX()) {
1062 ArrayRef<CostKindTblEntry> Tbl = AVX1VarDivCostTable;
1063 if (ExactI32)
1064 Tbl = AVX1ExactVarDivCostTable;
1065 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1066 if (auto KindCost = Entry->Cost[CostKind])
1067 return LT.first * *KindCost;
1068 }
1069
1070 static const CostKindTblEntry SSE2VarDivCostTable[] = {
1071 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } }, // unpack+cvt+divps sequence
1072 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1073 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1074 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 56 } },
1075 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1076 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1077 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1078 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 28 } },
1079 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } }, // cvt+divpd sequence
1080 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 44 } },
1081 };
1082
1083 static const CostKindTblEntry SSE2ExactVarDivCostTable[] = {
1084 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } }, // cvt+divps sequence
1085 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14 } },
1086 };
1087
1088 if (VarDivToFP && ST->hasSSE2()) {
1089 ArrayRef<CostKindTblEntry> Tbl = SSE2VarDivCostTable;
1090 if (ExactI32)
1091 Tbl = SSE2ExactVarDivCostTable;
1092 if (const auto *Entry = CostTableLookup(Tbl, ISD, Ty: LT.second))
1093 if (auto KindCost = Entry->Cost[CostKind])
1094 return LT.first * *KindCost;
1095 }
1096
1097 static const CostKindTblEntry AVX512BWUniformCostTable[] = {
1098 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + pand.
1099 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1100 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 8,.SizeAndLatencyCost: 12 } }, // psrlw, pand, pxor, psubb.
1101 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1102 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1103 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1104 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1105 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // psrlw + pand.
1106 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 15 } }, // psrlw, pand, pxor, psubb.
1107
1108 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw
1109 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw
1110 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrqw
1111 };
1112
1113 if (ST->hasBWI() && Op2Info.isUniform())
1114 if (const auto *Entry =
1115 CostTableLookup(Table: AVX512BWUniformCostTable, ISD, Ty: LT.second))
1116 if (auto KindCost = Entry->Cost[CostKind])
1117 return LT.first * *KindCost;
1118
1119 static const CostKindTblEntry AVX512UniformCostTable[] = {
1120 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + split.
1121 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrlw + split.
1122 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psraw + split.
1123
1124 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pslld
1125 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrld
1126 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrad
1127
1128 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1129 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1130 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1131 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1132 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1133 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1134 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraq
1135 };
1136
1137 if (ST->hasAVX512() && Op2Info.isUniform())
1138 if (const auto *Entry =
1139 CostTableLookup(Table: AVX512UniformCostTable, ISD, Ty: LT.second))
1140 if (auto KindCost = Entry->Cost[CostKind])
1141 return LT.first * *KindCost;
1142
1143 static const CostKindTblEntry AVX2UniformCostTable[] = {
1144 // Uniform splats are cheaper for the following instructions.
1145 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + pand.
1146 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1147 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1148 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1149 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1150 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // psrlw, pand, pxor, psubb.
1151
1152 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1153 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1154 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1155 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psllw.
1156 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrlw.
1157 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psraw.
1158
1159 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
1160 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld
1161 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad
1162 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pslld
1163 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrld
1164 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // psrad
1165
1166 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1167 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1168 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2 x psrad + shuffle.
1169 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq
1170 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq
1171 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // 2 x psrad + shuffle.
1172 };
1173
1174 if (ST->hasAVX2() && Op2Info.isUniform())
1175 if (const auto *Entry =
1176 CostTableLookup(Table: AVX2UniformCostTable, ISD, Ty: LT.second))
1177 if (auto KindCost = Entry->Cost[CostKind])
1178 return LT.first * *KindCost;
1179
1180 static const CostKindTblEntry AVXUniformCostTable[] = {
1181 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } }, // psllw + pand.
1182 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } }, // psrlw + pand.
1183 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // psrlw, pand, pxor, psubb.
1184 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 8,.CodeSizeCost: 11,.SizeAndLatencyCost: 14 } }, // psllw + pand + split.
1185 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 9,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // psrlw + pand + split.
1186 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 10,.LatencyCost: 11,.CodeSizeCost: 16,.SizeAndLatencyCost: 21 } }, // psrlw, pand, pxor, psubb + split.
1187
1188 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1189 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1190 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1191 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psllw + split.
1192 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrlw + split.
1193 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psraw + split.
1194
1195 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld.
1196 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld.
1197 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad.
1198 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // pslld + split.
1199 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrld + split.
1200 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // psrad + split.
1201
1202 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq.
1203 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq.
1204 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2 x psrad + shuffle.
1205 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psllq + split.
1206 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // psrlq + split.
1207 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 10,.SizeAndLatencyCost: 13 } }, // 2 x (2 x psrad + shuffle) + split.
1208 };
1209
1210 // XOP has faster vXi8 shifts.
1211 if (ST->hasAVX() && Op2Info.isUniform() &&
1212 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1213 if (const auto *Entry =
1214 CostTableLookup(Table: AVXUniformCostTable, ISD, Ty: LT.second))
1215 if (auto KindCost = Entry->Cost[CostKind])
1216 return LT.first * *KindCost;
1217
1218 static const CostKindTblEntry SSE2UniformCostTable[] = {
1219 // Uniform splats are cheaper for the following instructions.
1220 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } }, // psllw + pand.
1221 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // psrlw + pand.
1222 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 15, .CodeSizeCost: 9,.SizeAndLatencyCost: 13 } }, // pcmpgtb sequence.
1223
1224 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllw.
1225 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlw.
1226 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psraw.
1227
1228 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pslld
1229 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrld.
1230 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrad.
1231
1232 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psllq.
1233 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psrlq.
1234 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 9, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // 2*psrlq + xor + sub.
1235 };
1236
1237 if (ST->hasSSE2() && Op2Info.isUniform() &&
1238 (!ST->hasXOP() || LT.second.getScalarSizeInBits() != 8))
1239 if (const auto *Entry =
1240 CostTableLookup(Table: SSE2UniformCostTable, ISD, Ty: LT.second))
1241 if (auto KindCost = Entry->Cost[CostKind])
1242 return LT.first * *KindCost;
1243
1244 static const CostKindTblEntry AVX512DQCostTable[] = {
1245 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmullq
1246 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmullq
1247 { .ISD: ISD::MUL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } } // pmullq
1248 };
1249
1250 // Look for AVX512DQ lowering tricks for custom cases.
1251 if (ST->hasDQI())
1252 if (const auto *Entry = CostTableLookup(Table: AVX512DQCostTable, ISD, Ty: LT.second))
1253 if (auto KindCost = Entry->Cost[CostKind])
1254 return LT.first * *KindCost;
1255
1256 static const CostKindTblEntry AVX512BWCostTable[] = {
1257 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsllvw/pack sequence.
1258 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsrlvw/pack sequence.
1259 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/vpsravw/pack sequence.
1260 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 23,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // extend/vpsllvw/pack sequence.
1261 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 30,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // extend/vpsrlvw/pack sequence.
1262 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13,.CodeSizeCost: 24,.SizeAndLatencyCost: 30 } }, // extend/vpsravw/pack sequence.
1263 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 19,.CodeSizeCost: 13,.SizeAndLatencyCost: 15 } }, // extend/vpsllvw/pack sequence.
1264 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 27,.CodeSizeCost: 15,.SizeAndLatencyCost: 18 } }, // extend/vpsrlvw/pack sequence.
1265 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 15,.CodeSizeCost: 30,.SizeAndLatencyCost: 30 } }, // extend/vpsravw/pack sequence.
1266
1267 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1268 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1269 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1270 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1271 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1272 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1273 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvw
1274 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvw
1275 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsravw
1276
1277 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddb
1278 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddw
1279
1280 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddb
1281 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddw
1282 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddd
1283 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddq
1284
1285 { .ISD: ISD::SUB, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubb
1286 { .ISD: ISD::SUB, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubw
1287
1288 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // extend/pmullw/trunc
1289 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // pmaddubsw
1290 { .ISD: ISD::MUL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10 } }, // pmaddubsw
1291 { .ISD: ISD::MUL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1292
1293 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubb
1294 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubw
1295 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubd
1296 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubq
1297 };
1298
1299 // Look for AVX512BW lowering tricks for custom cases.
1300 if (ST->hasBWI())
1301 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTable, ISD, Ty: LT.second))
1302 if (auto KindCost = Entry->Cost[CostKind])
1303 return LT.first * *KindCost;
1304
1305 static const CostKindTblEntry AVX512CostTable[] = {
1306 { .ISD: ISD::SHL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19,.CodeSizeCost: 27,.SizeAndLatencyCost: 33 } }, // vpblendv+split sequence.
1307 { .ISD: ISD::SRL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19,.CodeSizeCost: 30,.SizeAndLatencyCost: 36 } }, // vpblendv+split sequence.
1308 { .ISD: ISD::SRA, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 37, .LatencyCost: 37,.CodeSizeCost: 51,.SizeAndLatencyCost: 63 } }, // vpblendv+split sequence.
1309
1310 { .ISD: ISD::SHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsrlvd/pack sequence.
1311 { .ISD: ISD::SRL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsrlvd/pack sequence.
1312 { .ISD: ISD::SRA, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // 2*extend/vpsravd/pack sequence.
1313
1314 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1315 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1316 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1317 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1318 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1319 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1320 { .ISD: ISD::SHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1321 { .ISD: ISD::SRL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1322 { .ISD: ISD::SRA, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1323
1324 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1325 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1326 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1327 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1328 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1329 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1330 { .ISD: ISD::SHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1331 { .ISD: ISD::SRL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1332 { .ISD: ISD::SRA, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1333
1334 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*paddb + split
1335 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*paddw + split
1336
1337 { .ISD: ISD::SUB, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*psubb + split
1338 { .ISD: ISD::SUB, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // 2*psubw + split
1339
1340 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1341 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1342 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1343 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1344
1345 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1346 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1347 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1348 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1349
1350 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1351 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1352 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1353 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1354
1355 { .ISD: ISD::MUL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1356 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1357 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld (Skylake from agner.org)
1358 { .ISD: ISD::MUL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // 3*pmuludq/3*shift/2*add
1359 { .ISD: ISD::MUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Skylake from http://www.agner.org/
1360
1361 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1362
1363 { .ISD: ISD::FNEG, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Skylake from http://www.agner.org/
1364 { .ISD: ISD::FADD, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1365 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1366 { .ISD: ISD::FSUB, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1367 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1368 { .ISD: ISD::FMUL, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1369 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1370 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1371 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1372
1373 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1374 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1375 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1376 { .ISD: ISD::FDIV, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 23, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
1377
1378 { .ISD: ISD::FNEG, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Skylake from http://www.agner.org/
1379 { .ISD: ISD::FADD, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1380 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1381 { .ISD: ISD::FSUB, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1382 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1383 { .ISD: ISD::FMUL, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1384 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1385 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1386 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1387
1388 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1389 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1390 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
1391 { .ISD: ISD::FDIV, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
1392 };
1393
1394 if (ST->hasAVX512())
1395 if (const auto *Entry = CostTableLookup(Table: AVX512CostTable, ISD, Ty: LT.second))
1396 if (auto KindCost = Entry->Cost[CostKind])
1397 return LT.first * *KindCost;
1398
1399 static const CostKindTblEntry AVX2ShiftCostTable[] = {
1400 // Shifts on vXi64/vXi32 on AVX2 is legal even though we declare to
1401 // customize them to detect the cases where shift amount is a scalar one.
1402 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsllvd (Haswell from agner.org)
1403 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsrlvd (Haswell from agner.org)
1404 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsravd (Haswell from agner.org)
1405 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsllvd (Haswell from agner.org)
1406 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsrlvd (Haswell from agner.org)
1407 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vpsravd (Haswell from agner.org)
1408 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllvq (Haswell from agner.org)
1409 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsrlvq (Haswell from agner.org)
1410 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpsllvq (Haswell from agner.org)
1411 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpsrlvq (Haswell from agner.org)
1412 };
1413
1414 if (ST->hasAVX512()) {
1415 if (ISD == ISD::SHL && LT.second == MVT::v32i16 && Op2Info.isConstant())
1416 // On AVX512, a packed v32i16 shift left by a constant build_vector
1417 // is lowered into a vector multiply (vpmullw).
1418 return getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
1419 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1420 }
1421
1422 // Look for AVX2 lowering tricks (XOP is always better at v4i32 shifts).
1423 if (ST->hasAVX2() && !(ST->hasXOP() && LT.second == MVT::v4i32)) {
1424 if (ISD == ISD::SHL && LT.second == MVT::v16i16 &&
1425 Op2Info.isConstant())
1426 // On AVX2, a packed v16i16 shift left by a constant build_vector
1427 // is lowered into a vector multiply (vpmullw).
1428 return getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
1429 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1430
1431 if (const auto *Entry = CostTableLookup(Table: AVX2ShiftCostTable, ISD, Ty: LT.second))
1432 if (auto KindCost = Entry->Cost[CostKind])
1433 return LT.first * *KindCost;
1434 }
1435
1436 static const CostKindTblEntry XOPShiftCostTable[] = {
1437 // 128bit shifts take 1cy, but right shifts require negation beforehand.
1438 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1439 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1440 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1441 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1442 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1443 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1444 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1445 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1446 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1447 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1448 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1449 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1450 // 256bit shifts require splitting if AVX2 didn't catch them above.
1451 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1452 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1453 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1454 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1455 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1456 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1457 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1458 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1459 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1460 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1461 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1462 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
1463 };
1464
1465 // Look for XOP lowering tricks.
1466 if (ST->hasXOP()) {
1467 // If the right shift is constant then we'll fold the negation so
1468 // it's as cheap as a left shift.
1469 int ShiftISD = ISD;
1470 if ((ShiftISD == ISD::SRL || ShiftISD == ISD::SRA) && Op2Info.isConstant())
1471 ShiftISD = ISD::SHL;
1472 if (const auto *Entry =
1473 CostTableLookup(Table: XOPShiftCostTable, ISD: ShiftISD, Ty: LT.second))
1474 if (auto KindCost = Entry->Cost[CostKind])
1475 return LT.first * *KindCost;
1476 }
1477
1478 if (ISD == ISD::SHL && !Op2Info.isUniform() && Op2Info.isConstant()) {
1479 MVT VT = LT.second;
1480 // Vector shift left by non uniform constant can be lowered
1481 // into vector multiply.
1482 if (((VT == MVT::v8i16 || VT == MVT::v4i32) && ST->hasSSE2()) ||
1483 ((VT == MVT::v16i16 || VT == MVT::v8i32) && ST->hasAVX()))
1484 ISD = ISD::MUL;
1485 }
1486
1487 static const CostKindTblEntry GLMCostTable[] = {
1488 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 19, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divss
1489 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 35, .LatencyCost: 36, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divps
1490 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 33, .LatencyCost: 34, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divsd
1491 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 65, .LatencyCost: 66, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divpd
1492 };
1493
1494 if (ST->useGLMDivSqrtCosts())
1495 if (const auto *Entry = CostTableLookup(Table: GLMCostTable, ISD, Ty: LT.second))
1496 if (auto KindCost = Entry->Cost[CostKind])
1497 return LT.first * *KindCost;
1498
1499 static const CostKindTblEntry SLMCostTable[] = {
1500 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } }, // pmulld
1501 { .ISD: ISD::MUL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1502 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulsd
1503 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulss
1504 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulpd
1505 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // mulps
1506 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 19, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divss
1507 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 39, .LatencyCost: 39, .CodeSizeCost: 1, .SizeAndLatencyCost: 6 } }, // divps
1508 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 34, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // divsd
1509 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 69, .LatencyCost: 69, .CodeSizeCost: 1, .SizeAndLatencyCost: 6 } }, // divpd
1510 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // addpd
1511 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // subpd
1512 // v2i64/v4i64 mul is custom lowered as a series of long:
1513 // multiplies(3), shifts(3) and adds(2)
1514 // slm muldq version throughput is 2 and addq throughput 4
1515 // thus: 3X2 (muldq throughput) + 3X1 (shift throughput) +
1516 // 3X4 (addq throughput) = 17
1517 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 22, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
1518 // slm addq\subq throughput is 4
1519 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1520 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1521 };
1522
1523 if (ST->useSLMArithCosts())
1524 if (const auto *Entry = CostTableLookup(Table: SLMCostTable, ISD, Ty: LT.second))
1525 if (auto KindCost = Entry->Cost[CostKind])
1526 return LT.first * *KindCost;
1527
1528 static const CostKindTblEntry AVX2CostTable[] = {
1529 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 21,.CodeSizeCost: 11,.SizeAndLatencyCost: 16 } }, // vpblendvb sequence.
1530 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 23,.CodeSizeCost: 11,.SizeAndLatencyCost: 22 } }, // vpblendvb sequence.
1531 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsrlvd/pack sequence.
1532 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsrlvd/pack sequence.
1533
1534 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 27,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // vpblendvb sequence.
1535 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 30,.CodeSizeCost: 12,.SizeAndLatencyCost: 24 } }, // vpblendvb sequence.
1536 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsrlvd/pack sequence.
1537 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsrlvd/pack sequence.
1538
1539 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 17,.CodeSizeCost: 24,.SizeAndLatencyCost: 30 } }, // vpblendvb sequence.
1540 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 20,.CodeSizeCost: 24,.SizeAndLatencyCost: 43 } }, // vpblendvb sequence.
1541 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 5,.SizeAndLatencyCost: 10 } }, // extend/vpsravd/pack sequence.
1542 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // extend/vpsravd/pack sequence.
1543 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // srl/xor/sub sequence.
1544 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 9 } }, // srl/xor/sub sequence.
1545
1546 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubb
1547 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddb
1548 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubw
1549 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddw
1550 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubd
1551 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddd
1552 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubq
1553 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddq
1554
1555 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18, .CodeSizeCost: 6,.SizeAndLatencyCost: 12 } }, // extend/pmullw/pack
1556 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 8,.SizeAndLatencyCost: 16 } }, // pmaddubsw
1557 { .ISD: ISD::MUL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmullw
1558 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld
1559 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pmulld
1560 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 8,.SizeAndLatencyCost: 13 } }, // 3*pmuludq/3*shift/2*add
1561 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // 3*pmuludq/3*shift/2*add
1562
1563 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1564
1565 { .ISD: ISD::FNEG, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorpd
1566 { .ISD: ISD::FNEG, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1567
1568 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddsd
1569 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddss
1570 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddpd
1571 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vaddps
1572 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vaddpd
1573 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vaddps
1574
1575 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubsd
1576 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubss
1577 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubpd
1578 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsubps
1579 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vsubpd
1580 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vsubps
1581
1582 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulsd
1583 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulss
1584 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulpd
1585 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vmulps
1586 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vmulpd
1587 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vmulps
1588
1589 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivss
1590 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivps
1591 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vdivps
1592 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivsd
1593 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vdivpd
1594 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vdivpd
1595 };
1596
1597 // Look for AVX2 lowering tricks for custom cases.
1598 if (ST->hasAVX2())
1599 if (const auto *Entry = CostTableLookup(Table: AVX2CostTable, ISD, Ty: LT.second))
1600 if (auto KindCost = Entry->Cost[CostKind])
1601 return LT.first * *KindCost;
1602
1603 static const CostKindTblEntry AVX1CostTable[] = {
1604 // We don't have to scalarize unsupported ops. We can issue two half-sized
1605 // operations and we only need to extract the upper YMM half.
1606 // Two ops + 1 extract + 1 insert = 4.
1607 { .ISD: ISD::MUL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 11, .CodeSizeCost: 18, .SizeAndLatencyCost: 19 } }, // pmaddubsw + split
1608 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } }, // 2*pmaddubsw/3*and/psllw/or
1609 { .ISD: ISD::MUL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // pmullw + split
1610 { .ISD: ISD::MUL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } }, // pmulld + split
1611 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pmulld
1612 { .ISD: ISD::MUL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 15, .CodeSizeCost: 19, .SizeAndLatencyCost: 20 } },
1613
1614 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // pmuludq + split
1615
1616 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1617 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1618 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1619 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vandps
1620
1621 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1622 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1623 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1624 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vorps
1625
1626 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1627 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1628 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1629 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vxorps
1630
1631 { .ISD: ISD::SUB, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubb + split
1632 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddb + split
1633 { .ISD: ISD::SUB, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubw + split
1634 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddw + split
1635 { .ISD: ISD::SUB, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubd + split
1636 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddd + split
1637 { .ISD: ISD::SUB, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // psubq + split
1638 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // paddq + split
1639 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // psubq
1640 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // paddq
1641
1642 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 21,.CodeSizeCost: 11,.SizeAndLatencyCost: 17 } }, // pblendvb sequence.
1643 { .ISD: ISD::SHL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22,.CodeSizeCost: 27,.SizeAndLatencyCost: 40 } }, // pblendvb sequence + split.
1644 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9,.CodeSizeCost: 11,.SizeAndLatencyCost: 11 } }, // pblendvb sequence.
1645 { .ISD: ISD::SHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 24,.SizeAndLatencyCost: 25 } }, // pblendvb sequence + split.
1646 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // pslld/paddd/cvttps2dq/pmulld
1647 { .ISD: ISD::SHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11,.CodeSizeCost: 12,.SizeAndLatencyCost: 17 } }, // pslld/paddd/cvttps2dq/pmulld + split
1648 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // Shift each lane + blend.
1649 { .ISD: ISD::SHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // Shift each lane + blend + split.
1650
1651 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 27,.CodeSizeCost: 12,.SizeAndLatencyCost: 18 } }, // pblendvb sequence.
1652 { .ISD: ISD::SRL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 23, .LatencyCost: 23,.CodeSizeCost: 30,.SizeAndLatencyCost: 43 } }, // pblendvb sequence + split.
1653 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 14,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1654 { .ISD: ISD::SRL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30,.CodeSizeCost: 31,.SizeAndLatencyCost: 48 } }, // pblendvb sequence + split.
1655 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // Shift each lane + blend.
1656 { .ISD: ISD::SRL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14,.CodeSizeCost: 26,.SizeAndLatencyCost: 34 } }, // Shift each lane + blend + split.
1657 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // Shift each lane + blend.
1658 { .ISD: ISD::SRL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 15 } }, // Shift each lane + blend + split.
1659
1660 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 22,.CodeSizeCost: 24,.SizeAndLatencyCost: 36 } }, // pblendvb sequence.
1661 { .ISD: ISD::SRA, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 45,.CodeSizeCost: 51,.SizeAndLatencyCost: 76 } }, // pblendvb sequence + split.
1662 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 16,.CodeSizeCost: 14,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1663 { .ISD: ISD::SRA, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30,.CodeSizeCost: 31,.SizeAndLatencyCost: 48 } }, // pblendvb sequence + split.
1664 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 7,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // Shift each lane + blend.
1665 { .ISD: ISD::SRA, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14,.CodeSizeCost: 26,.SizeAndLatencyCost: 34 } }, // Shift each lane + blend + split.
1666 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6,.CodeSizeCost: 10,.SizeAndLatencyCost: 14 } }, // Shift each lane + blend.
1667 { .ISD: ISD::SRA, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 22,.SizeAndLatencyCost: 30 } }, // Shift each lane + blend + split.
1668
1669 { .ISD: ISD::FNEG, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1670 { .ISD: ISD::FNEG, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1671
1672 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1673 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1674 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1675 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1676 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1677 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1678
1679 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1680 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1681 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1682 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BDVER2 from http://www.agner.org/
1683 { .ISD: ISD::FSUB, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1684 { .ISD: ISD::FSUB, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BDVER2 from http://www.agner.org/
1685
1686 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1687 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1688 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1689 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // BTVER2 from http://www.agner.org/
1690 { .ISD: ISD::FMUL, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1691 { .ISD: ISD::FMUL, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BTVER2 from http://www.agner.org/
1692
1693 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1694 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1695 { .ISD: ISD::FDIV, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 29, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // SNB from http://www.agner.org/
1696 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1697 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // SNB from http://www.agner.org/
1698 { .ISD: ISD::FDIV, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 45, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // SNB from http://www.agner.org/
1699 };
1700
1701 if (ST->hasAVX())
1702 if (const auto *Entry = CostTableLookup(Table: AVX1CostTable, ISD, Ty: LT.second))
1703 if (auto KindCost = Entry->Cost[CostKind])
1704 return LT.first * *KindCost;
1705
1706 static const CostKindTblEntry SSE42CostTable[] = {
1707 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1708 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1709 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1710 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1711
1712 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1713 { .ISD: ISD::FSUB, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1714 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1715 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1716
1717 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1718 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1719 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1720 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1721
1722 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1723 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1724 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1725 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 22, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
1726
1727 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 10 } } // 3*pmuludq/3*shift/2*add
1728 };
1729
1730 if (ST->hasSSE42())
1731 if (const auto *Entry = CostTableLookup(Table: SSE42CostTable, ISD, Ty: LT.second))
1732 if (auto KindCost = Entry->Cost[CostKind])
1733 return LT.first * *KindCost;
1734
1735 static const CostKindTblEntry SSE41CostTable[] = {
1736 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 24,.CodeSizeCost: 17,.SizeAndLatencyCost: 22 } }, // pblendvb sequence.
1737 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11 } }, // pblendvb sequence.
1738 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 4,.SizeAndLatencyCost: 10 } }, // pslld/paddd/cvttps2dq/pmulld
1739
1740 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 27,.CodeSizeCost: 18,.SizeAndLatencyCost: 24 } }, // pblendvb sequence.
1741 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 26,.CodeSizeCost: 23,.SizeAndLatencyCost: 27 } }, // pblendvb sequence.
1742 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 17,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1743 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1744
1745 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 41,.CodeSizeCost: 30,.SizeAndLatencyCost: 36 } }, // pblendvb sequence.
1746 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 26,.CodeSizeCost: 23,.SizeAndLatencyCost: 27 } }, // pblendvb sequence.
1747 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 17,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1748 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 17, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1749
1750 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } } // pmulld (Nehalem from agner.org)
1751 };
1752
1753 if (ST->hasSSE41())
1754 if (const auto *Entry = CostTableLookup(Table: SSE41CostTable, ISD, Ty: LT.second))
1755 if (auto KindCost = Entry->Cost[CostKind])
1756 return LT.first * *KindCost;
1757
1758 static const CostKindTblEntry SSSE3CostTable[] = {
1759 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 18,.CodeSizeCost: 10,.SizeAndLatencyCost: 12 } }, // 2*pmaddubsw/3*and/psllw/or
1760 };
1761
1762 if (ST->hasSSSE3())
1763 if (const auto *Entry = CostTableLookup(Table: SSSE3CostTable, ISD, Ty: LT.second))
1764 if (auto KindCost = Entry->Cost[CostKind])
1765 return LT.first * *KindCost;
1766
1767 static const CostKindTblEntry SSE2CostTable[] = {
1768 // We don't correctly identify costs of casts because they are marked as
1769 // custom.
1770 { .ISD: ISD::SHL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 21,.CodeSizeCost: 26,.SizeAndLatencyCost: 28 } }, // cmpgtb sequence.
1771 { .ISD: ISD::SHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 27,.CodeSizeCost: 16,.SizeAndLatencyCost: 20 } }, // cmpgtw sequence.
1772 { .ISD: ISD::SHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 19,.CodeSizeCost: 10,.SizeAndLatencyCost: 12 } }, // pslld/paddd/cvttps2dq/pmuludq.
1773 { .ISD: ISD::SHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1774
1775 { .ISD: ISD::SRL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 28,.CodeSizeCost: 27,.SizeAndLatencyCost: 30 } }, // cmpgtb sequence.
1776 { .ISD: ISD::SRL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19,.CodeSizeCost: 31,.SizeAndLatencyCost: 31 } }, // cmpgtw sequence.
1777 { .ISD: ISD::SRL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1778 { .ISD: ISD::SRL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } }, // splat+shuffle sequence.
1779
1780 { .ISD: ISD::SRA, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 30,.CodeSizeCost: 54,.SizeAndLatencyCost: 54 } }, // unpacked cmpgtb sequence.
1781 { .ISD: ISD::SRA, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19,.CodeSizeCost: 31,.SizeAndLatencyCost: 31 } }, // cmpgtw sequence.
1782 { .ISD: ISD::SRA, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 12,.CodeSizeCost: 15,.SizeAndLatencyCost: 19 } }, // Shift each lane + blend.
1783 { .ISD: ISD::SRA, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11,.CodeSizeCost: 12,.SizeAndLatencyCost: 16 } }, // srl/xor/sub splat+shuffle sequence.
1784
1785 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1786 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1787 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1788 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pand
1789
1790 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1791 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1792 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1793 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // por
1794
1795 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1796 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1797 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1798 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pxor
1799
1800 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // paddq
1801 { .ISD: ISD::SUB, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // psubq
1802
1803 { .ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18,.CodeSizeCost: 12,.SizeAndLatencyCost: 12 } }, // 2*unpack/2*pmullw/2*and/pack
1804 { .ISD: ISD::MUL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pmullw
1805 { .ISD: ISD::MUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // 3*pmuludq/4*shuffle
1806 { .ISD: ISD::MUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 10,.CodeSizeCost: 10,.SizeAndLatencyCost: 10 } }, // 3*pmuludq/3*shift/2*add
1807
1808 { .ISD: X86ISD::PMULUDQ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1809
1810 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 23, .LatencyCost: 23, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1811 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 39, .LatencyCost: 39, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1812 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 38, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1813 { .ISD: ISD::FDIV, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 69, .LatencyCost: 69, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1814
1815 { .ISD: ISD::FNEG, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1816 { .ISD: ISD::FNEG, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1817 { .ISD: ISD::FNEG, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1818 { .ISD: ISD::FNEG, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1819
1820 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1821 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1822 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1823
1824 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1825 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1826 { .ISD: ISD::FSUB, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1827
1828 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1829 { .ISD: ISD::FMUL, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium IV from http://www.agner.org/
1830 };
1831
1832 if (ST->hasSSE2())
1833 if (const auto *Entry = CostTableLookup(Table: SSE2CostTable, ISD, Ty: LT.second))
1834 if (auto KindCost = Entry->Cost[CostKind])
1835 return LT.first * *KindCost;
1836
1837 static const CostKindTblEntry SSE1CostTable[] = {
1838 { .ISD: ISD::FDIV, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1839 { .ISD: ISD::FDIV, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 34, .LatencyCost: 48, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1840
1841 { .ISD: ISD::FNEG, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
1842 { .ISD: ISD::FNEG, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
1843
1844 { .ISD: ISD::FADD, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1845 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1846
1847 { .ISD: ISD::FSUB, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1848 { .ISD: ISD::FSUB, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1849
1850 { .ISD: ISD::FMUL, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1851 { .ISD: ISD::FMUL, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Pentium III from http://www.agner.org/
1852 };
1853
1854 if (ST->hasSSE1())
1855 if (const auto *Entry = CostTableLookup(Table: SSE1CostTable, ISD, Ty: LT.second))
1856 if (auto KindCost = Entry->Cost[CostKind])
1857 return LT.first * *KindCost;
1858
1859 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
1860 { .ISD: ISD::ADD, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Core (Merom) from http://www.agner.org/
1861 { .ISD: ISD::SUB, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1 } }, // Core (Merom) from http://www.agner.org/
1862 { .ISD: ISD::MUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
1863 };
1864
1865 if (ST->is64Bit())
1866 if (const auto *Entry = CostTableLookup(Table: X64CostTbl, ISD, Ty: LT.second))
1867 if (auto KindCost = Entry->Cost[CostKind])
1868 return LT.first * *KindCost;
1869
1870 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
1871 { .ISD: ISD::ADD, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1872 { .ISD: ISD::ADD, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1873 { .ISD: ISD::ADD, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1874
1875 { .ISD: ISD::SUB, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1876 { .ISD: ISD::SUB, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1877 { .ISD: ISD::SUB, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1 } }, // Pentium III from http://www.agner.org/
1878
1879 { .ISD: ISD::MUL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1880 { .ISD: ISD::MUL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1881 { .ISD: ISD::MUL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
1882
1883 { .ISD: ISD::FNEG, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // (x87)
1884 { .ISD: ISD::FADD, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1885 { .ISD: ISD::FSUB, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1886 { .ISD: ISD::FMUL, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1887 { .ISD: ISD::FDIV, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 38, .LatencyCost: 38, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // (x87)
1888 };
1889
1890 if (const auto *Entry = CostTableLookup(Table: X86CostTbl, ISD, Ty: LT.second))
1891 if (auto KindCost = Entry->Cost[CostKind])
1892 return LT.first * *KindCost;
1893
1894 // It is not a good idea to vectorize division. We have to scalarize it and
1895 // in the process we will often end up having to spilling regular
1896 // registers. The overhead of division is going to dominate most kernels
1897 // anyways so try hard to prevent vectorization of division - it is
1898 // generally a bad idea. Assume somewhat arbitrarily that we have to be able
1899 // to hide "20 cycles" for each lane.
1900 if (CostKind == TTI::TCK_RecipThroughput && LT.second.isVector() &&
1901 (ISD == ISD::SDIV || ISD == ISD::SREM || ISD == ISD::UDIV ||
1902 ISD == ISD::UREM)) {
1903 InstructionCost ScalarCost =
1904 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind,
1905 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
1906 return 20 * LT.first * LT.second.getVectorNumElements() * ScalarCost;
1907 }
1908
1909 // Handle some basic single instruction code size cases.
1910 if (CostKind == TTI::TCK_CodeSize) {
1911 switch (ISD) {
1912 case ISD::FADD:
1913 case ISD::FSUB:
1914 case ISD::FMUL:
1915 case ISD::FDIV:
1916 case ISD::FNEG:
1917 case ISD::AND:
1918 case ISD::OR:
1919 case ISD::XOR:
1920 return LT.first;
1921 break;
1922 }
1923 }
1924
1925 // Fallback to the default implementation.
1926 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
1927 Args, CtxI);
1928}
1929
1930InstructionCost
1931X86TTIImpl::getAltInstrCost(VectorType *VecTy, unsigned Opcode0,
1932 unsigned Opcode1, const SmallBitVector &OpcodeMask,
1933 TTI::TargetCostKind CostKind,
1934 ArrayRef<const Value *> Scalars) const {
1935 if (isLegalAltInstr(VecTy, Opcode0, Opcode1, OpcodeMask, Scalars))
1936 return TTI::TCC_Basic;
1937 return InstructionCost::getInvalid();
1938}
1939
1940InstructionCost X86TTIImpl::getShuffleCost(
1941 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1942 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1943 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1944 TTI::VectorInstrContext VIC) const {
1945 assert((Mask.empty() || DstTy->isScalableTy() ||
1946 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1947 "Expected the Mask to match the return size if given");
1948 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1949 "Expected the same scalar types");
1950
1951 // 64-bit packed float vectors (v2f32) are widened to type v4f32.
1952 // 64-bit packed integer vectors (v2i32) are widened to type v4i32.
1953 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1954
1955 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1956
1957 // If all args are constant than this will be constant folded away.
1958 if (!Args.empty() &&
1959 all_of(Range&: Args, P: [](const Value *Arg) { return isa<Constant>(Val: Arg); }))
1960 return TTI::TCC_Free;
1961
1962 // Recognize a basic concat_vector shuffle.
1963 if (Kind == TTI::SK_PermuteTwoSrc &&
1964 Mask.size() == (2 * SrcTy->getElementCount().getKnownMinValue()) &&
1965 ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts: Mask.size()))
1966 return getShuffleCost(Kind: TTI::SK_InsertSubvector,
1967 DstTy: VectorType::getDoubleElementsVectorType(VTy: SrcTy),
1968 SrcTy: VectorType::getDoubleElementsVectorType(VTy: SrcTy),
1969 CostKind, Mask, Index: Mask.size() / 2, SubTp: SrcTy);
1970
1971 // Treat Transpose as 2-op shuffles - there's no difference in lowering.
1972 if (Kind == TTI::SK_Transpose)
1973 if (LT.second != MVT::v4f64 && LT.second != MVT::v4i64)
1974 Kind = TTI::SK_PermuteTwoSrc;
1975
1976 if (Kind == TTI::SK_Broadcast) {
1977 // For Broadcasts we are splatting the first element from the first input
1978 // register, so only need to reference that input and all the output
1979 // registers are the same.
1980 LT.first = 1;
1981
1982 // If we're broadcasting a load then AVX/AVX2 can do this for free.
1983 // If many-used-load whose every use is one of a small set of operations
1984 // that SLP can rewrite into a single vector lane, codegen can fold it into
1985 // the free broadcast.
1986 using namespace PatternMatch;
1987 auto IsBroadcastLoadFoldUser = [&](const User *U) {
1988 if (isa<InsertElementInst>(Val: U) && U->getOperand(i: 1) == Args[0])
1989 return true;
1990 if (U->getType()->isVectorTy())
1991 return false;
1992 // Terminators (return/branch/switch/indirectbr/resume/invoke EH)
1993 // and phis carry the value across control flow.
1994 if (const auto *I = dyn_cast<Instruction>(Val: U))
1995 if (I->isTerminator() ||
1996 isa<PHINode, AtomicRMWInst, AtomicCmpXchgInst>(Val: I))
1997 return false;
1998 // Only pure calls can be folded.
1999 if (const auto *CB = dyn_cast<CallBase>(Val: U))
2000 return CB->doesNotAccessMemory() && !CB->mayHaveSideEffects();
2001 return true;
2002 };
2003 auto IsFoldableSLPBroadcastLoad = [&]() {
2004 if (!match(V: Args[0], P: m_Load(Op: m_Value())))
2005 return false;
2006 auto *FVT = dyn_cast<FixedVectorType>(Val: DstTy);
2007 if (!FVT)
2008 return false;
2009 // getNumUses() counts each Use, matching the per-lane broadcast
2010 // accounting (a use like `op %x, %x` consumes two broadcast lanes).
2011 if (Args[0]->getNumUses() != FVT->getNumElements())
2012 return false;
2013 return all_of(Range: Args[0]->users(), P: IsBroadcastLoadFoldUser);
2014 };
2015 if (!Args.empty() &&
2016 (match(V: Args[0], P: m_OneUse(SubPattern: m_Load(Op: m_Value()))) ||
2017 IsFoldableSLPBroadcastLoad()) &&
2018 (ST->hasAVX2() ||
2019 (ST->hasAVX() && LT.second.getScalarSizeInBits() >= 32)))
2020 return TTI::TCC_Free;
2021 }
2022
2023 // Attempt to detect a cheaper inlane shuffle, avoiding 128-bit subvector
2024 // permutation.
2025 // Attempt to detect a shuffle mask with a single defined element.
2026 bool IsInLaneShuffle = false;
2027 bool IsSingleElementMask = false;
2028 if (SrcTy->getPrimitiveSizeInBits() > 0 &&
2029 (SrcTy->getPrimitiveSizeInBits() % 128) == 0 &&
2030 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
2031 Mask.size() == SrcTy->getElementCount().getKnownMinValue()) {
2032 unsigned NumLanes = SrcTy->getPrimitiveSizeInBits() / 128;
2033 unsigned NumEltsPerLane = Mask.size() / NumLanes;
2034 if ((Mask.size() % NumLanes) == 0) {
2035 IsInLaneShuffle = all_of(Range: enumerate(First&: Mask), P: [&](const auto &P) {
2036 return P.value() == PoisonMaskElem ||
2037 ((P.value() % Mask.size()) / NumEltsPerLane) ==
2038 (P.index() / NumEltsPerLane);
2039 });
2040 IsSingleElementMask =
2041 (Mask.size() - 1) == static_cast<unsigned>(count_if(Range&: Mask, P: [](int M) {
2042 return M == PoisonMaskElem;
2043 }));
2044 }
2045 }
2046
2047 // Treat <X x bfloat> shuffles as <X x half>.
2048 if (LT.second.isVectorOf(EltVT: MVT::bf16))
2049 LT.second = LT.second.changeVectorElementType(EltVT: MVT::f16);
2050
2051 // Subvector extractions are free if they start at the beginning of a
2052 // vector and cheap if the subvectors are aligned.
2053 if (Kind == TTI::SK_ExtractSubvector && LT.second.isVector()) {
2054 int NumElts = LT.second.getVectorNumElements();
2055 if ((Index % NumElts) == 0)
2056 return TTI::TCC_Free;
2057 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
2058 if (SubLT.second.isVector()) {
2059 int NumSubElts = SubLT.second.getVectorNumElements();
2060 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2061 return SubLT.first;
2062 // Handle some cases for widening legalization. For now we only handle
2063 // cases where the original subvector was naturally aligned and evenly
2064 // fit in its legalized subvector type.
2065 // FIXME: Remove some of the alignment restrictions.
2066 // FIXME: We can use permq for 64-bit or larger extracts from 256-bit
2067 // vectors.
2068 int OrigSubElts = cast<FixedVectorType>(Val: SubTp)->getNumElements();
2069 if (NumSubElts > OrigSubElts && (Index % OrigSubElts) == 0 &&
2070 (NumSubElts % OrigSubElts) == 0 &&
2071 LT.second.getVectorElementType() ==
2072 SubLT.second.getVectorElementType() &&
2073 LT.second.getVectorElementType().getSizeInBits() ==
2074 SrcTy->getElementType()->getPrimitiveSizeInBits()) {
2075 assert(NumElts >= NumSubElts && NumElts > OrigSubElts &&
2076 "Unexpected number of elements!");
2077 auto *VecTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2078 NumElts: LT.second.getVectorNumElements());
2079 auto *SubTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2080 NumElts: SubLT.second.getVectorNumElements());
2081 int ExtractIndex = alignDown(Value: (Index % NumElts), Align: NumSubElts);
2082 InstructionCost ExtractCost =
2083 getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: VecTy, SrcTy: VecTy, CostKind, Mask: {},
2084 Index: ExtractIndex, SubTp: SubTy);
2085
2086 // If the original size is 32-bits or more, we can use pshufd. Otherwise
2087 // if we have SSSE3 we can use pshufb.
2088 if (SubTp->getPrimitiveSizeInBits() >= 32 || ST->hasSSSE3())
2089 return ExtractCost + 1; // pshufd or pshufb
2090
2091 assert(SubTp->getPrimitiveSizeInBits() == 16 &&
2092 "Unexpected vector size");
2093
2094 return ExtractCost + 2; // worst case pshufhw + pshufd
2095 }
2096 }
2097 // If the extract subvector is not optimal, treat it as single op shuffle.
2098 Kind = TTI::SK_PermuteSingleSrc;
2099 }
2100
2101 // Subvector insertions are cheap if the subvectors are aligned.
2102 // Note that in general, the insertion starting at the beginning of a vector
2103 // isn't free, because we need to preserve the rest of the wide vector,
2104 // but if the destination vector legalizes to the same width as the subvector
2105 // then the insertion will simplify to a (free) register copy.
2106 if (Kind == TTI::SK_InsertSubvector && LT.second.isVector()) {
2107 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: DstTy);
2108 int NumElts = DstLT.second.getVectorNumElements();
2109 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
2110 if (SubLT.second.isVector()) {
2111 int NumSubElts = SubLT.second.getVectorNumElements();
2112 bool MatchingTypes =
2113 NumElts == NumSubElts &&
2114 (SubTp->getElementCount().getKnownMinValue() % NumSubElts) == 0;
2115 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
2116 return MatchingTypes ? TTI::TCC_Free : SubLT.first;
2117 }
2118
2119 // Attempt to match MOVSS (Idx == 0) or INSERTPS pattern. This will have
2120 // been matched by improveShuffleKindFromMask as a SK_InsertSubvector of
2121 // v1f32 (legalised to f32) into a v4f32.
2122 if (LT.first == 1 && LT.second == MVT::v4f32 && SubLT.first == 1 &&
2123 SubLT.second == MVT::f32 && (Index == 0 || ST->hasSSE41()))
2124 return 1;
2125
2126 // If the insertion is the lowest subvector then it will be blended
2127 // otherwise treat it like a 2-op shuffle.
2128 Kind =
2129 (Index == 0 && LT.first == 1) ? TTI::SK_Select : TTI::SK_PermuteTwoSrc;
2130 }
2131
2132 // Handle some common (illegal) sub-vector types as they are often very cheap
2133 // to shuffle even on targets without PSHUFB.
2134 EVT VT = TLI->getValueType(DL, Ty: SrcTy);
2135 if (VT.isSimple() && VT.isVector() && VT.getSizeInBits() < 128 &&
2136 !ST->hasSSSE3()) {
2137 static const CostKindTblEntry SSE2SubVectorShuffleTbl[] = {
2138 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2139 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2140 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2141 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2142 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2143
2144 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2145 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2146 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // punpck/pshuflw/packus
2147 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2148
2149 {.ISD: TTI::SK_Splice, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2150 {.ISD: TTI::SK_Splice, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2151 {.ISD: TTI::SK_Splice, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2152 {.ISD: TTI::SK_Splice, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck+psrldq
2153
2154 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2155 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck/pshuflw
2156 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 7,.LatencyCost: 7,.CodeSizeCost: 7,.SizeAndLatencyCost: 7}}, // punpck/pshuflw
2157 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // punpck/pshuflw
2158 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // punpck
2159
2160 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2161 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pshuflw
2162 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // punpck/pshuflw
2163 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // punpck/pshuflw
2164 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // punpck
2165 };
2166
2167 if (ST->hasSSE2())
2168 if (const auto *Entry =
2169 CostTableLookup(Table: SSE2SubVectorShuffleTbl, ISD: Kind, Ty: VT.getSimpleVT()))
2170 if (auto KindCost = Entry->Cost[CostKind])
2171 return LT.first * *KindCost;
2172 }
2173
2174 // We are going to permute multiple sources and the result will be in multiple
2175 // destinations. Providing an accurate cost only for splits where the element
2176 // type remains the same.
2177 if (LT.first != 1) {
2178 MVT LegalVT = LT.second;
2179 if (LegalVT.isVector() &&
2180 LegalVT.getVectorElementType().getSizeInBits() ==
2181 SrcTy->getElementType()->getPrimitiveSizeInBits() &&
2182 LegalVT.getVectorNumElements() <
2183 cast<FixedVectorType>(Val: SrcTy)->getNumElements()) {
2184 unsigned VecTySize = DL.getTypeStoreSize(Ty: SrcTy);
2185 unsigned LegalVTSize = LegalVT.getStoreSize();
2186 // Number of source vectors after legalization:
2187 unsigned NumOfSrcs = (VecTySize + LegalVTSize - 1) / LegalVTSize;
2188 // Number of destination vectors after legalization:
2189 InstructionCost NumOfDests = LT.first;
2190
2191 auto *SingleOpTy = FixedVectorType::get(ElementType: SrcTy->getElementType(),
2192 NumElts: LegalVT.getVectorNumElements());
2193
2194 if (!Mask.empty() && NumOfDests.isValid()) {
2195 // Try to perform better estimation of the permutation.
2196 // 1. Split the source/destination vectors into real registers.
2197 // 2. Do the mask analysis to identify which real registers are
2198 // permuted. If more than 1 source registers are used for the
2199 // destination register building, the cost for this destination register
2200 // is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
2201 // source register is used, build mask and calculate the cost as a cost
2202 // of PermuteSingleSrc.
2203 // Also, for the single register permute we try to identify if the
2204 // destination register is just a copy of the source register or the
2205 // copy of the previous destination register (the cost is
2206 // TTI::TCC_Basic). If the source register is just reused, the cost for
2207 // this operation is TTI::TCC_Free.
2208 NumOfDests =
2209 getTypeLegalizationCost(
2210 Ty: FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: Mask.size()))
2211 .first;
2212 unsigned E = NumOfDests.getValue();
2213 unsigned NormalizedVF =
2214 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
2215 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
2216 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
2217 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
2218 copy(Range&: Mask, Out: NormalizedMask.begin());
2219 unsigned PrevSrcReg = 0;
2220 ArrayRef<int> PrevRegMask;
2221 InstructionCost Cost = 0;
2222 processShuffleMasks(
2223 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
2224 SingleInputAction: [this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask,
2225 &Cost](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
2226 if (!ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size())) {
2227 // Check if the previous register can be just copied to the next
2228 // one.
2229 if (PrevRegMask.empty() || PrevSrcReg != SrcReg ||
2230 PrevRegMask != RegMask)
2231 Cost +=
2232 getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
2233 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
2234 else
2235 // Just a copy of previous destination register.
2236 Cost += TTI::TCC_Basic;
2237 return;
2238 }
2239 if (SrcReg != DestReg &&
2240 any_of(Range&: RegMask, P: not_equal_to(Arg: PoisonMaskElem))) {
2241 // Just a copy of the source register.
2242 Cost += TTI::TCC_Free;
2243 }
2244 PrevSrcReg = SrcReg;
2245 PrevRegMask = RegMask;
2246 },
2247 ManyInputsAction: [this, SingleOpTy, CostKind,
2248 &Cost](ArrayRef<int> RegMask, unsigned /*Unused*/,
2249 unsigned /*Unused*/, bool /*Unused*/) {
2250 Cost += getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
2251 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
2252 });
2253 return Cost;
2254 }
2255
2256 InstructionCost NumOfShuffles = (NumOfSrcs - 1) * NumOfDests;
2257 return NumOfShuffles * getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
2258 SrcTy: SingleOpTy, CostKind, Mask: {}, Index: 0,
2259 SubTp: nullptr);
2260 }
2261
2262 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2263 SubTp);
2264 }
2265
2266 // If we're just moving a single element around (probably as an alternative to
2267 // extracting it), we can assume this is cheap.
2268 if (LT.first == 1 && IsInLaneShuffle && IsSingleElementMask)
2269 return TTI::TCC_Basic;
2270
2271 static const CostKindTblEntry AVX512VBMIShuffleTbl[] = {
2272 { .ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2273 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2274 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2275 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermb
2276 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2b
2277 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2b
2278 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } } // vpermt2b
2279 };
2280
2281 if (ST->hasVBMI())
2282 if (const auto *Entry =
2283 CostTableLookup(Table: AVX512VBMIShuffleTbl, ISD: Kind, Ty: LT.second))
2284 if (auto KindCost = Entry->Cost[CostKind])
2285 return LT.first * *KindCost;
2286
2287 static const CostKindTblEntry AVX512BWInLaneShuffleTbl[] = {
2288 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*vpshufb + vpor
2289 };
2290
2291 if (IsInLaneShuffle && ST->hasBWI())
2292 if (const auto *Entry =
2293 CostTableLookup(Table: AVX512BWInLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2294 if (auto KindCost = Entry->Cost[CostKind])
2295 return LT.first * *KindCost;
2296
2297 static const CostKindTblEntry AVX512BWShuffleTbl[] = {
2298 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2299 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2300 { .ISD: TTI::SK_Broadcast, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2301
2302 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermw
2303 { .ISD: TTI::SK_Reverse, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermw
2304 { .ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2305 { .ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2306 { .ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // pshufb + vshufi64x2
2307
2308 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2309 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2310 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2311 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermw
2312 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } }, // extend to v32i16
2313
2314 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2315 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32f16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2316 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2317 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpermt2w
2318 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 19, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } }, // 6 * v32i8 + 1
2319
2320 { .ISD: TTI::SK_Select, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmw
2321 { .ISD: TTI::SK_Select, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmb
2322
2323 { .ISD: TTI::SK_Splice, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2324 { .ISD: TTI::SK_Splice, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2325 { .ISD: TTI::SK_Splice, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vshufi64x2 + palignr
2326 };
2327
2328 if (ST->hasBWI())
2329 if (const auto *Entry =
2330 CostTableLookup(Table: AVX512BWShuffleTbl, ISD: Kind, Ty: LT.second))
2331 if (auto KindCost = Entry->Cost[CostKind])
2332 return LT.first * *KindCost;
2333
2334 static const CostKindTblEntry AVX512InLaneShuffleTbl[] = {
2335 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2336 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2337 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2338 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2339 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2340 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2341 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2342 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2343 };
2344
2345 if (IsInLaneShuffle && ST->hasAVX512())
2346 if (const auto *Entry =
2347 CostTableLookup(Table: AVX512InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2348 if (auto KindCost = Entry->Cost[CostKind])
2349 return LT.first * *KindCost;
2350
2351 static const CostKindTblEntry AVX512ShuffleTbl[] = {
2352 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastsd
2353 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastsd
2354 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastss
2355 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vbroadcastss
2356 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastq
2357 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastq
2358 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastd
2359 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastd
2360 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2361 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2362 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2363 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2364 {.ISD: TTI::SK_Broadcast, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2365 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 }}, // vpbroadcastb
2366
2367 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermpd
2368 {.ISD: TTI::SK_Reverse, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermps
2369 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermq
2370 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // vpermd
2371 {.ISD: TTI::SK_Reverse, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2372 {.ISD: TTI::SK_Reverse, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2373 {.ISD: TTI::SK_Reverse, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // per mca
2374
2375 {.ISD: TTI::SK_Splice, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2376 {.ISD: TTI::SK_Splice, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2377 {.ISD: TTI::SK_Splice, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2378 {.ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2379 {.ISD: TTI::SK_Splice, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2380 {.ISD: TTI::SK_Splice, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2381 {.ISD: TTI::SK_Splice, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2382 {.ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpalignd
2383 {.ISD: TTI::SK_Splice, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2384 {.ISD: TTI::SK_Splice, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2385 {.ISD: TTI::SK_Splice, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // split + palignr
2386
2387 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2388 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2389 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2390 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2391 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2392 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2393 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2394 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2395 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2396 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2397 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2398 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2399 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pshufb
2400
2401 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2pd
2402 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2ps
2403 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2q
2404 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2d
2405 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2pd
2406 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2ps
2407 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2q
2408 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermt2d
2409 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2410 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2411 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2412 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2413
2414 // FIXME: This just applies the type legalization cost rules above
2415 // assuming these completely split.
2416 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2417 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2418 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
2419 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2420 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2421 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
2422
2423 {.ISD: TTI::SK_Select, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2424 {.ISD: TTI::SK_Select, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2425 {.ISD: TTI::SK_Select, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
2426 {.ISD: TTI::SK_Select, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmpd
2427 {.ISD: TTI::SK_Select, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmps
2428 {.ISD: TTI::SK_Select, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmq
2429 {.ISD: TTI::SK_Select, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vblendmd
2430 };
2431
2432 if (ST->hasAVX512())
2433 if (const auto *Entry = CostTableLookup(Table: AVX512ShuffleTbl, ISD: Kind, Ty: LT.second))
2434 if (auto KindCost = Entry->Cost[CostKind])
2435 return LT.first * *KindCost;
2436
2437 static const CostKindTblEntry AVX2InLaneShuffleTbl[] = {
2438 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2439 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2440 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2441
2442 { .ISD: TTI::SK_Transpose, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2443 { .ISD: TTI::SK_Transpose, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2444
2445 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufpd + vblendpd
2446 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufps + vblendps
2447 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufd + vpblendd
2448 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufd + vpblendd
2449 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2450 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2451 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpshufb + vpor
2452 };
2453
2454 if (IsInLaneShuffle && ST->hasAVX2())
2455 if (const auto *Entry =
2456 CostTableLookup(Table: AVX2InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2457 if (auto KindCost = Entry->Cost[CostKind])
2458 return LT.first * *KindCost;
2459
2460 static const CostKindTblEntry AVX2ShuffleTbl[] = {
2461 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vbroadcastpd
2462 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vbroadcastps
2463 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastq
2464 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastd
2465 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastw
2466 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2467 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastw
2468 { .ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastw
2469 { .ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpbroadcastb
2470 { .ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpbroadcastb
2471
2472 { .ISD: TTI::SK_Reverse, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpermpd
2473 { .ISD: TTI::SK_Reverse, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermps
2474 { .ISD: TTI::SK_Reverse, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vpermq
2475 { .ISD: TTI::SK_Reverse, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vpermd
2476 { .ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2477 { .ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2478 { .ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // vperm2i128 + pshufb
2479
2480 { .ISD: TTI::SK_Select, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2481 { .ISD: TTI::SK_Select, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2482 { .ISD: TTI::SK_Select, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpblendvb
2483
2484 { .ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2485 { .ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2486 { .ISD: TTI::SK_Splice, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2487 { .ISD: TTI::SK_Splice, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2488 { .ISD: TTI::SK_Splice, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2i128 + vpalignr
2489
2490 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermpd
2491 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermps
2492 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermq
2493 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermd
2494 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2495 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2496 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
2497
2498 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermpd + vblendpd
2499 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermps + vblendps
2500 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermq + vpblendd
2501 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // 2*vpermd + vpblendd
2502 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2503 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2504 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
2505 };
2506
2507 if (ST->hasAVX2())
2508 if (const auto *Entry = CostTableLookup(Table: AVX2ShuffleTbl, ISD: Kind, Ty: LT.second))
2509 if (auto KindCost = Entry->Cost[CostKind])
2510 return LT.first * *KindCost;
2511
2512 static const CostKindTblEntry XOPShuffleTbl[] = {
2513 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2pd
2514 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2ps
2515 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2pd
2516 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vperm2f128 + vpermil2ps
2517 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16,.Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*vpperm
2518 // + vinsertf128
2519 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*vpperm
2520 // + vinsertf128
2521
2522 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 6*vpperm
2523 // + vinsertf128
2524
2525 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpperm
2526 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 6*vpperm
2527 // + vinsertf128
2528 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpperm
2529 };
2530
2531 if (ST->hasXOP())
2532 if (const auto *Entry = CostTableLookup(Table: XOPShuffleTbl, ISD: Kind, Ty: LT.second))
2533 if (auto KindCost = Entry->Cost[CostKind])
2534 return LT.first * *KindCost;
2535
2536 static const CostKindTblEntry AVX1InLaneShuffleTbl[] = {
2537 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilpd
2538 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilpd
2539 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilps
2540 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilps
2541
2542 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2543 // + vpor + vinsertf128
2544 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2545 // + vpor + vinsertf128
2546 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } }, // vextractf128 + 2*pshufb
2547 // + vpor + vinsertf128
2548
2549 { .ISD: TTI::SK_Transpose, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2550 { .ISD: TTI::SK_Transpose, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vshufpd/vunpck
2551
2552 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufpd + vblendpd
2553 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vshufps + vblendps
2554 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpermilpd + vblendpd
2555 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // 2*vpermilps + vblendps
2556 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2557 // + 2*vpor + vinsertf128
2558 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2559 // + 2*vpor + vinsertf128
2560 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } }, // 2*vextractf128 + 4*pshufb
2561 // + 2*vpor + vinsertf128
2562 };
2563
2564 if (IsInLaneShuffle && ST->hasAVX())
2565 if (const auto *Entry =
2566 CostTableLookup(Table: AVX1InLaneShuffleTbl, ISD: Kind, Ty: LT.second))
2567 if (auto KindCost = Entry->Cost[CostKind])
2568 return LT.first * *KindCost;
2569
2570 static const CostKindTblEntry AVX1ShuffleTbl[] = {
2571 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilpd
2572 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilps
2573 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilpd
2574 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 2,.SizeAndLatencyCost: 3}}, // vperm2f128 + vpermilps
2575 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 4}}, // vpshuflw + vpshufd + vinsertf128
2576 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 4}}, // vpshuflw + vpshufd + vinsertf128
2577 {.ISD: TTI::SK_Broadcast, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 4,.CodeSizeCost: 3,.SizeAndLatencyCost: 6}}, // vpshufb + vinsertf128
2578
2579 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 6,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vpermilpd
2580 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 7,.CodeSizeCost: 2,.SizeAndLatencyCost: 4}}, // vperm2f128 + vpermilps
2581 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 6,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vpermilpd
2582 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 7,.CodeSizeCost: 2,.SizeAndLatencyCost: 4}}, // vperm2f128 + vpermilps
2583 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2584 // + vinsertf128
2585 {.ISD: TTI::SK_Reverse, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2586 // + vinsertf128
2587 {.ISD: TTI::SK_Reverse, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 9,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // vextractf128 + 2*pshufb
2588 // + vinsertf128
2589
2590 {.ISD: TTI::SK_Select, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendpd
2591 {.ISD: TTI::SK_Select, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendpd
2592 {.ISD: TTI::SK_Select, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendps
2593 {.ISD: TTI::SK_Select, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // vblendps
2594 {.ISD: TTI::SK_Select, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2595 {.ISD: TTI::SK_Select, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2596 {.ISD: TTI::SK_Select, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // vpand + vpandn + vpor
2597
2598 {.ISD: TTI::SK_Splice, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + shufpd
2599 {.ISD: TTI::SK_Splice, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + shufpd
2600 {.ISD: TTI::SK_Splice, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2601 {.ISD: TTI::SK_Splice, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2602 {.ISD: TTI::SK_Splice, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2603 {.ISD: TTI::SK_Splice, .Type: MVT::v16f16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2604 {.ISD: TTI::SK_Splice, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 5,.CodeSizeCost: 5,.SizeAndLatencyCost: 5}}, // 2*vperm2f128 + 2*vpalignr + vinsertf128
2605
2606 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vshufpd
2607 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2}}, // vperm2f128 + vshufpd
2608 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2609 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2610 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i16,.Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2611 // + 2*por + vinsertf128
2612 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16f16,.Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2613 // + 2*por + vinsertf128
2614 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 8,.LatencyCost: 8,.CodeSizeCost: 8,.SizeAndLatencyCost: 8}}, // vextractf128 + 4*pshufb
2615 // + 2*por + vinsertf128
2616
2617 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // 2*vperm2f128 + vshufpd
2618 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 3,.CodeSizeCost: 3,.SizeAndLatencyCost: 3}}, // 2*vperm2f128 + vshufpd
2619 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2620 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 4,.CodeSizeCost: 4,.SizeAndLatencyCost: 4}}, // 2*vperm2f128 + 2*vshufps
2621 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i16,.Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2622 // + 4*por + vinsertf128
2623 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16f16,.Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2624 // + 4*por + vinsertf128
2625 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 15,.LatencyCost: 15,.CodeSizeCost: 15,.SizeAndLatencyCost: 15}}, // 2*vextractf128 + 8*pshufb
2626 // + 4*por + vinsertf128
2627 };
2628
2629 if (ST->hasAVX())
2630 if (const auto *Entry = CostTableLookup(Table: AVX1ShuffleTbl, ISD: Kind, Ty: LT.second))
2631 if (auto KindCost = Entry->Cost[CostKind])
2632 return LT.first * *KindCost;
2633
2634 static const CostKindTblEntry SSE41ShuffleTbl[] = {
2635 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2636 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // movsd
2637 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2638 {.ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // blendps
2639 {.ISD: TTI::SK_Select, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2640 {.ISD: TTI::SK_Select, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}}, // pblendw
2641 {.ISD: TTI::SK_Select, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1}} // pblendvb
2642 };
2643
2644 if (ST->hasSSE41())
2645 if (const auto *Entry = CostTableLookup(Table: SSE41ShuffleTbl, ISD: Kind, Ty: LT.second))
2646 if (auto KindCost = Entry->Cost[CostKind])
2647 return LT.first * *KindCost;
2648
2649 static const CostKindTblEntry SSSE3ShuffleTbl[] = {
2650 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2651 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2652 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshufb
2653
2654 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2655 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2656 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2}}, // pshufb
2657
2658 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2659 {.ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2660 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2661 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2662 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // palignr
2663
2664 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2665 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2666 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufb
2667
2668 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2669 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2670 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // 2*pshufb + por
2671 };
2672
2673 if (ST->hasSSSE3())
2674 if (const auto *Entry = CostTableLookup(Table: SSSE3ShuffleTbl, ISD: Kind, Ty: LT.second))
2675 if (auto KindCost = Entry->Cost[CostKind])
2676 return LT.first * *KindCost;
2677
2678 static const CostKindTblEntry SSE2ShuffleTbl[] = {
2679 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2680 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2681 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2682 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshuflw + pshufd
2683 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // pshuflw + pshufd
2684 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 4}}, // unpck + pshuflw + pshufd
2685
2686 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2687 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2688 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2689 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pshuflw + pshufhw + pshufd
2690 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pshuflw + pshufhw + pshufd
2691 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 6,.CodeSizeCost: 11,.SizeAndLatencyCost: 11}}, // 2*pshuflw + 2*pshufhw
2692 // + 2*pshufd + 2*unpck + packus
2693
2694 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // movsd
2695 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // movsd
2696 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*shufps
2697 {.ISD: TTI::SK_Select, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2698 {.ISD: TTI::SK_Select, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2699 {.ISD: TTI::SK_Select, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // pand + pandn + por
2700
2701 {.ISD: TTI::SK_Splice, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2702 {.ISD: TTI::SK_Splice, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2703 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*{unpck,movsd,pshufd}
2704 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2705 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2706 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}}, // psrldq + psrlldq + por
2707
2708 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2709 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2710 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // pshufd
2711 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}}, // 2*pshuflw + 2*pshufhw
2712 // + pshufd/unpck
2713 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}}, // 2*pshuflw + 2*pshufhw
2714 // + pshufd/unpck
2715 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 8, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 10}}, // 2*pshuflw + 2*pshufhw
2716 // + 2*pshufd + 2*unpck + 2*packus
2717
2718 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2719 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1}}, // shufpd
2720 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}}, // 2*{unpck,movsd,pshufd}
2721 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}}, // blend+permute
2722 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v8f16, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}}, // blend+permute
2723 {.ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 13, .CodeSizeCost: 13, .SizeAndLatencyCost: 13}}, // blend+permute
2724 };
2725
2726 static const CostTblEntry SSE3BroadcastLoadTbl[] = {
2727 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: 0}, // broadcast handled by movddup
2728 };
2729
2730 if (ST->hasSSE2()) {
2731 bool IsLoad =
2732 llvm::any_of(Range&: Args, P: [](const auto &V) { return isa<LoadInst>(V); });
2733 if (ST->hasSSE3() && IsLoad)
2734 if (const auto *Entry =
2735 CostTableLookup(Table: SSE3BroadcastLoadTbl, ISD: Kind, Ty: LT.second)) {
2736 assert(isLegalBroadcastLoad(SrcTy->getElementType(),
2737 LT.second.getVectorElementCount()) &&
2738 "Table entry missing from isLegalBroadcastLoad()");
2739 return LT.first * Entry->Cost;
2740 }
2741
2742 if (const auto *Entry = CostTableLookup(Table: SSE2ShuffleTbl, ISD: Kind, Ty: LT.second))
2743 if (auto KindCost = Entry->Cost[CostKind])
2744 return LT.first * *KindCost;
2745 }
2746
2747 static const CostKindTblEntry SSE1ShuffleTbl[] = {
2748 { .ISD: TTI::SK_Broadcast, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2749 { .ISD: TTI::SK_Reverse, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2750 { .ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2751 { .ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2752 { .ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 1,.LatencyCost: 1,.CodeSizeCost: 1,.SizeAndLatencyCost: 1} }, // shufps
2753 { .ISD: TTI::SK_PermuteTwoSrc, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 2,.CodeSizeCost: 2,.SizeAndLatencyCost: 2} }, // 2*shufps
2754 };
2755
2756 if (ST->hasSSE1()) {
2757 if (LT.first == 1 && LT.second == MVT::v4f32 && Mask.size() == 4) {
2758 // SHUFPS: both pairs must come from the same source register.
2759 auto MatchSHUFPS = [](int X, int Y) {
2760 return X < 0 || Y < 0 || ((X & 4) == (Y & 4));
2761 };
2762 if (MatchSHUFPS(Mask[0], Mask[1]) && MatchSHUFPS(Mask[2], Mask[3]))
2763 return 1;
2764 }
2765 if (const auto *Entry = CostTableLookup(Table: SSE1ShuffleTbl, ISD: Kind, Ty: LT.second))
2766 if (auto KindCost = Entry->Cost[CostKind])
2767 return LT.first * *KindCost;
2768 }
2769
2770 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
2771 SubTp);
2772}
2773
2774InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
2775 Type *Src,
2776 TTI::CastContextHint CCH,
2777 TTI::TargetCostKind CostKind,
2778 const Instruction *I) const {
2779 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2780 assert(ISD && "Invalid opcode");
2781
2782 // A narrow (i8/i16) zero-extension used as a GEP *index* can be folded into
2783 // the addressing mode of the consuming memory op, but only if the source is
2784 // already materialised zero-extended in a full register. X86's SIB form
2785 // [base + index*scale + disp] reads the index at full width and does NOT
2786 // zero-extend a narrow index (unlike AArch64's uxtw-extended addressing), so
2787 // a "dirty" narrow source (e.g. an i16 add result used only as an index)
2788 // still needs a dedicated movzx and is not free. Price it as free only with
2789 // positive evidence that no movzx is required.
2790 if (ISD == ISD::ZERO_EXTEND && I && I->hasOneUse() && Src->isIntegerTy() &&
2791 Src->getScalarSizeInBits() < 32) {
2792 const Use &U = *I->use_begin();
2793 if (isa<GetElementPtrInst>(Val: U.getUser()) &&
2794 U.getOperandNo() != GetElementPtrInst::getPointerOperandIndex()) {
2795 const Value *Op = I->getOperand(i: 0);
2796 // Clean sources: an extending load, a zeroext argument, or a value whose
2797 // high bits are provably zero (e.g. from a shift/mask). These mirror the
2798 // proof-based reasoning the middle end uses elsewhere (ValueTracking and
2799 // InstCombine's canEvaluateZExtd); we intentionally do NOT treat a merely
2800 // multiply-used operand as clean, since that is a guess rather than
2801 // proof.
2802 if (isa<LoadInst>(Val: Op))
2803 return TTI::TCC_Free;
2804 if (const auto *A = dyn_cast<Argument>(Val: Op))
2805 if (A->hasAttribute(Kind: Attribute::ZExt))
2806 return TTI::TCC_Free;
2807 if (computeKnownBits(V: Op, DL: I->getDataLayout(), /*AC=*/nullptr, CtxI: I)
2808 .countMinLeadingZeros() > 0)
2809 return TTI::TCC_Free;
2810 }
2811 }
2812
2813 // The cost tables include both specific, custom (non-legal) src/dst type
2814 // conversions and generic, legalized types. We test for customs first, before
2815 // falling back to legalization.
2816 // FIXME: Need a better design of the cost table to handle non-simple types of
2817 // potential massive combinations (elem_num x src_type x dst_type).
2818 static const TypeConversionCostKindTblEntry AVX512BWConversionTbl[]{
2819 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2820 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2821
2822 // Mask sign extend has an instruction.
2823 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2824 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2825 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2826 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2827 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2828 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2829 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2830 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2831 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2832 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2833 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2834 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2835 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2836 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2837 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2838 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v64i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2839 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2840
2841 // Mask zero extend is a sext + shift.
2842 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2843 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2844 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2845 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2846 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2847 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2848 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2849 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2850 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2851 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2852 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2853 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2854 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2855 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2856 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2857 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v64i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2858 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2859
2860 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2861 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2862 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2863 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2864 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2865 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2866 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2867 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2868 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2869 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2870 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2871 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2872 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2873 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2874 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2875 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2876 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2877
2878 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2879 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // widen to zmm
2880 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2881 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2882 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
2883 };
2884
2885 static const TypeConversionCostKindTblEntry AVX512DQConversionTbl[] = {
2886 // Mask sign extend has an instruction.
2887 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2888 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2889 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2890 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2891 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2892 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2893 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2894 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2895
2896 // Mask zero extend is a sext + shift.
2897 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2898 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2899 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2900 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2901 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2902 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2903 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2904 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1, } },
2905
2906 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2907 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2908 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2909 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2910 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2911 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2912 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2913 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2914
2915 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2916 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2917
2918 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2919 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2920
2921 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2922 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i64, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2923
2924 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2925 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i64, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2926 };
2927
2928 // TODO: For AVX512DQ + AVX512VL, we also have cheap casts for 128-bit and
2929 // 256-bit wide vectors.
2930
2931 static const TypeConversionCostKindTblEntry AVX512FConversionTbl[] = {
2932 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2933 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2934 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v16f64, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // 2*vcvtps2pd+vextractf64x4
2935 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v16f32, .Src: MVT::v16f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps
2936 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
2937 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2938 { .ISD: ISD::FP_ROUND, .Dst: MVT::v16f16, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtps2ph
2939
2940 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2941 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2942 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2943 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2944 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2945 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2946 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
2947 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
2948 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2949 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2950 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpslld+vptestmd
2951 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
2952 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpsllq+vptestmq
2953 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpsllq+vptestmq
2954 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
2955 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2956 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2957 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2958 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2959 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdb
2960 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdw
2961 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovdw
2962 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2963 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpshufb
2964 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2965 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2966 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2967 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
2968 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2969 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2970 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i16, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
2971 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqd
2972 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpmovqd
2973 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },// 2*vpmovqd+concat+vpmovdb
2974
2975 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // extend to v16i32
2976 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2977 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i8, .Src: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2978
2979 // Sign extend is zmm vpternlogd+vptruncdb.
2980 // Zero extend is zmm broadcast load+vptruncdw.
2981 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2982 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2983 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2984 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2985 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2986 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2987 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2988 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2989
2990 // Sign extend is zmm vpternlogd+vptruncdw.
2991 // Zero extend is zmm vpternlogd+vptruncdw+vpsrlw.
2992 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2993 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2994 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2995 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2996 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2997 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2998 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
2999 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3000
3001 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3002 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3003 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3004 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3005 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd
3006 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogd+psrld
3007 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq
3008 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq+psrlq
3009 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq
3010 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // zmm vpternlogq+psrlq
3011
3012 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3013 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3014 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3015 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3016
3017 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3018 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3019 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3020 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3021 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3022 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3023 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3024 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3025 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3026 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3027
3028 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // FIXME: May not be right
3029 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i16, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // FIXME: May not be right
3030
3031 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3032 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3033 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3034 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3035 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3036 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3037 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3038 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3039
3040 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3041 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3042 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3043 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3044 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3045 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3046 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3047 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3048 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i64, .Cost: {.RecipThroughputCost: 26, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3049 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3050
3051 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3052 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3053 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v32f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3054 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v64i8, .Src: MVT::v64f32, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3055 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v64i8, .Src: MVT::v64f64, .Cost: {.RecipThroughputCost: 31, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3056 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3057 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3058 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i16, .Src: MVT::v32f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3059 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i16, .Src: MVT::v32f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3060 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3061 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i32, .Src: MVT::v16f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3062
3063 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3064 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3065 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i8, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3066 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i32, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3067 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3068 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3069 };
3070
3071 static const TypeConversionCostKindTblEntry AVX512BWVLConversionTbl[] {
3072 // Mask sign extend has an instruction.
3073 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3074 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3075 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3076 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3077 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3078 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3079 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3080 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3081 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3082 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3083 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3084 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3085 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3086 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3087 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3088 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3089 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3090
3091 // Mask zero extend is a sext + shift.
3092 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3093 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3094 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3095 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3096 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3097 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3098 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3099 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3100 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3101 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3102 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3103 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3104 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3105 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3106 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v32i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3107 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v32i8, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3108 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v64i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3109
3110 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3111 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3112 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3113 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3114 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3115 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3116 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3117 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3118 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3119 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3120 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3121 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3122 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3123 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3124 { .ISD: ISD::TRUNCATE, .Dst: MVT::v32i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3125 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3126 { .ISD: ISD::TRUNCATE, .Dst: MVT::v64i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3127
3128 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3129 };
3130
3131 static const TypeConversionCostKindTblEntry AVX512DQVLConversionTbl[] = {
3132 // Mask sign extend has an instruction.
3133 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3134 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3135 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3136 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3137 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3138 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3139 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3140 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3141
3142 // Mask zero extend is a sext + shift.
3143 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3144 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3145 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3146 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3147 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3148 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3149 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3150 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3151
3152 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3153 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3154 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3155 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3156 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3157 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3158 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3159 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3160
3161 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3162 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3163 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3164 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3165
3166 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3167 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3168 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3169 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3170
3171 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3172 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3173 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3174 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i64, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3175
3176 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3177 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3178 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3179 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i64, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3180 };
3181
3182 static const TypeConversionCostKindTblEntry AVX512VLConversionTbl[] = {
3183 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3184 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3185 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpslld+vptestmd
3186 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // split+2*v8i8
3187 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3188 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3189 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sext+vpsllq+vptestmq
3190 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // split+2*v8i16
3191 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3192 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3193 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3194 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpslld+vptestmd
3195 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
3196 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpsllq+vptestmq
3197 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqd
3198 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqb
3199 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovqw
3200 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpmovwb
3201
3202 // sign extend is vpcmpeq+maskedmove+vpmovdw+vpacksswb
3203 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw+vpackuswb
3204 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3205 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i8, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3206 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3207 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i8, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3208 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3209 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i8, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3210 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3211 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i8, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3212
3213 // sign extend is vpcmpeq+maskedmove+vpmovdw
3214 // zero extend is vpcmpeq+maskedmove+vpmovdw+vpsrlw
3215 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3216 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i16, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3217 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3218 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i16, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3219 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3220 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3221 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3222 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3223
3224 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3225 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i32, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3226 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3227 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3228 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3229 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3230 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd
3231 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogd+psrld
3232
3233 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3234 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3235 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq
3236 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpternlogq+psrlq
3237
3238 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3239 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3240 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3241 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3242 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3243 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3244 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3245 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3246 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3247 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3248 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3249 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3250
3251 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3252 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3253 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3254 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3255
3256 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3257 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3258 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3259 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3260 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3261 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3262 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3263 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3264 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3265 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3266 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3267 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3268 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3269
3270 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3271 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3272 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v32f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3273
3274 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3275 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3276 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3277 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3278 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3279 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3280 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3281 };
3282
3283 static const TypeConversionCostKindTblEntry AVX2ConversionTbl[] = {
3284 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3285 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3286 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3287 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3288 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3289 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3290
3291 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3292 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3293 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3294 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3295 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3296 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3297 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3298 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3299 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3300 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3301 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3302 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3303 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3304 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3305
3306 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3307
3308 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3309 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3310 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3311 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3312 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3313 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3314 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3315 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3316 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3317 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3318 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3319 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3320
3321 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3322 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3323
3324 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3325 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3326 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3327 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3328
3329 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3330 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3331 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3332 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3333 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3334 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3335 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3336 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3337
3338 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3339 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3340 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3341 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3342 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3343 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3344 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3345
3346 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3347 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3348 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3349 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3350 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3351 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3352 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3353 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3354 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3355 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3356 };
3357
3358 static const TypeConversionCostKindTblEntry AVXConversionTbl[] = {
3359 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3360 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3361 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3362 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3363 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3364 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3365
3366 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3367 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3368 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3369 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3370 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3371 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3372 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3373 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3374 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3375 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3376 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3377 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3378
3379 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3380 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3381 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3382 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3383 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3384
3385 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3386 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3387 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // and+extract+packuswb
3388 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3389 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3390 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3391 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // and+extract+2*packusdw
3392 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3393
3394 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3395 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3396 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3397 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3398 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3399 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3400 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3401 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3402 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3403 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3404 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3405 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3406
3407 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3408 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3409 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i1, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3410 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3411 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3412 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3413 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3414 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3415 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3416 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3417 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3418 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3419 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f64, .Src: MVT::v8i32, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3420 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 11, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3421 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 18, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3422 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3423 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3424
3425 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3426 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3427 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3428 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v32i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3429 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3430 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3431 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3432 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3433 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3434 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3435 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3436
3437 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3438 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3439 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v32i8, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3440 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v32i8, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3441 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3442 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3443 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3444 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3445 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3446 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3447 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3448 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3449 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3450
3451 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3452 { .ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3453 };
3454
3455 static const TypeConversionCostKindTblEntry SSE41ConversionTbl[] = {
3456 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3457 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3458 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3459 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3460 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3461 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3462 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3463 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3464 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3465 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3466 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3467 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3468
3469 // These truncates end up widening elements.
3470 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZBQ
3471 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZWQ
3472 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PMOVXZBD
3473
3474 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3475 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3476 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3477
3478 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3479 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3480 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3481 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3482 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3483 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3484 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3485 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3486 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3487 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3488 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3489
3490 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3491 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3492 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3493 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3494 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3495 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3496 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3497 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3498 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3499 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3500 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3501 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 12, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3502 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i64, .Cost: {.RecipThroughputCost: 22, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3503 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3504
3505 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3506 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3507 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3508 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3509 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3510 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3511 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3512 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3513 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3514 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3515
3516 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3517 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3518 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3519 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3520 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3521 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3522 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3523 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3524 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3525 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3526 };
3527
3528 static const TypeConversionCostKindTblEntry SSE2ConversionTbl[] = {
3529 // These are somewhat magic numbers justified by comparing the
3530 // output of llvm-mca for our various supported scheduler models
3531 // and basing it off the worst case scenario.
3532 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3533 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3534 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3535 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3536 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3537 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3538 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3539 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3540 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3541 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3542 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3543 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3544
3545 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3546 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3547 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3548 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3549 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3550 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3551 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3552 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3553 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3554 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3555 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3556 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3557 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v2i64, .Cost: {.RecipThroughputCost: 18, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3558
3559 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3560 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3561 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3562 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3563 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3564 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3565 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3566 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3567 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3568 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3569
3570 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3571 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3572 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3573 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: {.RecipThroughputCost: 15, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3574 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3575 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3576 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3577 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3578 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3579 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v2f64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3580
3581 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3582 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3583 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3584 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3585 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3586 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3587 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3588 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3589 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3590 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3591 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3592 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3593
3594 // These truncates are really widening elements.
3595 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD
3596 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLWD+DQ
3597 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i1, .Src: MVT::v2i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW+WD+PSHUFD
3598 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLWD
3599 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i1, .Src: MVT::v4i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW+WD
3600 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i1, .Src: MVT::v8i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PUNPCKLBW
3601
3602 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+PACKUSWB
3603 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3604 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+2*PACKUSWB
3605 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3606 { .ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3607 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3608 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3609 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: {.RecipThroughputCost: 10, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3610 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PAND+3*PACKUSWB
3611 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD+PSHUFLW
3612 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // PSHUFD
3613 };
3614
3615 static const TypeConversionCostKindTblEntry F16ConversionTbl[] = {
3616 { .ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3617 { .ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3618 { .ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3619 { .ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3620 { .ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
3621 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3622 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3623 { .ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vcvtph2ps+vcvtps2pd
3624 };
3625
3626 // Attempt to map directly to (simple) MVT types to let us match custom entries.
3627 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
3628 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
3629
3630 // If we're sign-extending a vector comparison result back to the comparison
3631 // width, this will be free without AVX512 (or for 8/16-bit types without
3632 // BWI).
3633 if (!ST->hasAVX512() || (!ST->hasBWI() && DstTy.getScalarSizeInBits() < 32)) {
3634 if (I && Opcode == Instruction::CastOps::SExt &&
3635 SrcTy.isFixedLengthVectorOf(EltVT: MVT::i1)) {
3636 if (auto *CmpI = dyn_cast<CmpInst>(Val: I->getOperand(i: 0))) {
3637 Type *CmpTy = CmpI->getOperand(i_nocapture: 0)->getType();
3638 if (CmpTy->getScalarSizeInBits() == DstTy.getScalarSizeInBits())
3639 return TTI::TCC_Free;
3640 }
3641 }
3642 }
3643
3644 // The function getSimpleVT only handles simple value types.
3645 if (SrcTy.isSimple() && DstTy.isSimple()) {
3646 MVT SimpleSrcTy = SrcTy.getSimpleVT();
3647 MVT SimpleDstTy = DstTy.getSimpleVT();
3648
3649 if (ST->useAVX512Regs()) {
3650 if (ST->hasBWI())
3651 if (const auto *Entry = ConvertCostTableLookup(
3652 Table: AVX512BWConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3653 if (auto KindCost = Entry->Cost[CostKind])
3654 return *KindCost;
3655
3656 if (ST->hasDQI())
3657 if (const auto *Entry = ConvertCostTableLookup(
3658 Table: AVX512DQConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3659 if (auto KindCost = Entry->Cost[CostKind])
3660 return *KindCost;
3661
3662 if (ST->hasAVX512())
3663 if (const auto *Entry = ConvertCostTableLookup(
3664 Table: AVX512FConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3665 if (auto KindCost = Entry->Cost[CostKind])
3666 return *KindCost;
3667 }
3668
3669 if (ST->hasBWI())
3670 if (const auto *Entry = ConvertCostTableLookup(
3671 Table: AVX512BWVLConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3672 if (auto KindCost = Entry->Cost[CostKind])
3673 return *KindCost;
3674
3675 if (ST->hasDQI())
3676 if (const auto *Entry = ConvertCostTableLookup(
3677 Table: AVX512DQVLConversionTbl, ISD, Dst: SimpleDstTy, Src: SimpleSrcTy))
3678 if (auto KindCost = Entry->Cost[CostKind])
3679 return *KindCost;
3680
3681 if (ST->hasAVX512())
3682 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512VLConversionTbl, ISD,
3683 Dst: SimpleDstTy, Src: SimpleSrcTy))
3684 if (auto KindCost = Entry->Cost[CostKind])
3685 return *KindCost;
3686
3687 if (ST->hasAVX2()) {
3688 if (const auto *Entry = ConvertCostTableLookup(Table: AVX2ConversionTbl, ISD,
3689 Dst: SimpleDstTy, Src: SimpleSrcTy))
3690 if (auto KindCost = Entry->Cost[CostKind])
3691 return *KindCost;
3692 }
3693
3694 if (ST->hasAVX()) {
3695 if (const auto *Entry = ConvertCostTableLookup(Table: AVXConversionTbl, ISD,
3696 Dst: SimpleDstTy, Src: SimpleSrcTy))
3697 if (auto KindCost = Entry->Cost[CostKind])
3698 return *KindCost;
3699 }
3700
3701 if (ST->hasF16C()) {
3702 if (const auto *Entry = ConvertCostTableLookup(Table: F16ConversionTbl, ISD,
3703 Dst: SimpleDstTy, Src: SimpleSrcTy))
3704 if (auto KindCost = Entry->Cost[CostKind])
3705 return *KindCost;
3706 }
3707
3708 if (ST->hasSSE41()) {
3709 if (const auto *Entry = ConvertCostTableLookup(Table: SSE41ConversionTbl, ISD,
3710 Dst: SimpleDstTy, Src: SimpleSrcTy))
3711 if (auto KindCost = Entry->Cost[CostKind])
3712 return *KindCost;
3713 }
3714
3715 if (ST->hasSSE2()) {
3716 if (const auto *Entry = ConvertCostTableLookup(Table: SSE2ConversionTbl, ISD,
3717 Dst: SimpleDstTy, Src: SimpleSrcTy))
3718 if (auto KindCost = Entry->Cost[CostKind])
3719 return *KindCost;
3720 }
3721
3722 if ((ISD == ISD::FP_ROUND && SimpleDstTy == MVT::f16) ||
3723 (ISD == ISD::FP_EXTEND && SimpleSrcTy == MVT::f16)) {
3724 // fp16 conversions not covered by any table entries require a libcall.
3725 // Return a large (arbitrary) number to model this.
3726 return InstructionCost(64);
3727 }
3728 }
3729
3730 // Fall back to legalized types.
3731 std::pair<InstructionCost, MVT> LTSrc = getTypeLegalizationCost(Ty: Src);
3732 std::pair<InstructionCost, MVT> LTDest = getTypeLegalizationCost(Ty: Dst);
3733
3734 // If we're truncating to the same legalized type - just assume its free.
3735 if (ISD == ISD::TRUNCATE && LTSrc.second == LTDest.second)
3736 return TTI::TCC_Free;
3737
3738 if (ST->useAVX512Regs()) {
3739 if (ST->hasBWI())
3740 if (const auto *Entry = ConvertCostTableLookup(
3741 Table: AVX512BWConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3742 if (auto KindCost = Entry->Cost[CostKind])
3743 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3744
3745 if (ST->hasDQI())
3746 if (const auto *Entry = ConvertCostTableLookup(
3747 Table: AVX512DQConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3748 if (auto KindCost = Entry->Cost[CostKind])
3749 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3750
3751 if (ST->hasAVX512())
3752 if (const auto *Entry = ConvertCostTableLookup(
3753 Table: AVX512FConversionTbl, ISD, Dst: LTDest.second, Src: LTSrc.second))
3754 if (auto KindCost = Entry->Cost[CostKind])
3755 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3756 }
3757
3758 if (ST->hasBWI())
3759 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512BWVLConversionTbl, ISD,
3760 Dst: LTDest.second, Src: LTSrc.second))
3761 if (auto KindCost = Entry->Cost[CostKind])
3762 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3763
3764 if (ST->hasDQI())
3765 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512DQVLConversionTbl, ISD,
3766 Dst: LTDest.second, Src: LTSrc.second))
3767 if (auto KindCost = Entry->Cost[CostKind])
3768 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3769
3770 if (ST->hasAVX512())
3771 if (const auto *Entry = ConvertCostTableLookup(Table: AVX512VLConversionTbl, ISD,
3772 Dst: LTDest.second, Src: LTSrc.second))
3773 if (auto KindCost = Entry->Cost[CostKind])
3774 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3775
3776 if (ST->hasAVX2())
3777 if (const auto *Entry = ConvertCostTableLookup(Table: AVX2ConversionTbl, ISD,
3778 Dst: LTDest.second, Src: LTSrc.second))
3779 if (auto KindCost = Entry->Cost[CostKind])
3780 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3781
3782 if (ST->hasAVX())
3783 if (const auto *Entry = ConvertCostTableLookup(Table: AVXConversionTbl, ISD,
3784 Dst: LTDest.second, Src: LTSrc.second))
3785 if (auto KindCost = Entry->Cost[CostKind])
3786 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3787
3788 if (ST->hasF16C()) {
3789 if (const auto *Entry = ConvertCostTableLookup(Table: F16ConversionTbl, ISD,
3790 Dst: LTDest.second, Src: LTSrc.second))
3791 if (auto KindCost = Entry->Cost[CostKind])
3792 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3793 }
3794
3795 if (ST->hasSSE41())
3796 if (const auto *Entry = ConvertCostTableLookup(Table: SSE41ConversionTbl, ISD,
3797 Dst: LTDest.second, Src: LTSrc.second))
3798 if (auto KindCost = Entry->Cost[CostKind])
3799 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3800
3801 if (ST->hasSSE2())
3802 if (const auto *Entry = ConvertCostTableLookup(Table: SSE2ConversionTbl, ISD,
3803 Dst: LTDest.second, Src: LTSrc.second))
3804 if (auto KindCost = Entry->Cost[CostKind])
3805 return std::max(a: LTSrc.first, b: LTDest.first) * *KindCost;
3806
3807 // Fallback, for i8/i16 sitofp/uitofp cases we need to extend to i32 for
3808 // sitofp.
3809 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
3810 1 < Src->getScalarSizeInBits() && Src->getScalarSizeInBits() < 32) {
3811 Type *ExtSrc = Src->getWithNewBitWidth(NewBitWidth: 32);
3812 unsigned ExtOpc =
3813 (ISD == ISD::SINT_TO_FP) ? Instruction::SExt : Instruction::ZExt;
3814
3815 // For scalar loads the extend would be free.
3816 InstructionCost ExtCost = 0;
3817 if (!(Src->isIntegerTy() && I && isa<LoadInst>(Val: I->getOperand(i: 0))))
3818 ExtCost = getCastInstrCost(Opcode: ExtOpc, Dst: ExtSrc, Src, CCH, CostKind);
3819
3820 return ExtCost + getCastInstrCost(Opcode: Instruction::SIToFP, Dst, Src: ExtSrc,
3821 CCH: TTI::CastContextHint::None, CostKind);
3822 }
3823
3824 // Fallback for fptosi/fptoui i8/i16 cases we need to truncate from fptosi
3825 // i32.
3826 if ((ISD == ISD::FP_TO_SINT || ISD == ISD::FP_TO_UINT) &&
3827 1 < Dst->getScalarSizeInBits() && Dst->getScalarSizeInBits() < 32) {
3828 Type *TruncDst = Dst->getWithNewBitWidth(NewBitWidth: 32);
3829 return getCastInstrCost(Opcode: Instruction::FPToSI, Dst: TruncDst, Src, CCH, CostKind) +
3830 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: TruncDst,
3831 CCH: TTI::CastContextHint::None, CostKind);
3832 }
3833
3834 // TODO: Allow non-throughput costs that aren't binary.
3835 auto AdjustCost = [&CostKind](InstructionCost Cost,
3836 InstructionCost N = 1) -> InstructionCost {
3837 if (CostKind != TTI::TCK_RecipThroughput)
3838 return Cost == 0 ? 0 : N;
3839 return Cost * N;
3840 };
3841 return AdjustCost(
3842 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
3843}
3844
3845// Additive cost for a predicated op's "predicate fanout": a select, masked
3846// load/store or gather/scatter keeps a compact <N x i1> mask, but if its data
3847// legalizes into more parts than the mask, codegen builds a sub-mask per extra
3848// part (kshiftr on AVX-512, unpack/extend on SSE/AVX) the per-part tables miss.
3849// FanoutCostPerPart is a tie-break: it beats the vectorizer's widest-VF bias.
3850static InstructionCost getPredicateFanoutCost(const X86TTIImpl &TTI,
3851 Type *DataVTy, Type *MaskVTy) {
3852 constexpr unsigned FanoutCostPerPart = 4;
3853 unsigned DataParts = TTI.getNumberOfParts(Tp: DataVTy);
3854 unsigned MaskParts = TTI.getNumberOfParts(Tp: MaskVTy);
3855 if (DataParts > MaskParts && MaskParts > 0)
3856 return InstructionCost((DataParts - MaskParts) * FanoutCostPerPart);
3857 return InstructionCost(0);
3858}
3859
3860InstructionCost X86TTIImpl::getCmpSelInstrCost(
3861 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
3862 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
3863 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
3864 // Early out if this type isn't scalar/vector integer/float.
3865 if (!(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy()))
3866 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
3867 Op1Info, Op2Info, I);
3868
3869 // Legalize the type.
3870 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
3871
3872 MVT MTy = LT.second;
3873
3874 int ISD = TLI->InstructionOpcodeToISD(Opcode);
3875 assert(ISD && "Invalid opcode");
3876
3877 // Predicate fanout (see getPredicateFanoutCost), AVX-512 only: on SSE/AVX a
3878 // select blend is already priced per part, so charging it there shrinks VF.
3879 InstructionCost MaskExpCost = 0;
3880 if (Opcode == Instruction::Select && ST->hasAVX512() &&
3881 isa<VectorType>(Val: ValTy) && isa_and_nonnull<VectorType>(Val: CondTy))
3882 MaskExpCost = getPredicateFanoutCost(TTI: *this, DataVTy: ValTy, MaskVTy: CondTy);
3883
3884 InstructionCost ExtraCost = 0;
3885 if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
3886 // Some vector comparison predicates cost extra instructions.
3887 // TODO: Adjust ExtraCost based on CostKind?
3888 // TODO: Should we invert this and assume worst case cmp costs
3889 // and reduce for particular predicates?
3890 if (MTy.isVector() &&
3891 !((ST->hasXOP() && (!ST->hasAVX2() || MTy.is128BitVector())) ||
3892 (ST->hasAVX512() && 32 <= MTy.getScalarSizeInBits()) ||
3893 ST->hasBWI())) {
3894 // Fallback to I if a specific predicate wasn't specified.
3895 CmpInst::Predicate Pred = VecPred;
3896 if (I && (Pred == CmpInst::BAD_ICMP_PREDICATE ||
3897 Pred == CmpInst::BAD_FCMP_PREDICATE))
3898 Pred = cast<CmpInst>(Val: I)->getPredicate();
3899
3900 bool CmpWithConstant = false;
3901 if (auto *CmpInstr = dyn_cast_or_null<CmpInst>(Val: I))
3902 CmpWithConstant = isa<Constant>(Val: CmpInstr->getOperand(i_nocapture: 1));
3903
3904 switch (Pred) {
3905 case CmpInst::Predicate::ICMP_NE:
3906 // xor(cmpeq(x,y),-1)
3907 ExtraCost = CmpWithConstant ? 0 : 1;
3908 break;
3909 case CmpInst::Predicate::ICMP_SGE:
3910 case CmpInst::Predicate::ICMP_SLE:
3911 // xor(cmpgt(x,y),-1)
3912 ExtraCost = CmpWithConstant ? 0 : 1;
3913 break;
3914 case CmpInst::Predicate::ICMP_ULT:
3915 case CmpInst::Predicate::ICMP_UGT:
3916 // cmpgt(xor(x,signbit),xor(y,signbit))
3917 // xor(cmpeq(pmaxu(x,y),x),-1)
3918 ExtraCost = CmpWithConstant ? 1 : 2;
3919 break;
3920 case CmpInst::Predicate::ICMP_ULE:
3921 case CmpInst::Predicate::ICMP_UGE:
3922 if ((ST->hasSSE41() && MTy.getScalarSizeInBits() == 32) ||
3923 (ST->hasSSE2() && MTy.getScalarSizeInBits() < 32)) {
3924 // cmpeq(psubus(x,y),0)
3925 // cmpeq(pminu(x,y),x)
3926 ExtraCost = 1;
3927 } else {
3928 // xor(cmpgt(xor(x,signbit),xor(y,signbit)),-1)
3929 ExtraCost = CmpWithConstant ? 2 : 3;
3930 }
3931 break;
3932 case CmpInst::Predicate::FCMP_ONE:
3933 case CmpInst::Predicate::FCMP_UEQ:
3934 // Without AVX we need to expand FCMP_ONE/FCMP_UEQ cases.
3935 // Use FCMP_UEQ expansion - FCMP_ONE should be the same.
3936 if (CondTy && !ST->hasAVX())
3937 return getCmpSelInstrCost(Opcode, ValTy, CondTy,
3938 VecPred: CmpInst::Predicate::FCMP_UNO, CostKind,
3939 Op1Info, Op2Info) +
3940 getCmpSelInstrCost(Opcode, ValTy, CondTy,
3941 VecPred: CmpInst::Predicate::FCMP_OEQ, CostKind,
3942 Op1Info, Op2Info) +
3943 getArithmeticInstrCost(Opcode: Instruction::Or, Ty: CondTy, CostKind);
3944
3945 break;
3946 case CmpInst::Predicate::BAD_ICMP_PREDICATE:
3947 case CmpInst::Predicate::BAD_FCMP_PREDICATE:
3948 // Assume worst case scenario and add the maximum extra cost.
3949 ExtraCost = 3;
3950 break;
3951 default:
3952 break;
3953 }
3954 }
3955 }
3956
3957 static const CostKindTblEntry SLMCostTbl[] = {
3958 // slm pcmpeq/pcmpgt throughput is 2
3959 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
3960 // slm pblendvb/blendvpd/blendvps throughput is 4
3961 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vblendvpd
3962 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vblendvps
3963 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3964 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3965 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3966 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // pblendvb
3967 };
3968
3969 static const CostKindTblEntry AVX512BWCostTbl[] = {
3970 { .ISD: ISD::SETCC, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3971 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3972 { .ISD: ISD::SETCC, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3973 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3974
3975 { .ISD: ISD::SELECT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3976 { .ISD: ISD::SELECT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3977 };
3978
3979 static const CostKindTblEntry AVX512CostTbl[] = {
3980 { .ISD: ISD::SETCC, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3981 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3982 { .ISD: ISD::SETCC, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3983 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3984
3985 { .ISD: ISD::SETCC, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3986 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3987 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3988 { .ISD: ISD::SETCC, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3989 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3990 { .ISD: ISD::SETCC, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
3991 { .ISD: ISD::SETCC, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
3992
3993 { .ISD: ISD::SELECT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3994 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3995 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3996 { .ISD: ISD::SELECT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3997 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3998 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
3999 { .ISD: ISD::SELECT, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4000 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4001 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4002 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4003 { .ISD: ISD::SELECT, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4004 { .ISD: ISD::SELECT, .Type: MVT::v8f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4005 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4006 { .ISD: ISD::SELECT, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4007
4008 { .ISD: ISD::SELECT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4009 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4010 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4011 { .ISD: ISD::SELECT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4012 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4013 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4014 };
4015
4016 static const CostKindTblEntry AVX2CostTbl[] = {
4017 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4018 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4019 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4020 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4021 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4022 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4023
4024 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4025 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4026 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4027 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4028
4029 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4030 { .ISD: ISD::SELECT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4031 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4032 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4033 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4034 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4035 };
4036
4037 static const CostKindTblEntry XOPCostTbl[] = {
4038 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4039 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4040 };
4041
4042 static const CostKindTblEntry AVX1CostTbl[] = {
4043 { .ISD: ISD::SETCC, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4044 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4045 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4046 { .ISD: ISD::SETCC, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4047 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4048 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4049
4050 // AVX1 does not support 8-wide integer compare.
4051 { .ISD: ISD::SETCC, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4052 { .ISD: ISD::SETCC, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4053 { .ISD: ISD::SETCC, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4054 { .ISD: ISD::SETCC, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4055
4056 { .ISD: ISD::SELECT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4057 { .ISD: ISD::SELECT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4058 { .ISD: ISD::SELECT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvpd
4059 { .ISD: ISD::SELECT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // vblendvps
4060 { .ISD: ISD::SELECT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // vandps + vandnps + vorps
4061 { .ISD: ISD::SELECT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // vandps + vandnps + vorps
4062 };
4063
4064 static const CostKindTblEntry SSE42CostTbl[] = {
4065 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4066 };
4067
4068 static const CostKindTblEntry SSE41CostTbl[] = {
4069 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4070 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4071
4072 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvpd
4073 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvpd
4074 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvps
4075 { .ISD: ISD::SELECT, .Type: MVT::f32 , .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // blendvps
4076 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4077 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4078 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4079 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // pblendvb
4080 };
4081
4082 static const CostKindTblEntry SSE2CostTbl[] = {
4083 { .ISD: ISD::SETCC, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4084 { .ISD: ISD::SETCC, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4085
4086 { .ISD: ISD::SETCC, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } }, // pcmpeqd/pcmpgtd expansion
4087 { .ISD: ISD::SETCC, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4088 { .ISD: ISD::SETCC, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4089 { .ISD: ISD::SETCC, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4090
4091 { .ISD: ISD::SELECT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andpd + andnpd + orpd
4092 { .ISD: ISD::SELECT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andpd + andnpd + orpd
4093 { .ISD: ISD::SELECT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4094 { .ISD: ISD::SELECT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4095 { .ISD: ISD::SELECT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4096 { .ISD: ISD::SELECT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // pand + pandn + por
4097 };
4098
4099 static const CostKindTblEntry SSE1CostTbl[] = {
4100 { .ISD: ISD::SETCC, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4101 { .ISD: ISD::SETCC, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4102
4103 { .ISD: ISD::SELECT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andps + andnps + orps
4104 { .ISD: ISD::SELECT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // andps + andnps + orps
4105 };
4106
4107 if (ST->useSLMArithCosts())
4108 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
4109 if (auto KindCost = Entry->Cost[CostKind])
4110 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4111
4112 if (ST->hasBWI())
4113 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
4114 if (auto KindCost = Entry->Cost[CostKind])
4115 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4116
4117 if (ST->hasAVX512())
4118 if (const auto *Entry = CostTableLookup(Table: AVX512CostTbl, ISD, Ty: MTy))
4119 if (auto KindCost = Entry->Cost[CostKind])
4120 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4121
4122 if (ST->hasAVX2())
4123 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
4124 if (auto KindCost = Entry->Cost[CostKind])
4125 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4126
4127 if (ST->hasXOP())
4128 if (const auto *Entry = CostTableLookup(Table: XOPCostTbl, ISD, Ty: MTy))
4129 if (auto KindCost = Entry->Cost[CostKind])
4130 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4131
4132 if (ST->hasAVX())
4133 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
4134 if (auto KindCost = Entry->Cost[CostKind])
4135 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4136
4137 if (ST->hasSSE42())
4138 if (const auto *Entry = CostTableLookup(Table: SSE42CostTbl, ISD, Ty: MTy))
4139 if (auto KindCost = Entry->Cost[CostKind])
4140 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4141
4142 if (ST->hasSSE41())
4143 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
4144 if (auto KindCost = Entry->Cost[CostKind])
4145 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4146
4147 if (ST->hasSSE2())
4148 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
4149 if (auto KindCost = Entry->Cost[CostKind])
4150 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4151
4152 if (ST->hasSSE1())
4153 if (const auto *Entry = CostTableLookup(Table: SSE1CostTbl, ISD, Ty: MTy))
4154 if (auto KindCost = Entry->Cost[CostKind])
4155 return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
4156
4157 // Assume a 3cy latency for fp select ops.
4158 if (CostKind == TTI::TCK_Latency && Opcode == Instruction::Select)
4159 if (ValTy->getScalarType()->isFloatingPointTy())
4160 return 3;
4161
4162 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
4163 Op1Info, Op2Info, I) +
4164 MaskExpCost;
4165}
4166
4167unsigned X86TTIImpl::getAtomicMemIntrinsicMaxElementSize() const { return 16; }
4168
4169/// Returns true if \p V is the square root, divided into with the reciprocal
4170/// allowed, and the target replaces the pair with the rsqrt* based estimate,
4171/// costed with the division. The estimate is formed for the legalized type
4172/// \p VT.
4173static bool isFoldedIntoRsqrtEstimate(const Value *V,
4174 const X86TargetLowering &TLI, MVT VT,
4175 TTI::TargetCostKind CostKind) {
4176 using namespace PatternMatch;
4177 if (CostKind != TTI::TCK_RecipThroughput && CostKind != TTI::TCK_Latency)
4178 return false;
4179 const auto *II = dyn_cast_if_present<IntrinsicInst>(Val: V);
4180 if (!II || II->getIntrinsicID() != Intrinsic::sqrt || !II->hasOneUse() ||
4181 !match(V: II->user_back(), P: m_FDiv(L: m_Value(), R: m_Specific(V: II))) ||
4182 !cast<FPMathOperator>(Val: II->user_back())->hasAllowReciprocal())
4183 return false;
4184 return TLI.hasSqrtEstimate(VT, /*Reciprocal=*/true) &&
4185 TLI.getRecipEstimateSqrtEnabled(VT, F: *II->getFunction()) !=
4186 TargetLoweringBase::ReciprocalEstimate::Disabled;
4187}
4188
4189InstructionCost
4190X86TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
4191 TTI::TargetCostKind CostKind) const {
4192 // Costs should match the codegen from:
4193 // BITREVERSE: llvm\test\CodeGen\X86\vector-bitreverse.ll
4194 // BSWAP: llvm\test\CodeGen\X86\bswap-vector.ll
4195 // CTLZ: llvm\test\CodeGen\X86\vector-lzcnt-*.ll
4196 // CTPOP: llvm\test\CodeGen\X86\vector-popcnt-*.ll
4197 // CTTZ: llvm\test\CodeGen\X86\vector-tzcnt-*.ll
4198
4199 // TODO: Overflow intrinsics (*ADDO, *SUBO, *MULO) with vector types are not
4200 // specialized in these tables yet.
4201 static const CostKindTblEntry AVX512VBMI2CostTbl[] = {
4202 { .ISD: ISD::FSHL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4203 { .ISD: ISD::FSHL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4204 { .ISD: ISD::FSHL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4205 { .ISD: ISD::FSHL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4206 { .ISD: ISD::FSHL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4207 { .ISD: ISD::FSHL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4208 { .ISD: ISD::FSHL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4209 { .ISD: ISD::FSHL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4210 { .ISD: ISD::FSHL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4211 { .ISD: ISD::ROTL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4212 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4213 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4214 { .ISD: ISD::ROTR, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4215 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4216 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4217 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4218 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4219 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4220 };
4221 static const CostKindTblEntry AVX512BITALGCostTbl[] = {
4222 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4223 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4224 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4225 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4226 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4227 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4228 };
4229 static const CostKindTblEntry AVX512VPOPCNTDQCostTbl[] = {
4230 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4231 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4232 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4233 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4234 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4235 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4236 };
4237 static const CostKindTblEntry AVX512CDCostTbl[] = {
4238 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4239 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4240 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 27, .CodeSizeCost: 23, .SizeAndLatencyCost: 27 } },
4241 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 16, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4242 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4243 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4244 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 19, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4245 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4246 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4247 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4248 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 15, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
4249 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4250
4251 { .ISD: ISD::CTTZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4252 { .ISD: ISD::CTTZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4253 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4254 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4255 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4256 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4257 };
4258 static const CostKindTblEntry AVX512BWCostTbl[] = {
4259 { .ISD: ISD::ABS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4260 { .ISD: ISD::ABS, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4261 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4262 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4263 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4264 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4265 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4266 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4267 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4268 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4269 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4270 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4271 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4272 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 12 } },
4273 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4274 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4275 { .ISD: ISD::BSWAP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4276 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4277 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4278 { .ISD: ISD::BSWAP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4279 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4280 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4281 { .ISD: ISD::BSWAP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4282 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 22, .CodeSizeCost: 23, .SizeAndLatencyCost: 23 } },
4283 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 23, .CodeSizeCost: 25, .SizeAndLatencyCost: 25 } },
4284 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 15, .SizeAndLatencyCost: 16 } },
4285 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 9 } },
4286 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4287 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4288 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 10, .SizeAndLatencyCost: 12 } },
4289 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4290 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4291 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4292 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4293 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4294 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4295 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4296 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4297 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4298 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4299 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4300 { .ISD: ISD::CTTZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4301 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4302 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4303 { .ISD: ISD::CTTZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4304 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v32i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4305 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v64i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4306 { .ISD: ISD::MULHS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4307 { .ISD: ISD::MULHU, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4308 { .ISD: ISD::ROTL, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } },
4309 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4310 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4311 { .ISD: ISD::ROTL, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4312 { .ISD: ISD::ROTL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
4313 { .ISD: ISD::ROTL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
4314 { .ISD: ISD::ROTR, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 8 } },
4315 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4316 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4317 { .ISD: ISD::ROTR, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 12, .SizeAndLatencyCost: 14 } },
4318 { .ISD: ISD::ROTR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 14, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
4319 { .ISD: ISD::ROTR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 14, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
4320 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4321 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4322 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4323 { .ISD: X86ISD::VROTLI, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4324 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4325 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4326 { .ISD: ISD::SADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4327 { .ISD: ISD::SADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4328 { .ISD: ISD::SMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4329 { .ISD: ISD::SMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4330 { .ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4331 { .ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4332 { .ISD: ISD::SMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4333 { .ISD: ISD::SMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 21, .CodeSizeCost: 17, .SizeAndLatencyCost: 18 } },
4334 { .ISD: ISD::UMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4335 { .ISD: ISD::UMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 15, .CodeSizeCost: 15, .SizeAndLatencyCost: 16 } },
4336 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4337 { .ISD: ISD::SSUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4338 { .ISD: ISD::UADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4339 { .ISD: ISD::UADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4340 { .ISD: ISD::UMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4341 { .ISD: ISD::UMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4342 { .ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4343 { .ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4344 { .ISD: ISD::USUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4345 { .ISD: ISD::USUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4346 };
4347 static const CostKindTblEntry AVX512CostTbl[] = {
4348 { .ISD: ISD::ABS, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4349 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4350 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4351 { .ISD: ISD::ABS, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4352 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4353 { .ISD: ISD::ABS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4354 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4355 { .ISD: ISD::ABS, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4356 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4357 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4358 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4359 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4360 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4361 { .ISD: ISD::BSWAP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4362 { .ISD: ISD::BSWAP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4363 { .ISD: ISD::BSWAP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4364 { .ISD: ISD::CTLZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 28, .CodeSizeCost: 32, .SizeAndLatencyCost: 32 } },
4365 { .ISD: ISD::CTLZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 30, .CodeSizeCost: 38, .SizeAndLatencyCost: 38 } },
4366 { .ISD: ISD::CTLZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 15, .CodeSizeCost: 29, .SizeAndLatencyCost: 29 } },
4367 { .ISD: ISD::CTLZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
4368 { .ISD: ISD::CTPOP, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
4369 { .ISD: ISD::CTPOP, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 19, .CodeSizeCost: 27, .SizeAndLatencyCost: 27 } },
4370 { .ISD: ISD::CTPOP, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 15, .CodeSizeCost: 22, .SizeAndLatencyCost: 22 } },
4371 { .ISD: ISD::CTPOP, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 11, .CodeSizeCost: 16, .SizeAndLatencyCost: 16 } },
4372 { .ISD: ISD::CTTZ, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4373 { .ISD: ISD::CTTZ, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4374 { .ISD: ISD::CTTZ, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 17, .CodeSizeCost: 27, .SizeAndLatencyCost: 27 } },
4375 { .ISD: ISD::CTTZ, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 21, .SizeAndLatencyCost: 21 } },
4376 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v2i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4377 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v4i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4378 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v8i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4379 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v16i1, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // kmov seq
4380 { .ISD: ISD::MULHS, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4381 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4382 { .ISD: ISD::MULHS, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4383 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4384 { .ISD: ISD::MULHU, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4385 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4386 { .ISD: ISD::MULHU, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4387 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4388 { .ISD: ISD::ROTL, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4389 { .ISD: ISD::ROTL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4390 { .ISD: ISD::ROTL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4391 { .ISD: ISD::ROTL, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4392 { .ISD: ISD::ROTL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4393 { .ISD: ISD::ROTL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4394 { .ISD: ISD::ROTR, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4395 { .ISD: ISD::ROTR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4396 { .ISD: ISD::ROTR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4397 { .ISD: ISD::ROTR, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4398 { .ISD: ISD::ROTR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4399 { .ISD: ISD::ROTR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4400 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4401 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4402 { .ISD: X86ISD::VROTLI, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4403 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4404 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4405 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4406 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4407 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4408 { .ISD: ISD::SADDSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4409 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4410 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4411 { .ISD: ISD::SADDSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4412 { .ISD: ISD::SADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4413 { .ISD: ISD::SADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4414 { .ISD: ISD::SMAX, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4415 { .ISD: ISD::SMAX, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4416 { .ISD: ISD::SMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4417 { .ISD: ISD::SMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4418 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4419 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4420 { .ISD: ISD::SMIN, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4421 { .ISD: ISD::SMIN, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4422 { .ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4423 { .ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4424 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4425 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4426 { .ISD: ISD::SMULO, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 44, .LatencyCost: 44, .CodeSizeCost: 81, .SizeAndLatencyCost: 93 } },
4427 { .ISD: ISD::SMULO, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4428 { .ISD: ISD::SMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4429 { .ISD: ISD::SMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 22, .LatencyCost: 28, .CodeSizeCost: 42, .SizeAndLatencyCost: 42 } },
4430 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4431 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4432 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4433 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4434 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 15, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4435 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 14, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } },
4436 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4437 { .ISD: ISD::SSUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4438 { .ISD: ISD::UMAX, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4439 { .ISD: ISD::UMAX, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4440 { .ISD: ISD::UMAX, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4441 { .ISD: ISD::UMAX, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4442 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4443 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4444 { .ISD: ISD::UMIN, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4445 { .ISD: ISD::UMIN, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4446 { .ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4447 { .ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4448 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4449 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4450 { .ISD: ISD::UMULO, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 52, .LatencyCost: 52, .CodeSizeCost: 95, .SizeAndLatencyCost: 104} },
4451 { .ISD: ISD::UMULO, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4452 { .ISD: ISD::UMULO, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 13, .CodeSizeCost: 16, .SizeAndLatencyCost: 16 } },
4453 { .ISD: ISD::UMULO, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 30, .SizeAndLatencyCost: 30 } },
4454 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4455 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4456 { .ISD: ISD::UADDSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4457 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4458 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4459 { .ISD: ISD::UADDSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4460 { .ISD: ISD::UADDSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4461 { .ISD: ISD::UADDSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4462 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4463 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4464 { .ISD: ISD::USUBSAT, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4465 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4466 { .ISD: ISD::USUBSAT, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4467 { .ISD: ISD::USUBSAT, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4468 { .ISD: ISD::USUBSAT, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4469 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4470 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4471 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4472 { .ISD: ISD::FMAXNUM, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4473 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4474 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4475 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4476 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4477 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4478 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4479 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4480 { .ISD: ISD::FSQRT, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
4481 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4482 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4483 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Skylake from http://www.agner.org/
4484 { .ISD: ISD::FSQRT, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // Skylake from http://www.agner.org/
4485 };
4486 static const CostKindTblEntry XOPCostTbl[] = {
4487 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4488 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4489 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4490 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4491 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4492 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4493 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4494 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4495 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4496 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4497 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4498 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } },
4499 // XOP: ROTL = VPROT(X,Y), ROTR = VPROT(X,SUB(0,Y))
4500 { .ISD: ISD::ROTL, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4501 { .ISD: ISD::ROTL, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4502 { .ISD: ISD::ROTL, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4503 { .ISD: ISD::ROTL, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4504 { .ISD: ISD::ROTL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4505 { .ISD: ISD::ROTL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4506 { .ISD: ISD::ROTL, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4507 { .ISD: ISD::ROTL, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4508 { .ISD: ISD::ROTR, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4509 { .ISD: ISD::ROTR, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4510 { .ISD: ISD::ROTR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4511 { .ISD: ISD::ROTR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 9 } },
4512 { .ISD: ISD::ROTR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4513 { .ISD: ISD::ROTR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4514 { .ISD: ISD::ROTR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4515 { .ISD: ISD::ROTR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4516 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4517 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4518 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4519 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4520 { .ISD: X86ISD::VROTLI, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4521 { .ISD: X86ISD::VROTLI, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4522 { .ISD: X86ISD::VROTLI, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4523 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4524 };
4525 static const CostKindTblEntry AVX2CostTbl[] = {
4526 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4527 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4528 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4529 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4530 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4531 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4532 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4533 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4534 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4535 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4536 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4537 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4538 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } },
4539 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 10, .SizeAndLatencyCost: 17 } },
4540 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4541 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 15 } },
4542 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4543 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4544 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4545 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4546 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4547 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4548 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 18, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4549 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 18, .CodeSizeCost: 24, .SizeAndLatencyCost: 44 } },
4550 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 20 } },
4551 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 16, .CodeSizeCost: 19, .SizeAndLatencyCost: 34 } },
4552 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } },
4553 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } },
4554 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 10 } },
4555 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 14 } },
4556 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 10, .SizeAndLatencyCost: 10 } },
4557 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4558 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4559 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4560 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4561 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 11, .SizeAndLatencyCost: 18 } },
4562 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4563 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4564 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 13 } },
4565 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 20 } },
4566 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 14, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
4567 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 17, .SizeAndLatencyCost: 24 } },
4568 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4569 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } },
4570 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4571 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 11, .SizeAndLatencyCost: 18 } },
4572 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 13 } }, // movmsk seq
4573 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 13 } }, // movmsk seq
4574 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 9, .SizeAndLatencyCost: 14 } }, // movmsk seq
4575 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } }, // movmsk seq
4576 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 9, .SizeAndLatencyCost: 14 } }, // movmsk seq
4577 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } },
4578 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4579 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } },
4580 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4581 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
4582 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4583 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 9 } },
4584 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 13 } },
4585 { .ISD: ISD::SADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4586 { .ISD: ISD::SADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4587 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4588 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4589 { .ISD: ISD::SMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4590 { .ISD: ISD::SMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4591 { .ISD: ISD::SMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4592 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4593 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4594 { .ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4595 { .ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4596 { .ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4597 { .ISD: ISD::SMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 33, .SizeAndLatencyCost: 37 } },
4598 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 13, .SizeAndLatencyCost: 15 } },
4599 { .ISD: ISD::SMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 20, .CodeSizeCost: 13, .SizeAndLatencyCost: 24 } },
4600 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 15, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4601 { .ISD: ISD::SMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 14, .CodeSizeCost: 8, .SizeAndLatencyCost: 14 } },
4602 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4603 { .ISD: ISD::SMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 15, .CodeSizeCost: 18, .SizeAndLatencyCost: 35 } },
4604 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 22, .CodeSizeCost: 14, .SizeAndLatencyCost: 21 } },
4605 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4606 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4607 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 14, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } },
4608 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 15, .CodeSizeCost: 9, .SizeAndLatencyCost: 16 } },
4609 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4610 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4611 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4612 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 10 } },
4613 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } },
4614 { .ISD: ISD::UADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4615 { .ISD: ISD::UADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4616 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4617 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
4618 { .ISD: ISD::UMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4619 { .ISD: ISD::UMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4620 { .ISD: ISD::UMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4621 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4622 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
4623 { .ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4624 { .ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4625 { .ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4626 { .ISD: ISD::UMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 24, .CodeSizeCost: 39, .SizeAndLatencyCost: 43 } },
4627 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 10, .CodeSizeCost: 15, .SizeAndLatencyCost: 19 } },
4628 { .ISD: ISD::UMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 23 } },
4629 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 12, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4630 { .ISD: ISD::UMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 8, .SizeAndLatencyCost: 13 } },
4631 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4632 { .ISD: ISD::UMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 13, .CodeSizeCost: 17, .SizeAndLatencyCost: 33 } },
4633 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 19, .CodeSizeCost: 13, .SizeAndLatencyCost: 20 } },
4634 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4635 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 10 } },
4636 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
4637 { .ISD: ISD::USUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4638 { .ISD: ISD::USUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
4639 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4640 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4641 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4642 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4643 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4644 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 6 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4645 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtss
4646 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtps
4647 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtps
4648 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtsd
4649 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtpd
4650 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtpd
4651 };
4652 static const CostKindTblEntry AVX1CostTbl[] = {
4653 { .ISD: ISD::ABS, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // VBLENDVPD(X,VPSUBQ(0,X),X)
4654 { .ISD: ISD::ABS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4655 { .ISD: ISD::ABS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4656 { .ISD: ISD::ABS, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4657 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4658 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4659 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4660 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4661 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 20, .CodeSizeCost: 20, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4662 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 13, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4663 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 15, .CodeSizeCost: 17, .SizeAndLatencyCost: 26 } }, // 2 x 128-bit Op + extract/insert
4664 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4665 { .ISD: ISD::BSWAP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4666 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4667 { .ISD: ISD::BSWAP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4668 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4669 { .ISD: ISD::BSWAP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 10 } },
4670 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
4671 { .ISD: ISD::CTLZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 29, .LatencyCost: 33, .CodeSizeCost: 49, .SizeAndLatencyCost: 58 } }, // 2 x 128-bit Op + extract/insert
4672 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 24, .CodeSizeCost: 24, .SizeAndLatencyCost: 28 } },
4673 { .ISD: ISD::CTLZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 28, .CodeSizeCost: 39, .SizeAndLatencyCost: 48 } }, // 2 x 128-bit Op + extract/insert
4674 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 20, .CodeSizeCost: 19, .SizeAndLatencyCost: 23 } },
4675 { .ISD: ISD::CTLZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 29, .SizeAndLatencyCost: 38 } }, // 2 x 128-bit Op + extract/insert
4676 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 16, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4677 { .ISD: ISD::CTLZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 15, .CodeSizeCost: 19, .SizeAndLatencyCost: 28 } }, // 2 x 128-bit Op + extract/insert
4678 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4679 { .ISD: ISD::CTPOP, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 18, .CodeSizeCost: 19, .SizeAndLatencyCost: 28 } }, // 2 x 128-bit Op + extract/insert
4680 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 14, .CodeSizeCost: 10, .SizeAndLatencyCost: 14 } },
4681 { .ISD: ISD::CTPOP, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 27, .SizeAndLatencyCost: 36 } }, // 2 x 128-bit Op + extract/insert
4682 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 20, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4683 { .ISD: ISD::CTPOP, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 21, .CodeSizeCost: 22, .SizeAndLatencyCost: 31 } }, // 2 x 128-bit Op + extract/insert
4684 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 18, .CodeSizeCost: 11, .SizeAndLatencyCost: 15 } },
4685 { .ISD: ISD::CTPOP, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 15, .CodeSizeCost: 16, .SizeAndLatencyCost: 25 } }, // 2 x 128-bit Op + extract/insert
4686 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 12, .CodeSizeCost: 8, .SizeAndLatencyCost: 12 } },
4687 { .ISD: ISD::CTTZ, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 17, .LatencyCost: 22, .CodeSizeCost: 24, .SizeAndLatencyCost: 33 } }, // 2 x 128-bit Op + extract/insert
4688 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 19, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4689 { .ISD: ISD::CTTZ, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 27, .CodeSizeCost: 32, .SizeAndLatencyCost: 41 } }, // 2 x 128-bit Op + extract/insert
4690 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 24, .CodeSizeCost: 17, .SizeAndLatencyCost: 21 } },
4691 { .ISD: ISD::CTTZ, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 27, .SizeAndLatencyCost: 36 } }, // 2 x 128-bit Op + extract/insert
4692 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 21, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4693 { .ISD: ISD::CTTZ, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 18, .CodeSizeCost: 21, .SizeAndLatencyCost: 30 } }, // 2 x 128-bit Op + extract/insert
4694 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 16, .CodeSizeCost: 11, .SizeAndLatencyCost: 15 } },
4695 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4696 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4697 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 13, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } }, // movmsk seq
4698 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4699 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 19, .SizeAndLatencyCost: 24 } }, // movmsk seq
4700 { .ISD: ISD::MULHS, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4701 { .ISD: ISD::MULHS, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4702 { .ISD: ISD::MULHU, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4703 { .ISD: ISD::MULHU, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } },
4704 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
4705 { .ISD: ISD::SADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 20, .CodeSizeCost: 15, .SizeAndLatencyCost: 25 } }, // 2 x 128-bit Op + extract/insert
4706 { .ISD: ISD::SADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 18, .CodeSizeCost: 14, .SizeAndLatencyCost: 24 } }, // 2 x 128-bit Op + extract/insert
4707 { .ISD: ISD::SADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4708 { .ISD: ISD::SADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4709 { .ISD: ISD::SMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // 2 x 128-bit Op + extract/insert
4710 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
4711 { .ISD: ISD::SMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4712 { .ISD: ISD::SMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4713 { .ISD: ISD::SMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4714 { .ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 12 } }, // 2 x 128-bit Op + extract/insert
4715 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4716 { .ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4717 { .ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4718 { .ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4719 { .ISD: ISD::SMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 33, .SizeAndLatencyCost: 37 } },
4720 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4721 { .ISD: ISD::SMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 24, .SizeAndLatencyCost: 29 } },
4722 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 15, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4723 { .ISD: ISD::SMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 14, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } },
4724 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4725 { .ISD: ISD::SMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 37, .SizeAndLatencyCost: 39 } },
4726 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 22, .CodeSizeCost: 18, .SizeAndLatencyCost: 21 } },
4727 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 9, .SizeAndLatencyCost: 13 } },
4728 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 21, .CodeSizeCost: 18, .SizeAndLatencyCost: 29 } }, // 2 x 128-bit Op + extract/insert
4729 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 19, .CodeSizeCost: 18, .SizeAndLatencyCost: 29 } }, // 2 x 128-bit Op + extract/insert
4730 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4731 { .ISD: ISD::SSUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4732 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4733 { .ISD: ISD::UADDSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } }, // 2 x 128-bit Op + extract/insert
4734 { .ISD: ISD::UADDSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 10, .SizeAndLatencyCost: 11 } }, // 2 x 128-bit Op + extract/insert
4735 { .ISD: ISD::UADDSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4736 { .ISD: ISD::UADDSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4737 { .ISD: ISD::UMAX, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 11, .SizeAndLatencyCost: 17 } }, // 2 x 128-bit Op + extract/insert
4738 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } },
4739 { .ISD: ISD::UMAX, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4740 { .ISD: ISD::UMAX, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4741 { .ISD: ISD::UMAX, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4742 { .ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 10, .CodeSizeCost: 11, .SizeAndLatencyCost: 17 } }, // 2 x 128-bit Op + extract/insert
4743 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 7 } },
4744 { .ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4745 { .ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4746 { .ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4747 { .ISD: ISD::UMULO, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 24, .LatencyCost: 26, .CodeSizeCost: 39, .SizeAndLatencyCost: 45 } },
4748 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 12, .CodeSizeCost: 15, .SizeAndLatencyCost: 20 } },
4749 { .ISD: ISD::UMULO, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 15, .CodeSizeCost: 23, .SizeAndLatencyCost: 28 } },
4750 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 12, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } },
4751 { .ISD: ISD::UMULO, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 14 } },
4752 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4753 { .ISD: ISD::UMULO, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 19, .CodeSizeCost: 35, .SizeAndLatencyCost: 37 } },
4754 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 19, .CodeSizeCost: 17, .SizeAndLatencyCost: 20 } },
4755 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4756 { .ISD: ISD::USUBSAT, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 15 } }, // 2 x 128-bit Op + extract/insert
4757 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 8 } }, // 2 x 128-bit Op + extract/insert
4758 { .ISD: ISD::USUBSAT, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4759 { .ISD: ISD::USUBSAT, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4760 { .ISD: ISD::USUBSAT, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // 2 x 128-bit Op + extract/insert
4761 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4762 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4763 { .ISD: ISD::FMAXNUM, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 10 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4764 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4765 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4766 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 7, .CodeSizeCost: 3, .SizeAndLatencyCost: 10 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4767 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtss
4768 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 21, .LatencyCost: 21, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtps
4769 { .ISD: ISD::FSQRT, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 42, .LatencyCost: 42, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtps
4770 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 27, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtsd
4771 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 27, .LatencyCost: 27, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vsqrtpd
4772 { .ISD: ISD::FSQRT, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 54, .LatencyCost: 54, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } }, // vsqrtpd
4773 { .ISD: ISD::VECTOR_SHUFFLE_VAR, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // vpaddq+vpermilpd
4774 { .ISD: ISD::VECTOR_SHUFFLE_VAR, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // vpermilps
4775 };
4776 static const CostKindTblEntry GFNICostTbl[] = {
4777 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4778 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // gf2p8affineqb
4779 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // gf2p8affineqb
4780 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } }, // gf2p8affineqb
4781 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4782 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4783 { .ISD: ISD::BITREVERSE, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4784 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4785 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4786 { .ISD: ISD::BITREVERSE, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4787 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4788 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4789 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4790 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 8, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4791 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4792 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 9, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } }, // gf2p8affineqb
4793 { .ISD: X86ISD::VROTLI, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4794 { .ISD: X86ISD::VROTLI, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4795 { .ISD: X86ISD::VROTLI, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 6, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // gf2p8affineqb
4796 };
4797 static const CostKindTblEntry GLMCostTbl[] = {
4798 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtss
4799 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 37, .LatencyCost: 41, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtps
4800 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 34, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtsd
4801 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 67, .LatencyCost: 71, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtpd
4802 };
4803 static const CostKindTblEntry SLMCostTbl[] = {
4804 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4805 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4806 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4807 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 20, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtss
4808 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 40, .LatencyCost: 41, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtps
4809 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 35, .LatencyCost: 35, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // sqrtsd
4810 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 70, .LatencyCost: 71, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } }, // sqrtpd
4811 };
4812 static const CostKindTblEntry SSE42CostTbl[] = {
4813 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // MAXSS + CMPUNORDSS + BLENDVPS
4814 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // MAXPS + CMPUNORDPS + BLENDVPS
4815 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } }, // MAXSD + CMPUNORDSD + BLENDVPD
4816 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // MAXPD + CMPUNORDPD + BLENDVPD
4817 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4818 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 18, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4819 };
4820 static const CostKindTblEntry SSE41CostTbl[] = {
4821 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // BLENDVPD(X,PSUBQ(0,X),X)
4822 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4823 { .ISD: ISD::MULHS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4824 { .ISD: ISD::MULHU, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4825 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 14, .CodeSizeCost: 17, .SizeAndLatencyCost: 21 } },
4826 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 8, .SizeAndLatencyCost: 10 } },
4827 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 19, .CodeSizeCost: 25, .SizeAndLatencyCost: 29 } },
4828 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 10, .SizeAndLatencyCost: 12 } },
4829 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4830 { .ISD: ISD::SMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4831 { .ISD: ISD::SMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4832 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
4833 { .ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4834 { .ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4835 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 11, .CodeSizeCost: 13, .SizeAndLatencyCost: 17 } },
4836 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 24, .CodeSizeCost: 13, .SizeAndLatencyCost: 19 } },
4837 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 9, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4838 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 22, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4839 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4840 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4841 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4842 { .ISD: ISD::USUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } },
4843 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4844 { .ISD: ISD::UMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4845 { .ISD: ISD::UMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4846 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 7 } },
4847 { .ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4848 { .ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4849 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 20, .CodeSizeCost: 15, .SizeAndLatencyCost: 20 } },
4850 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 12, .SizeAndLatencyCost: 18 } },
4851 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4852 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 18, .SizeAndLatencyCost: 20 } },
4853 };
4854 static const CostKindTblEntry SSSE3CostTbl[] = {
4855 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4856 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4857 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4858 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4859 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4860 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 11, .SizeAndLatencyCost: 21 } },
4861 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4862 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4863 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4864 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 5 } },
4865 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 28, .CodeSizeCost: 28, .SizeAndLatencyCost: 35 } },
4866 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 20, .CodeSizeCost: 22, .SizeAndLatencyCost: 28 } },
4867 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 17, .CodeSizeCost: 16, .SizeAndLatencyCost: 22 } },
4868 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 15, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4869 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 12, .SizeAndLatencyCost: 18 } },
4870 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 24, .CodeSizeCost: 16, .SizeAndLatencyCost: 22 } },
4871 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 18, .CodeSizeCost: 14, .SizeAndLatencyCost: 20 } },
4872 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 10, .SizeAndLatencyCost: 16 } },
4873 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 25, .CodeSizeCost: 15, .SizeAndLatencyCost: 22 } },
4874 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 26, .CodeSizeCost: 19, .SizeAndLatencyCost: 25 } },
4875 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 20, .CodeSizeCost: 17, .SizeAndLatencyCost: 23 } },
4876 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 16, .CodeSizeCost: 13, .SizeAndLatencyCost: 19 } },
4877 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4878 { .ISD: ISD::VECTOR_SHUFFLE_VAR, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 3, .SizeAndLatencyCost: 5 } }, // pmul+padd+pshufb
4879 { .ISD: ISD::VECTOR_SHUFFLE_VAR, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // pshufb
4880 };
4881 static const CostKindTblEntry SSE2CostTbl[] = {
4882 { .ISD: ISD::ABS, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4883 { .ISD: ISD::ABS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4 } },
4884 { .ISD: ISD::ABS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4885 { .ISD: ISD::ABS, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4886 { .ISD: ISD::BITREVERSE, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 32, .SizeAndLatencyCost: 32 } },
4887 { .ISD: ISD::BITREVERSE, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 30, .SizeAndLatencyCost: 30 } },
4888 { .ISD: ISD::BITREVERSE, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 20, .CodeSizeCost: 25, .SizeAndLatencyCost: 25 } },
4889 { .ISD: ISD::BITREVERSE, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 11, .LatencyCost: 12, .CodeSizeCost: 21, .SizeAndLatencyCost: 21 } },
4890 { .ISD: ISD::BSWAP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 6, .CodeSizeCost: 11, .SizeAndLatencyCost: 11 } },
4891 { .ISD: ISD::BSWAP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 9, .SizeAndLatencyCost: 9 } },
4892 { .ISD: ISD::BSWAP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } },
4893 { .ISD: ISD::CTLZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 45, .CodeSizeCost: 36, .SizeAndLatencyCost: 38 } },
4894 { .ISD: ISD::CTLZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 45, .CodeSizeCost: 38, .SizeAndLatencyCost: 40 } },
4895 { .ISD: ISD::CTLZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 38, .CodeSizeCost: 32, .SizeAndLatencyCost: 34 } },
4896 { .ISD: ISD::CTLZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 39, .CodeSizeCost: 29, .SizeAndLatencyCost: 32 } },
4897 { .ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 26, .CodeSizeCost: 16, .SizeAndLatencyCost: 18 } },
4898 { .ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 15, .LatencyCost: 29, .CodeSizeCost: 21, .SizeAndLatencyCost: 23 } },
4899 { .ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 25, .CodeSizeCost: 18, .SizeAndLatencyCost: 20 } },
4900 { .ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 21, .CodeSizeCost: 14, .SizeAndLatencyCost: 16 } },
4901 { .ISD: ISD::CTTZ, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 14, .LatencyCost: 28, .CodeSizeCost: 19, .SizeAndLatencyCost: 21 } },
4902 { .ISD: ISD::CTTZ, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 18, .LatencyCost: 31, .CodeSizeCost: 24, .SizeAndLatencyCost: 26 } },
4903 { .ISD: ISD::CTTZ, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 27, .CodeSizeCost: 21, .SizeAndLatencyCost: 23 } },
4904 { .ISD: ISD::CTTZ, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 23, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
4905 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 13, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } }, // movmsk seq
4906 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 11 } }, // movmsk seq
4907 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 13, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } }, // movmsk seq
4908 { .ISD: ISD::MASK_BEFOREFIRST, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 13, .CodeSizeCost: 11, .SizeAndLatencyCost: 13 } }, // movmsk seq
4909 { .ISD: ISD::MULHS, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 11, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4910 { .ISD: ISD::MULHS, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4911 { .ISD: ISD::MULHU, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4912 { .ISD: ISD::MULHU, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 5, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4913 { .ISD: ISD::SADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 12, .LatencyCost: 14, .CodeSizeCost: 24, .SizeAndLatencyCost: 24 } },
4914 { .ISD: ISD::SADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 11, .CodeSizeCost: 11, .SizeAndLatencyCost: 12 } },
4915 { .ISD: ISD::SADDSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4916 { .ISD: ISD::SADDSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4917 { .ISD: ISD::SMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4918 { .ISD: ISD::SMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4919 { .ISD: ISD::SMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4920 { .ISD: ISD::SMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4921 { .ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4922 { .ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4923 { .ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4924 { .ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5 } },
4925 { .ISD: ISD::SMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 30, .LatencyCost: 33, .CodeSizeCost: 13, .SizeAndLatencyCost: 23 } },
4926 { .ISD: ISD::SMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 20, .LatencyCost: 24, .CodeSizeCost: 23, .SizeAndLatencyCost: 23 } },
4927 { .ISD: ISD::SMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 10, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4928 { .ISD: ISD::SMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 23, .CodeSizeCost: 24, .SizeAndLatencyCost: 25 } },
4929 { .ISD: ISD::SSUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 16, .LatencyCost: 19, .CodeSizeCost: 31, .SizeAndLatencyCost: 31 } },
4930 { .ISD: ISD::SSUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 14, .CodeSizeCost: 12, .SizeAndLatencyCost: 13 } },
4931 { .ISD: ISD::SSUBSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4932 { .ISD: ISD::SSUBSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4933 { .ISD: ISD::UADDSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 13, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4934 { .ISD: ISD::UADDSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4935 { .ISD: ISD::UADDSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4936 { .ISD: ISD::UADDSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4937 { .ISD: ISD::UMAX, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4938 { .ISD: ISD::UMAX, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4939 { .ISD: ISD::UMAX, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4940 { .ISD: ISD::UMAX, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4941 { .ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
4942 { .ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 8 } },
4943 { .ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } },
4944 { .ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4945 { .ISD: ISD::UMULO, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 30, .LatencyCost: 33, .CodeSizeCost: 15, .SizeAndLatencyCost: 29 } },
4946 { .ISD: ISD::UMULO, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 19, .LatencyCost: 22, .CodeSizeCost: 14, .SizeAndLatencyCost: 18 } },
4947 { .ISD: ISD::UMULO, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4948 { .ISD: ISD::UMULO, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 13, .LatencyCost: 19, .CodeSizeCost: 20, .SizeAndLatencyCost: 20 } },
4949 { .ISD: ISD::USUBSAT, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 10, .CodeSizeCost: 14, .SizeAndLatencyCost: 14 } },
4950 { .ISD: ISD::USUBSAT, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4951 { .ISD: ISD::USUBSAT, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4952 { .ISD: ISD::USUBSAT, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4953 { .ISD: ISD::FMAXNUM, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4954 { .ISD: ISD::FMAXNUM, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4955 { .ISD: ISD::FSQRT, .Type: MVT::f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4956 { .ISD: ISD::FSQRT, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 32, .LatencyCost: 32, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // Nehalem from http://www.agner.org/
4957 };
4958 static const CostKindTblEntry SSE1CostTbl[] = {
4959 { .ISD: ISD::FMAXNUM, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 7, .SizeAndLatencyCost: 7 } },
4960 { .ISD: ISD::FMAXNUM, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
4961 { .ISD: ISD::FSQRT, .Type: MVT::f32, .Cost: { .RecipThroughputCost: 28, .LatencyCost: 30, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
4962 { .ISD: ISD::FSQRT, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 56, .LatencyCost: 56, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // Pentium III from http://www.agner.org/
4963 };
4964 static const CostKindTblEntry BMI64CostTbl[] = { // 64-bit targets
4965 { .ISD: ISD::CTTZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4966 };
4967 static const CostKindTblEntry BMI32CostTbl[] = { // 32 or 64-bit targets
4968 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4969 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4970 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4971 };
4972 static const CostKindTblEntry LZCNT64CostTbl[] = { // 64-bit targets
4973 { .ISD: ISD::CTLZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4974 };
4975 static const CostKindTblEntry LZCNT32CostTbl[] = { // 32 or 64-bit targets
4976 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4977 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4978 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
4979 };
4980 static const CostKindTblEntry POPCNT64CostTbl[] = { // 64-bit targets
4981 { .ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // popcnt
4982 };
4983 static const CostKindTblEntry POPCNT32CostTbl[] = { // 32 or 64-bit targets
4984 { .ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } }, // popcnt
4985 { .ISD: ISD::CTPOP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // popcnt(zext())
4986 { .ISD: ISD::CTPOP, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // popcnt(zext())
4987 };
4988 static const CostKindTblEntry PCLMULCostTbl[] = {
4989 { .ISD: ISD::CLMUL, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+2xPCLMUL+unpack
4990 { .ISD: ISD::CLMUL, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 18, .CodeSizeCost: 12, .SizeAndLatencyCost: 16 } }, // MOV+4xPCLMUL+unpack
4991 { .ISD: ISD::CLMUL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4992 { .ISD: ISD::CLMUL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4993 { .ISD: ISD::CLMUL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4994 { .ISD: ISD::CLMUL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 12, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } }, // MOV+PCLMUL+MOV
4995 };
4996 static const CostKindTblEntry X64CostTbl[] = { // 64-bit targets
4997 { .ISD: ISD::ABS, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+CMOV
4998 { .ISD: ISD::BITREVERSE, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 12, .CodeSizeCost: 20, .SizeAndLatencyCost: 22 } },
4999 { .ISD: ISD::BSWAP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } },
5000 { .ISD: ISD::CTLZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
5001 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
5002 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
5003 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 3 } }, // MOV+BSR+XOR
5004 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i64,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5005 { .ISD: ISD::CTTZ, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5006 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5007 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5008 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // MOV+BSF
5009 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i64,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5010 { .ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 10, .LatencyCost: 6, .CodeSizeCost: 19, .SizeAndLatencyCost: 19 } },
5011 { .ISD: ISD::ROTL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5012 { .ISD: ISD::ROTR, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5013 { .ISD: X86ISD::VROTLI, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5014 { .ISD: ISD::FSHL, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 4 } },
5015 { .ISD: ISD::SADDSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5016 { .ISD: ISD::SSUBSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5017 { .ISD: ISD::UADDSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5018 { .ISD: ISD::USUBSAT, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5019 { .ISD: ISD::SMAX, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5020 { .ISD: ISD::SMIN, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5021 { .ISD: ISD::UMAX, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5022 { .ISD: ISD::UMIN, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 3, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5023 { .ISD: ISD::SADDO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5024 { .ISD: ISD::UADDO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5025 { .ISD: ISD::SMULO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5026 { .ISD: ISD::UMULO, .Type: MVT::i64, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5027 };
5028 static const CostKindTblEntry X86CostTbl[] = { // 32 or 64-bit targets
5029 { .ISD: ISD::ABS, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA or SUB+CMOV
5030 { .ISD: ISD::ABS, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA or SUB+CMOV
5031 { .ISD: ISD::ABS, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 3 } }, // SUB+XOR+SRA
5032 { .ISD: ISD::BITREVERSE, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
5033 { .ISD: ISD::BITREVERSE, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 12, .CodeSizeCost: 17, .SizeAndLatencyCost: 19 } },
5034 { .ISD: ISD::BITREVERSE, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 9, .CodeSizeCost: 13, .SizeAndLatencyCost: 14 } },
5035 { .ISD: ISD::BSWAP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5036 { .ISD: ISD::BSWAP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // ROL
5037 { .ISD: ISD::CTLZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // BSR+XOR or BSR+XOR+CMOV
5038 { .ISD: ISD::CTLZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 5 } }, // BSR+XOR or BSR+XOR+CMOV
5039 { .ISD: ISD::CTLZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6 } }, // BSR+XOR or BSR+XOR+CMOV
5040 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i32,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5041 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2 } }, // BSR+XOR
5042 { .ISD: ISD::CTLZ_ZERO_POISON,.Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // BSR+XOR
5043 { .ISD: ISD::CTTZ, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5044 { .ISD: ISD::CTTZ, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5045 { .ISD: ISD::CTTZ, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } }, // TEST+BSF+CMOV/BRANCH
5046 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i32,.Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5047 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5048 { .ISD: ISD::CTTZ_ZERO_POISON,.Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 1, .SizeAndLatencyCost: 2 } }, // BSF
5049 { .ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 8, .LatencyCost: 7, .CodeSizeCost: 15, .SizeAndLatencyCost: 15 } },
5050 { .ISD: ISD::CTPOP, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 8, .CodeSizeCost: 17, .SizeAndLatencyCost: 17 } },
5051 { .ISD: ISD::CTPOP, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6 } },
5052 { .ISD: ISD::ROTL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5053 { .ISD: ISD::ROTL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5054 { .ISD: ISD::ROTL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5055 { .ISD: ISD::ROTR, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5056 { .ISD: ISD::ROTR, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5057 { .ISD: ISD::ROTR, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
5058 { .ISD: X86ISD::VROTLI, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5059 { .ISD: X86ISD::VROTLI, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5060 { .ISD: X86ISD::VROTLI, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1 } },
5061 { .ISD: ISD::FSHL, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 1, .SizeAndLatencyCost: 4 } },
5062 { .ISD: ISD::FSHL, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 5 } },
5063 { .ISD: ISD::FSHL, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 5 } },
5064 { .ISD: ISD::SADDSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 6, .SizeAndLatencyCost: 9 } },
5065 { .ISD: ISD::SADDSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5066 { .ISD: ISD::SADDSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5067 { .ISD: ISD::SSUBSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5068 { .ISD: ISD::SSUBSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 7, .SizeAndLatencyCost: 10 } },
5069 { .ISD: ISD::SSUBSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 4, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 11 } },
5070 { .ISD: ISD::UADDSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5071 { .ISD: ISD::UADDSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5072 { .ISD: ISD::UADDSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
5073 { .ISD: ISD::USUBSAT, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5074 { .ISD: ISD::USUBSAT, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 7 } },
5075 { .ISD: ISD::USUBSAT, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 8 } },
5076 { .ISD: ISD::SMAX, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5077 { .ISD: ISD::SMAX, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5078 { .ISD: ISD::SMAX, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5079 { .ISD: ISD::SMIN, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5080 { .ISD: ISD::SMIN, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5081 { .ISD: ISD::SMIN, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5082 { .ISD: ISD::UMAX, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5083 { .ISD: ISD::UMAX, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5084 { .ISD: ISD::UMAX, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5085 { .ISD: ISD::UMIN, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 3 } },
5086 { .ISD: ISD::UMIN, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5087 { .ISD: ISD::UMIN, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 1, .LatencyCost: 4, .CodeSizeCost: 2, .SizeAndLatencyCost: 4 } },
5088 { .ISD: ISD::SADDO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5089 { .ISD: ISD::SADDO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5090 { .ISD: ISD::SADDO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5091 { .ISD: ISD::UADDO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5092 { .ISD: ISD::UADDO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5093 { .ISD: ISD::UADDO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5094 { .ISD: ISD::SMULO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5095 { .ISD: ISD::SMULO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5096 { .ISD: ISD::SMULO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5097 { .ISD: ISD::UMULO, .Type: MVT::i32, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 8 } },
5098 { .ISD: ISD::UMULO, .Type: MVT::i16, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 9 } },
5099 { .ISD: ISD::UMULO, .Type: MVT::i8, .Cost: { .RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 6 } },
5100 };
5101
5102 Type *RetTy = ICA.getReturnType();
5103 Type *OpTy = RetTy;
5104 Intrinsic::ID IID = ICA.getID();
5105 unsigned ISD = ISD::DELETED_NODE;
5106 switch (IID) {
5107 default:
5108 break;
5109 case Intrinsic::abs:
5110 ISD = ISD::ABS;
5111 break;
5112 case Intrinsic::bitreverse:
5113 ISD = ISD::BITREVERSE;
5114 break;
5115 case Intrinsic::bswap:
5116 ISD = ISD::BSWAP;
5117 break;
5118 case Intrinsic::ctlz:
5119 ISD = ISD::CTLZ;
5120 break;
5121 case Intrinsic::ctpop:
5122 ISD = ISD::CTPOP;
5123 break;
5124 case Intrinsic::cttz:
5125 ISD = ISD::CTTZ;
5126 break;
5127 case Intrinsic::fshl:
5128 ISD = ISD::FSHL;
5129 if (!ICA.isTypeBasedOnly()) {
5130 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5131 if (Args[0] == Args[1]) {
5132 ISD = ISD::ROTL;
5133 // Handle uniform constant rotation amounts.
5134 // TODO: Handle funnel-shift cases.
5135 const APInt *Amt;
5136 if (Args[2] &&
5137 PatternMatch::match(V: Args[2], P: PatternMatch::m_APIntAllowPoison(Res&: Amt)))
5138 ISD = X86ISD::VROTLI;
5139 }
5140 }
5141 break;
5142 case Intrinsic::fshr:
5143 // FSHR has same costs so don't duplicate.
5144 ISD = ISD::FSHL;
5145 if (!ICA.isTypeBasedOnly()) {
5146 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5147 if (Args[0] == Args[1]) {
5148 ISD = ISD::ROTR;
5149 // Handle uniform constant rotation amount.
5150 // TODO: Handle funnel-shift cases.
5151 const APInt *Amt;
5152 if (Args[2] &&
5153 PatternMatch::match(V: Args[2], P: PatternMatch::m_APIntAllowPoison(Res&: Amt)))
5154 ISD = X86ISD::VROTLI;
5155 }
5156 }
5157 break;
5158 case Intrinsic::lrint:
5159 case Intrinsic::llrint: {
5160 // X86 can use the CVTP2SI instructions to lower lrint/llrint calls, which
5161 // have the same costs as the CVTTP2SI (fptosi) instructions
5162 const SmallVectorImpl<Type *> &ArgTys = ICA.getArgTypes();
5163 return getCastInstrCost(Opcode: Instruction::FPToSI, Dst: RetTy, Src: ArgTys[0],
5164 CCH: TTI::CastContextHint::None, CostKind);
5165 }
5166 case Intrinsic::mask_beforefirst:
5167 ISD = ISD::MASK_BEFOREFIRST;
5168 break;
5169 case Intrinsic::maxnum:
5170 case Intrinsic::minnum:
5171 // FMINNUM has same costs so don't duplicate.
5172 ISD = ISD::FMAXNUM;
5173 break;
5174 case Intrinsic::sadd_sat:
5175 ISD = ISD::SADDSAT;
5176 break;
5177 case Intrinsic::smax:
5178 ISD = ISD::SMAX;
5179 break;
5180 case Intrinsic::smin:
5181 ISD = ISD::SMIN;
5182 break;
5183 case Intrinsic::smulh:
5184 ISD = ISD::MULHS;
5185 break;
5186 case Intrinsic::ssub_sat:
5187 ISD = ISD::SSUBSAT;
5188 break;
5189 case Intrinsic::uadd_sat:
5190 ISD = ISD::UADDSAT;
5191 break;
5192 case Intrinsic::umax:
5193 ISD = ISD::UMAX;
5194 break;
5195 case Intrinsic::umin:
5196 ISD = ISD::UMIN;
5197 break;
5198 case Intrinsic::usub_sat:
5199 ISD = ISD::USUBSAT;
5200 break;
5201 case Intrinsic::umulh:
5202 ISD = ISD::MULHU;
5203 break;
5204 case Intrinsic::sqrt:
5205 // The estimate sequence is costed with the division it is folded into.
5206 if (isFoldedIntoRsqrtEstimate(V: ICA.getInst(), TLI: *TLI,
5207 VT: getTypeLegalizationCost(Ty: RetTy).second,
5208 CostKind))
5209 return TTI::TCC_Free;
5210 ISD = ISD::FSQRT;
5211 break;
5212 case Intrinsic::sadd_with_overflow:
5213 case Intrinsic::ssub_with_overflow:
5214 // SSUBO has same costs so don't duplicate.
5215 ISD = ISD::SADDO;
5216 OpTy = RetTy->getContainedType(i: 0);
5217 break;
5218 case Intrinsic::uadd_with_overflow:
5219 case Intrinsic::usub_with_overflow:
5220 // USUBO has same costs so don't duplicate.
5221 ISD = ISD::UADDO;
5222 OpTy = RetTy->getContainedType(i: 0);
5223 break;
5224 case Intrinsic::smul_with_overflow:
5225 ISD = ISD::SMULO;
5226 OpTy = RetTy->getContainedType(i: 0);
5227 break;
5228 case Intrinsic::umul_with_overflow:
5229 ISD = ISD::UMULO;
5230 OpTy = RetTy->getContainedType(i: 0);
5231 break;
5232 case Intrinsic::vector_shuffle:
5233 // TODO: mask op zext/trunc costs.
5234 ISD = ISD::VECTOR_SHUFFLE_VAR;
5235 break;
5236 case Intrinsic::clmul:
5237 ISD = ISD::CLMUL;
5238 break;
5239 }
5240
5241 if (ISD != ISD::DELETED_NODE) {
5242 auto adjustTableCost = [&](int ISD, unsigned Cost,
5243 std::pair<InstructionCost, MVT> LT,
5244 FastMathFlags FMF) -> InstructionCost {
5245 InstructionCost LegalizationCost = LT.first;
5246 MVT MTy = LT.second;
5247
5248 // If there are no NANs to deal with, then these are reduced to a
5249 // single MIN** or MAX** instruction instead of the MIN/CMP/SELECT that we
5250 // assume is used in the non-fast case.
5251 if (ISD == ISD::FMAXNUM || ISD == ISD::FMINNUM) {
5252 if (FMF.noNaNs())
5253 return LegalizationCost * 1;
5254 }
5255
5256 // For cases where some ops can be folded into a load/store, assume free.
5257 if (MTy.isScalarInteger()) {
5258 if (ISD == ISD::BSWAP && ST->hasMOVBE() && ST->hasFastMOVBE()) {
5259 if (const Instruction *II = ICA.getInst()) {
5260 if (II->hasOneUse() && isa<StoreInst>(Val: II->user_back()))
5261 return TTI::TCC_Free;
5262 if (auto *LI = dyn_cast<LoadInst>(Val: II->getOperand(i: 0))) {
5263 if (LI->hasOneUse())
5264 return TTI::TCC_Free;
5265 }
5266 }
5267 }
5268 }
5269
5270 return LegalizationCost * (int)Cost;
5271 };
5272
5273 // Legalize the type.
5274 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: OpTy);
5275 MVT MTy = LT.second;
5276
5277 // Without BMI/LZCNT see if we're only looking for a *_ZERO_POISON cost.
5278 if (((ISD == ISD::CTTZ && !ST->hasBMI()) ||
5279 (ISD == ISD::CTLZ && !ST->hasLZCNT())) &&
5280 !MTy.isVector() && !ICA.isTypeBasedOnly()) {
5281 const SmallVectorImpl<const Value *> &Args = ICA.getArgs();
5282 if (auto *Cst = dyn_cast<ConstantInt>(Val: Args[1]))
5283 if (Cst->isAllOnesValue())
5284 ISD =
5285 ISD == ISD::CTTZ ? ISD::CTTZ_ZERO_POISON : ISD::CTLZ_ZERO_POISON;
5286 }
5287
5288 // FSQRT is a single instruction.
5289 if (ISD == ISD::FSQRT && CostKind == TTI::TCK_CodeSize)
5290 return LT.first;
5291
5292 if (ST->useGLMDivSqrtCosts())
5293 if (const auto *Entry = CostTableLookup(Table: GLMCostTbl, ISD, Ty: MTy))
5294 if (auto KindCost = Entry->Cost[CostKind])
5295 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5296
5297 if (ST->useSLMArithCosts())
5298 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
5299 if (auto KindCost = Entry->Cost[CostKind])
5300 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5301
5302 if (ST->hasVBMI2())
5303 if (const auto *Entry = CostTableLookup(Table: AVX512VBMI2CostTbl, ISD, Ty: MTy))
5304 if (auto KindCost = Entry->Cost[CostKind])
5305 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5306
5307 if (ST->hasBITALG())
5308 if (const auto *Entry = CostTableLookup(Table: AVX512BITALGCostTbl, ISD, Ty: MTy))
5309 if (auto KindCost = Entry->Cost[CostKind])
5310 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5311
5312 if (ST->hasVPOPCNTDQ())
5313 if (const auto *Entry = CostTableLookup(Table: AVX512VPOPCNTDQCostTbl, ISD, Ty: MTy))
5314 if (auto KindCost = Entry->Cost[CostKind])
5315 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5316
5317 if (ST->hasGFNI())
5318 if (const auto *Entry = CostTableLookup(Table: GFNICostTbl, ISD, Ty: MTy))
5319 if (auto KindCost = Entry->Cost[CostKind])
5320 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5321
5322 if (ST->hasCDI())
5323 if (const auto *Entry = CostTableLookup(Table: AVX512CDCostTbl, ISD, Ty: MTy))
5324 if (auto KindCost = Entry->Cost[CostKind])
5325 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5326
5327 if (ST->hasBWI())
5328 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
5329 if (auto KindCost = Entry->Cost[CostKind])
5330 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5331
5332 if (ST->hasAVX512())
5333 if (const auto *Entry = CostTableLookup(Table: AVX512CostTbl, ISD, Ty: MTy))
5334 if (auto KindCost = Entry->Cost[CostKind])
5335 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5336
5337 if (ST->hasXOP())
5338 if (const auto *Entry = CostTableLookup(Table: XOPCostTbl, ISD, Ty: MTy))
5339 if (auto KindCost = Entry->Cost[CostKind])
5340 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5341
5342 if (ST->hasAVX2())
5343 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
5344 if (auto KindCost = Entry->Cost[CostKind])
5345 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5346
5347 if (ST->hasAVX())
5348 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
5349 if (auto KindCost = Entry->Cost[CostKind])
5350 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5351
5352 if (ST->hasSSE42())
5353 if (const auto *Entry = CostTableLookup(Table: SSE42CostTbl, ISD, Ty: MTy))
5354 if (auto KindCost = Entry->Cost[CostKind])
5355 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5356
5357 if (ST->hasSSE41())
5358 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
5359 if (auto KindCost = Entry->Cost[CostKind])
5360 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5361
5362 if (ST->hasSSSE3())
5363 if (const auto *Entry = CostTableLookup(Table: SSSE3CostTbl, ISD, Ty: MTy))
5364 if (auto KindCost = Entry->Cost[CostKind])
5365 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5366
5367 if (ST->hasSSE2())
5368 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
5369 if (auto KindCost = Entry->Cost[CostKind])
5370 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5371
5372 if (ST->hasSSE1())
5373 if (const auto *Entry = CostTableLookup(Table: SSE1CostTbl, ISD, Ty: MTy))
5374 if (auto KindCost = Entry->Cost[CostKind])
5375 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5376
5377 if (ST->hasBMI()) {
5378 if (ST->is64Bit())
5379 if (const auto *Entry = CostTableLookup(Table: BMI64CostTbl, ISD, Ty: MTy))
5380 if (auto KindCost = Entry->Cost[CostKind])
5381 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5382
5383 if (const auto *Entry = CostTableLookup(Table: BMI32CostTbl, ISD, Ty: MTy))
5384 if (auto KindCost = Entry->Cost[CostKind])
5385 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5386 }
5387
5388 if (ST->hasLZCNT()) {
5389 if (ST->is64Bit())
5390 if (const auto *Entry = CostTableLookup(Table: LZCNT64CostTbl, ISD, Ty: MTy))
5391 if (auto KindCost = Entry->Cost[CostKind])
5392 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5393
5394 if (const auto *Entry = CostTableLookup(Table: LZCNT32CostTbl, ISD, Ty: MTy))
5395 if (auto KindCost = Entry->Cost[CostKind])
5396 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5397 }
5398
5399 if (ST->hasPOPCNT()) {
5400 if (ST->is64Bit())
5401 if (const auto *Entry = CostTableLookup(Table: POPCNT64CostTbl, ISD, Ty: MTy))
5402 if (auto KindCost = Entry->Cost[CostKind])
5403 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5404
5405 if (const auto *Entry = CostTableLookup(Table: POPCNT32CostTbl, ISD, Ty: MTy))
5406 if (auto KindCost = Entry->Cost[CostKind])
5407 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5408 }
5409
5410 // FIXME: PCLMUL w/ AVX/AVX512 and VPCLMULQDQ are not handled properly.
5411 if (ST->hasPCLMUL())
5412 if (const auto *Entry = CostTableLookup(Table: PCLMULCostTbl, ISD, Ty: MTy))
5413 if (auto KindCost = Entry->Cost[CostKind])
5414 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5415
5416 if (ST->is64Bit())
5417 if (const auto *Entry = CostTableLookup(Table: X64CostTbl, ISD, Ty: MTy))
5418 if (auto KindCost = Entry->Cost[CostKind])
5419 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5420
5421 if (const auto *Entry = CostTableLookup(Table: X86CostTbl, ISD, Ty: MTy))
5422 if (auto KindCost = Entry->Cost[CostKind])
5423 return adjustTableCost(Entry->ISD, *KindCost, LT, ICA.getFlags());
5424
5425 // Without arg data, we need to compute the expanded costs of custom lowered
5426 // intrinsics to prevent use of the (very low) default costs.
5427 if (ICA.isTypeBasedOnly() &&
5428 (IID == Intrinsic::fshl || IID == Intrinsic::fshr)) {
5429 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
5430 InstructionCost Cost = 0;
5431 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Or, Ty: RetTy, CostKind);
5432 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Sub, Ty: RetTy, CostKind);
5433 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::Shl, Ty: RetTy, CostKind);
5434 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::LShr, Ty: RetTy, CostKind);
5435 Cost += getArithmeticInstrCost(Opcode: BinaryOperator::And, Ty: RetTy, CostKind);
5436 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy,
5437 VecPred: CmpInst::ICMP_EQ, CostKind);
5438 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
5439 VecPred: CmpInst::ICMP_EQ, CostKind);
5440 return Cost;
5441 }
5442 }
5443
5444 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
5445}
5446
5447InstructionCost X86TTIImpl::getVectorInstrCost(
5448 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
5449 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
5450 static const CostTblEntry SLMCostTbl[] = {
5451 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i8, .Cost: 4 },
5452 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i16, .Cost: 4 },
5453 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i32, .Cost: 4 },
5454 { .ISD: ISD::EXTRACT_VECTOR_ELT, .Type: MVT::i64, .Cost: 7 }
5455 };
5456
5457 assert(Val->isVectorTy() && "This must be a vector type");
5458 auto *VT = cast<VectorType>(Val);
5459 if (VT->isScalableTy())
5460 return InstructionCost::getInvalid();
5461
5462 Type *ScalarType = Val->getScalarType();
5463 InstructionCost RegisterFileMoveCost = 0;
5464
5465 // Non-immediate extraction/insertion can be handled as a sequence of
5466 // aliased loads+stores via the stack.
5467 if (Index == -1U && (Opcode == Instruction::ExtractElement ||
5468 Opcode == Instruction::InsertElement)) {
5469 // TODO: On some SSE41+ targets, we expand to cmp+splat+select patterns:
5470 // inselt N0, N1, N2 --> select (SplatN2 == {0,1,2...}) ? SplatN1 : N0.
5471
5472 // TODO: Move this to BasicTTIImpl.h? We'd need better gep + index handling.
5473 assert(isa<FixedVectorType>(Val) && "Fixed vector type expected");
5474 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
5475 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
5476
5477 // Extract - store vector to stack, load scalar.
5478 if (Opcode == Instruction::ExtractElement) {
5479 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
5480 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
5481 CostKind);
5482 }
5483 // Insert - store vector to stack, store scalar, load vector.
5484 if (Opcode == Instruction::InsertElement) {
5485 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
5486 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
5487 CostKind) +
5488 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind);
5489 }
5490 }
5491
5492 if (Index != -1U && (Opcode == Instruction::ExtractElement ||
5493 Opcode == Instruction::InsertElement)) {
5494 // Extraction of vXi1 elements are now efficiently handled by MOVMSK.
5495 if (Opcode == Instruction::ExtractElement &&
5496 ScalarType->getScalarSizeInBits() == 1 &&
5497 cast<FixedVectorType>(Val)->getNumElements() > 1)
5498 return 1;
5499
5500 // Legalize the type.
5501 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
5502
5503 // This type is legalized to a scalar type.
5504 if (!LT.second.isVector())
5505 return TTI::TCC_Free;
5506
5507 // The type may be split. Normalize the index to the new type.
5508 unsigned SizeInBits = LT.second.getSizeInBits();
5509 unsigned NumElts = LT.second.getVectorNumElements();
5510 unsigned SubNumElts = NumElts;
5511 Index = Index % NumElts;
5512
5513 // For >128-bit vectors, we need to extract higher 128-bit subvectors.
5514 // For inserts, we also need to insert the subvector back.
5515 if (SizeInBits > 128) {
5516 assert((SizeInBits % 128) == 0 && "Illegal vector");
5517 unsigned NumSubVecs = SizeInBits / 128;
5518 SubNumElts = NumElts / NumSubVecs;
5519 if (SubNumElts <= Index) {
5520 RegisterFileMoveCost += (Opcode == Instruction::InsertElement ? 2 : 1);
5521 Index %= SubNumElts;
5522 }
5523 }
5524
5525 MVT MScalarTy = LT.second.getScalarType();
5526 auto IsCheapPInsrPExtrInsertPS = [&]() {
5527 // Assume pinsr/pextr XMM <-> GPR is relatively cheap on all targets.
5528 // Inserting f32 into index0 is just movss.
5529 // Also, assume insertps is relatively cheap on all >= SSE41 targets.
5530 return (MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5531 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5532 (MScalarTy == MVT::f32 && ST->hasSSE1() && Index == 0 &&
5533 Opcode == Instruction::InsertElement) ||
5534 (MScalarTy == MVT::f32 && ST->hasSSE41() &&
5535 Opcode == Instruction::InsertElement);
5536 };
5537
5538 if (Index == 0) {
5539 // Floating point scalars are already located in index #0.
5540 // Many insertions to #0 can fold away for scalar fp-ops, so let's assume
5541 // true for all.
5542 if (ScalarType->isFloatingPointTy() &&
5543 (Opcode != Instruction::InsertElement || !Op0 ||
5544 isa<UndefValue>(Val: Op0)))
5545 return RegisterFileMoveCost;
5546
5547 if (Opcode == Instruction::InsertElement &&
5548 isa_and_nonnull<UndefValue>(Val: Op0)) {
5549 // Consider the gather cost to be cheap.
5550 if (isa_and_nonnull<LoadInst>(Val: Op1))
5551 return RegisterFileMoveCost;
5552 if (!IsCheapPInsrPExtrInsertPS()) {
5553 // mov constant-to-GPR + movd/movq GPR -> XMM.
5554 if (isa_and_nonnull<Constant>(Val: Op1) && Op1->getType()->isIntegerTy())
5555 return 2 + RegisterFileMoveCost;
5556 // Assume movd/movq GPR -> XMM is relatively cheap on all targets.
5557 return 1 + RegisterFileMoveCost;
5558 }
5559 }
5560
5561 // Assume movd/movq XMM -> GPR is relatively cheap on all targets.
5562 if (ScalarType->isIntegerTy() && Opcode == Instruction::ExtractElement)
5563 return 1 + RegisterFileMoveCost;
5564 }
5565
5566 int ISD = TLI->InstructionOpcodeToISD(Opcode);
5567 assert(ISD && "Unexpected vector opcode");
5568 if (ST->useSLMArithCosts())
5569 if (auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MScalarTy))
5570 return Entry->Cost + RegisterFileMoveCost;
5571
5572 // Consider cheap cases.
5573 if (IsCheapPInsrPExtrInsertPS())
5574 return 1 + RegisterFileMoveCost;
5575
5576 // For extractions we just need to shuffle the element to index 0, which
5577 // should be very cheap (assume cost = 1). For insertions we need to shuffle
5578 // the elements to its destination. In both cases we must handle the
5579 // subvector move(s).
5580 // If the vector type is already less than 128-bits then don't reduce it.
5581 // TODO: Under what circumstances should we shuffle using the full width?
5582 InstructionCost ShuffleCost = 1;
5583 if (Opcode == Instruction::InsertElement) {
5584 auto *SubTy = cast<VectorType>(Val);
5585 EVT VT = TLI->getValueType(DL, Ty: Val);
5586 if (VT.getScalarType() != MScalarTy || VT.getSizeInBits() >= 128)
5587 SubTy = FixedVectorType::get(ElementType: ScalarType, NumElts: SubNumElts);
5588 ShuffleCost = getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SubTy, SrcTy: SubTy,
5589 CostKind, Mask: {}, Index: 0, SubTp: SubTy);
5590 }
5591 int IntOrFpCost = ScalarType->isFloatingPointTy() ? 0 : 1;
5592 return ShuffleCost + IntOrFpCost + RegisterFileMoveCost;
5593 }
5594
5595 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
5596 VIC) +
5597 RegisterFileMoveCost;
5598}
5599
5600InstructionCost X86TTIImpl::getScalarizationOverhead(
5601 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
5602 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
5603 TTI::VectorInstrContext VIC) const {
5604 assert(DemandedElts.getBitWidth() ==
5605 cast<FixedVectorType>(Ty)->getNumElements() &&
5606 "Vector size mismatch");
5607
5608 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5609 MVT MScalarTy = LT.second.getScalarType();
5610 unsigned LegalVectorBitWidth = LT.second.getSizeInBits();
5611 InstructionCost Cost = 0;
5612 // The square roots, folded with the divisions into the estimate, are
5613 // computed separately to be inserted.
5614 if (Insert && !VL.empty())
5615 for (unsigned I : seq(Size: DemandedElts.getBitWidth()))
5616 if (DemandedElts[I] &&
5617 isFoldedIntoRsqrtEstimate(V: VL[I], TLI: *TLI, VT: MScalarTy, CostKind))
5618 Cost += getIntrinsicInstrCost(
5619 ICA: IntrinsicCostAttributes(Intrinsic::sqrt, VL[I]->getType(),
5620 {VL[I]->getType()}),
5621 CostKind);
5622
5623 constexpr unsigned LaneBitWidth = 128;
5624 assert((LegalVectorBitWidth < LaneBitWidth ||
5625 (LegalVectorBitWidth % LaneBitWidth) == 0) &&
5626 "Illegal vector");
5627
5628 const int NumLegalVectors = LT.first.getValue();
5629 assert(NumLegalVectors >= 0 && "Negative cost!");
5630
5631 // For insertions, a ISD::BUILD_VECTOR style vector initialization can be much
5632 // cheaper than an accumulation of ISD::INSERT_VECTOR_ELT. SLPVectorizer has
5633 // a special heuristic regarding poison input which is passed here in
5634 // ForPoisonSrc.
5635 if (Insert && !ForPoisonSrc) {
5636 // This is nearly identical to BaseT::getScalarizationOverhead(), except
5637 // it is passing nullptr to getVectorInstrCost() for Op0 (instead of
5638 // Constant::getNullValue()), which makes the X86TTIImpl
5639 // getVectorInstrCost() return 0 instead of 1.
5640 for (unsigned I : seq(Size: DemandedElts.getBitWidth())) {
5641 if (!DemandedElts[I])
5642 continue;
5643 Cost += getVectorInstrCost(Opcode: Instruction::InsertElement, Val: Ty, CostKind, Index: I,
5644 Op0: Constant::getNullValue(Ty),
5645 Op1: VL.empty() ? nullptr : VL[I],
5646 VIC: TTI::VectorInstrContext::None);
5647 }
5648 return Cost;
5649 }
5650
5651 if (Insert) {
5652 if ((MScalarTy == MVT::i16 && ST->hasSSE2()) ||
5653 (MScalarTy.isInteger() && ST->hasSSE41()) ||
5654 (MScalarTy == MVT::f32 && ST->hasSSE41())) {
5655 // For types we can insert directly, insertion into 128-bit sub vectors is
5656 // cheap, followed by a cheap chain of concatenations.
5657 if (LegalVectorBitWidth <= LaneBitWidth) {
5658 Cost += BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, Insert,
5659 /*Extract*/ false, CostKind);
5660 } else {
5661 // In each 128-lane, if at least one index is demanded but not all
5662 // indices are demanded and this 128-lane is not the first 128-lane of
5663 // the legalized-vector, then this 128-lane needs a extracti128; If in
5664 // each 128-lane, there is at least one demanded index, this 128-lane
5665 // needs a inserti128.
5666
5667 // The following cases will help you build a better understanding:
5668 // Assume we insert several elements into a v8i32 vector in avx2,
5669 // Case#1: inserting into 1th index needs vpinsrd + inserti128.
5670 // Case#2: inserting into 5th index needs extracti128 + vpinsrd +
5671 // inserti128.
5672 // Case#3: inserting into 4,5,6,7 index needs 4*vpinsrd + inserti128.
5673 assert((LegalVectorBitWidth % LaneBitWidth) == 0 && "Illegal vector");
5674 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5675 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5676 unsigned NumLegalElts =
5677 LT.second.getVectorNumElements() * NumLegalVectors;
5678 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5679 "Vector has been legalized to smaller element count");
5680 assert((NumLegalElts % NumLanesTotal) == 0 &&
5681 "Unexpected elts per lane");
5682 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5683
5684 APInt WidenedDemandedElts = DemandedElts.zext(width: NumLegalElts);
5685 auto *LaneTy =
5686 FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumEltsPerLane);
5687
5688 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5689 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5690 numBits: NumEltsPerLane, bitPosition: NumEltsPerLane * I);
5691 if (LaneEltMask.isZero())
5692 continue;
5693 // FIXME: we don't need to extract if all non-demanded elements
5694 // are legalization-inserted padding.
5695 if (!LaneEltMask.isAllOnes())
5696 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
5697 Mask: {}, Index: I * NumEltsPerLane, SubTp: LaneTy);
5698 Cost += BaseT::getScalarizationOverhead(InTy: LaneTy, DemandedElts: LaneEltMask, Insert,
5699 /*Extract*/ false, CostKind);
5700 }
5701
5702 APInt AffectedLanes =
5703 APIntOps::ScaleBitMask(A: WidenedDemandedElts, NewBitWidth: NumLanesTotal);
5704 APInt FullyAffectedLegalVectors = APIntOps::ScaleBitMask(
5705 A: AffectedLanes, NewBitWidth: NumLegalVectors, /*MatchAllBits=*/true);
5706 for (int LegalVec = 0; LegalVec != NumLegalVectors; ++LegalVec) {
5707 for (unsigned Lane = 0; Lane != NumLegalLanes; ++Lane) {
5708 unsigned I = NumLegalLanes * LegalVec + Lane;
5709 // No need to insert unaffected lane; or lane 0 of each legal vector
5710 // iff ALL lanes of that vector were affected and will be inserted.
5711 if (!AffectedLanes[I] ||
5712 (Lane == 0 && FullyAffectedLegalVectors[LegalVec]))
5713 continue;
5714 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
5715 Mask: {}, Index: I * NumEltsPerLane, SubTp: LaneTy);
5716 }
5717 }
5718 }
5719 } else if (LT.second.isVector()) {
5720 // Without fast insertion, we need to use MOVD/MOVQ to pass each demanded
5721 // integer element as a SCALAR_TO_VECTOR, then we build the vector as a
5722 // series of UNPCK followed by CONCAT_VECTORS - all of these can be
5723 // considered cheap.
5724 if (Ty->isIntOrIntVectorTy())
5725 Cost += DemandedElts.popcount();
5726
5727 // Get the smaller of the legalized or original pow2-extended number of
5728 // vector elements, which represents the number of unpacks we'll end up
5729 // performing.
5730 unsigned NumElts = LT.second.getVectorNumElements();
5731 unsigned Pow2Elts =
5732 PowerOf2Ceil(A: cast<FixedVectorType>(Val: Ty)->getNumElements());
5733 Cost += (std::min<unsigned>(a: NumElts, b: Pow2Elts) - 1) * LT.first;
5734 }
5735 }
5736
5737 if (Extract) {
5738 // vXi1 can be efficiently extracted with MOVMSK.
5739 // TODO: AVX512 predicate mask handling.
5740 // NOTE: This doesn't work well for roundtrip scalarization.
5741 if (!Insert && Ty->getScalarSizeInBits() == 1 && !ST->hasAVX512()) {
5742 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
5743 unsigned MaxElts = ST->hasAVX2() ? 32 : 16;
5744 unsigned MOVMSKCost = (NumElts + MaxElts - 1) / MaxElts;
5745 return MOVMSKCost;
5746 }
5747
5748 if (LT.second.isVector()) {
5749 unsigned NumLegalElts =
5750 LT.second.getVectorNumElements() * NumLegalVectors;
5751 assert(NumLegalElts >= DemandedElts.getBitWidth() &&
5752 "Vector has been legalized to smaller element count");
5753
5754 // If we're extracting elements from a 128-bit subvector lane,
5755 // we only need to extract each lane once, not for every element.
5756 if (LegalVectorBitWidth > LaneBitWidth) {
5757 unsigned NumLegalLanes = LegalVectorBitWidth / LaneBitWidth;
5758 unsigned NumLanesTotal = NumLegalLanes * NumLegalVectors;
5759 assert((NumLegalElts % NumLanesTotal) == 0 &&
5760 "Unexpected elts per lane");
5761 unsigned NumEltsPerLane = NumLegalElts / NumLanesTotal;
5762
5763 // Add cost for each demanded 128-bit subvector extraction.
5764 // Luckily this is a lot easier than for insertion.
5765 APInt WidenedDemandedElts = DemandedElts.zext(width: NumLegalElts);
5766 auto *LaneTy =
5767 FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumEltsPerLane);
5768
5769 for (unsigned I = 0; I != NumLanesTotal; ++I) {
5770 APInt LaneEltMask = WidenedDemandedElts.extractBits(
5771 numBits: NumEltsPerLane, bitPosition: I * NumEltsPerLane);
5772 if (LaneEltMask.isZero())
5773 continue;
5774 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind, Mask: {},
5775 Index: I * NumEltsPerLane, SubTp: LaneTy);
5776 Cost += BaseT::getScalarizationOverhead(
5777 InTy: LaneTy, DemandedElts: LaneEltMask, /*Insert*/ false, Extract, CostKind);
5778 }
5779
5780 return Cost;
5781 }
5782 }
5783
5784 // Fallback to default extraction.
5785 Cost += BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, /*Insert*/ false,
5786 Extract, CostKind);
5787 }
5788
5789 return Cost;
5790}
5791
5792InstructionCost
5793X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor,
5794 int VF, const APInt &DemandedDstElts,
5795 TTI::TargetCostKind CostKind) const {
5796 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: EltTy);
5797 // We don't differentiate element types here, only element bit width.
5798 EltTy = IntegerType::getIntNTy(C&: EltTy->getContext(), N: EltTyBits);
5799
5800 auto bailout = [&]() {
5801 return BaseT::getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
5802 DemandedDstElts, CostKind);
5803 };
5804
5805 // For now, only deal with AVX512 cases.
5806 if (!ST->hasAVX512())
5807 return bailout();
5808
5809 // Do we have a native shuffle for this element type, or should we promote?
5810 unsigned PromEltTyBits = EltTyBits;
5811 switch (EltTyBits) {
5812 case 32:
5813 case 64:
5814 break; // AVX512F.
5815 case 16:
5816 if (!ST->hasBWI())
5817 PromEltTyBits = 32; // promote to i32, AVX512F.
5818 break; // AVX512BW
5819 case 8:
5820 if (!ST->hasVBMI())
5821 PromEltTyBits = 32; // promote to i32, AVX512F.
5822 break; // AVX512VBMI
5823 case 1:
5824 // There is no support for shuffling i1 elements. We *must* promote.
5825 if (ST->hasBWI()) {
5826 if (ST->hasVBMI())
5827 PromEltTyBits = 8; // promote to i8, AVX512VBMI.
5828 else
5829 PromEltTyBits = 16; // promote to i16, AVX512BW.
5830 break;
5831 }
5832 PromEltTyBits = 32; // promote to i32, AVX512F.
5833 break;
5834 default:
5835 return bailout();
5836 }
5837 auto *PromEltTy = IntegerType::getIntNTy(C&: EltTy->getContext(), N: PromEltTyBits);
5838
5839 auto *SrcVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: VF);
5840 auto *PromSrcVecTy = FixedVectorType::get(ElementType: PromEltTy, NumElts: VF);
5841
5842 int NumDstElements = VF * ReplicationFactor;
5843 auto *PromDstVecTy = FixedVectorType::get(ElementType: PromEltTy, NumElts: NumDstElements);
5844 auto *DstVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumDstElements);
5845
5846 // Legalize the types.
5847 MVT LegalSrcVecTy = getTypeLegalizationCost(Ty: SrcVecTy).second;
5848 MVT LegalPromSrcVecTy = getTypeLegalizationCost(Ty: PromSrcVecTy).second;
5849 MVT LegalPromDstVecTy = getTypeLegalizationCost(Ty: PromDstVecTy).second;
5850 MVT LegalDstVecTy = getTypeLegalizationCost(Ty: DstVecTy).second;
5851 // They should have legalized into vector types.
5852 if (!LegalSrcVecTy.isVector() || !LegalPromSrcVecTy.isVector() ||
5853 !LegalPromDstVecTy.isVector() || !LegalDstVecTy.isVector())
5854 return bailout();
5855
5856 if (PromEltTyBits != EltTyBits) {
5857 // If we have to perform the shuffle with wider elt type than our data type,
5858 // then we will first need to anyext (we don't care about the new bits)
5859 // the source elements, and then truncate Dst elements.
5860 InstructionCost PromotionCost;
5861 PromotionCost += getCastInstrCost(
5862 Opcode: Instruction::SExt, /*Dst=*/PromSrcVecTy, /*Src=*/SrcVecTy,
5863 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5864 PromotionCost +=
5865 getCastInstrCost(Opcode: Instruction::Trunc, /*Dst=*/DstVecTy,
5866 /*Src=*/PromDstVecTy,
5867 CCH: TargetTransformInfo::CastContextHint::None, CostKind);
5868 return PromotionCost + getReplicationShuffleCost(EltTy: PromEltTy,
5869 ReplicationFactor, VF,
5870 DemandedDstElts, CostKind);
5871 }
5872
5873 assert(LegalSrcVecTy.getScalarSizeInBits() == EltTyBits &&
5874 LegalSrcVecTy.getScalarType() == LegalDstVecTy.getScalarType() &&
5875 "We expect that the legalization doesn't affect the element width, "
5876 "doesn't coalesce/split elements.");
5877
5878 unsigned NumEltsPerDstVec = LegalDstVecTy.getVectorNumElements();
5879 unsigned NumDstVectors =
5880 divideCeil(Numerator: DstVecTy->getNumElements(), Denominator: NumEltsPerDstVec);
5881
5882 auto *SingleDstVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumEltsPerDstVec);
5883
5884 // Not all the produced Dst elements may be demanded. In our case,
5885 // given that a single Dst vector is formed by a single shuffle,
5886 // if all elements that will form a single Dst vector aren't demanded,
5887 // then we won't need to do that shuffle, so adjust the cost accordingly.
5888 APInt DemandedDstVectors = APIntOps::ScaleBitMask(
5889 A: DemandedDstElts.zext(width: NumDstVectors * NumEltsPerDstVec), NewBitWidth: NumDstVectors);
5890 unsigned NumDstVectorsDemanded = DemandedDstVectors.popcount();
5891
5892 InstructionCost SingleShuffleCost =
5893 getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleDstVecTy, SrcTy: SingleDstVecTy,
5894 CostKind, /*Mask=*/{},
5895 /*Index=*/0, /*SubTp=*/nullptr);
5896 return NumDstVectorsDemanded * SingleShuffleCost;
5897}
5898
5899InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
5900 Align Alignment,
5901 unsigned AddressSpace,
5902 TTI::TargetCostKind CostKind,
5903 TTI::OperandValueInfo OpInfo,
5904 const Instruction *I) const {
5905 // FIXME: Load latency isn't handled here
5906 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
5907 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5908 CostKind, OpInfo, I);
5909
5910 // TODO: Handle other cost kinds.
5911 if (CostKind != TTI::TCK_RecipThroughput) {
5912 if (auto *SI = dyn_cast_or_null<StoreInst>(Val: I)) {
5913 // Store instruction with index and scale costs 2 Uops.
5914 // Check the preceding GEP to identify non-const indices.
5915 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: SI->getPointerOperand())) {
5916 if (!all_of(Range: GEP->indices(), P: [](Value *V) { return isa<Constant>(Val: V); }))
5917 return TTI::TCC_Basic * 2;
5918 }
5919 }
5920 return TTI::TCC_Basic;
5921 }
5922
5923 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5924 "Invalid Opcode");
5925 // Type legalization can't handle structs
5926 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
5927 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5928 CostKind, OpInfo, I);
5929
5930 // Legalize the type.
5931 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
5932
5933 auto *VTy = dyn_cast<FixedVectorType>(Val: Src);
5934
5935 InstructionCost Cost = 0;
5936
5937 // Add a cost for constant load to vector.
5938 if (Opcode == Instruction::Store && OpInfo.isConstant())
5939 Cost += getMemoryOpCost(Opcode: Instruction::Load, Src, Alignment: DL.getABITypeAlign(Ty: Src),
5940 /*AddressSpace=*/0, CostKind, OpInfo);
5941
5942 // Handle the simple case of non-vectors.
5943 // NOTE: this assumes that legalization never creates vector from scalars!
5944 if (!VTy || !LT.second.isVector()) {
5945 // Each load/store unit costs 1.
5946 return (LT.second.isFloatingPoint() ? Cost : 0) + LT.first * 1;
5947 }
5948
5949 bool IsLoad = Opcode == Instruction::Load;
5950
5951 Type *EltTy = VTy->getElementType();
5952
5953 const int EltTyBits = DL.getTypeSizeInBits(Ty: EltTy);
5954
5955 // Source of truth: how many elements were there in the original IR vector?
5956 const unsigned SrcNumElt = VTy->getNumElements();
5957
5958 // How far have we gotten?
5959 int NumEltRemaining = SrcNumElt;
5960 // Note that we intentionally capture by-reference, NumEltRemaining changes.
5961 auto NumEltDone = [&]() { return SrcNumElt - NumEltRemaining; };
5962
5963 const int MaxLegalOpSizeBytes = divideCeil(Numerator: LT.second.getSizeInBits(), Denominator: 8);
5964
5965 // Note that even if we can store 64 bits of an XMM, we still operate on XMM.
5966 const unsigned XMMBits = 128;
5967 if (XMMBits % EltTyBits != 0)
5968 // Vector size must be a multiple of the element size. I.e. no padding.
5969 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5970 CostKind, OpInfo, I);
5971 const int NumEltPerXMM = XMMBits / EltTyBits;
5972
5973 auto *XMMVecTy = FixedVectorType::get(ElementType: EltTy, NumElts: NumEltPerXMM);
5974
5975 for (int CurrOpSizeBytes = MaxLegalOpSizeBytes, SubVecEltsLeft = 0;
5976 NumEltRemaining > 0; CurrOpSizeBytes /= 2) {
5977 // How many elements would a single op deal with at once?
5978 if ((8 * CurrOpSizeBytes) % EltTyBits != 0)
5979 // Vector size must be a multiple of the element size. I.e. no padding.
5980 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
5981 CostKind, OpInfo, I);
5982 int CurrNumEltPerOp = (8 * CurrOpSizeBytes) / EltTyBits;
5983
5984 assert(CurrOpSizeBytes > 0 && CurrNumEltPerOp > 0 && "How'd we get here?");
5985 assert((((NumEltRemaining * EltTyBits) < (2 * 8 * CurrOpSizeBytes)) ||
5986 (CurrOpSizeBytes == MaxLegalOpSizeBytes)) &&
5987 "Unless we haven't halved the op size yet, "
5988 "we have less than two op's sized units of work left.");
5989
5990 auto *CurrVecTy = CurrNumEltPerOp > NumEltPerXMM
5991 ? FixedVectorType::get(ElementType: EltTy, NumElts: CurrNumEltPerOp)
5992 : XMMVecTy;
5993
5994 assert(CurrVecTy->getNumElements() % CurrNumEltPerOp == 0 &&
5995 "After halving sizes, the vector elt count is no longer a multiple "
5996 "of number of elements per operation?");
5997 auto *CoalescedVecTy =
5998 CurrNumEltPerOp == 1
5999 ? CurrVecTy
6000 : FixedVectorType::get(
6001 ElementType: IntegerType::get(C&: Src->getContext(),
6002 NumBits: EltTyBits * CurrNumEltPerOp),
6003 NumElts: CurrVecTy->getNumElements() / CurrNumEltPerOp);
6004 assert(DL.getTypeSizeInBits(CoalescedVecTy) ==
6005 DL.getTypeSizeInBits(CurrVecTy) &&
6006 "coalesciing elements doesn't change vector width.");
6007
6008 while (NumEltRemaining > 0) {
6009 assert(SubVecEltsLeft >= 0 && "Subreg element count overconsumtion?");
6010
6011 // Can we use this vector size, as per the remaining element count?
6012 // Iff the vector is naturally aligned, we can do a wide load regardless.
6013 if (NumEltRemaining < CurrNumEltPerOp &&
6014 (!IsLoad || Alignment < CurrOpSizeBytes) && CurrOpSizeBytes != 1)
6015 break; // Try smalled vector size.
6016
6017 // This isn't exactly right. We're using slow unaligned 32-byte accesses
6018 // as a proxy for a double-pumped AVX memory interface such as on
6019 // Sandybridge.
6020 // Sub-32-bit loads/stores will be slower either with PINSR*/PEXTR* or
6021 // will be scalarized.
6022 //
6023 // For a vector load, each non-0th 1/2/4-byte in-lane remainder chunk is
6024 // materialized by a *single* folded PINSR*(mem) that both loads and
6025 // inserts the lane (1B->PINSRB, 2B->PINSRW, 4B->PINSRD; the byte and
6026 // dword folds need SSE4.1, the word fold only SSE2).
6027 // When that fold is available the chunk is one instruction, so it must be
6028 // priced once here (as a plain load) and the separate lane-insert charge
6029 // below must be skipped - otherwise the folded insert is double-counted.
6030 // Stores (the symmetric PEXTR*(mem) fold) are left unchanged here.
6031 bool Is0thSubVec = (NumEltDone() % LT.second.getVectorNumElements()) == 0;
6032 bool FoldedInLaneInsert = IsLoad && !Is0thSubVec &&
6033 ((CurrOpSizeBytes == 1 && ST->hasSSE41()) ||
6034 (CurrOpSizeBytes == 2 && ST->hasSSE2()) ||
6035 (CurrOpSizeBytes == 4 && ST->hasSSE41()));
6036 if (CurrOpSizeBytes == 32 && ST->isUnalignedMem32Slow())
6037 Cost += 2;
6038 else if (CurrOpSizeBytes < 4 && !FoldedInLaneInsert)
6039 Cost += 2;
6040 else
6041 Cost += 1;
6042
6043 // If we're loading a uniform value, then we don't need to split the load,
6044 // loading just a single (widest) vector can be reused by all splits.
6045 if (IsLoad && OpInfo.isUniform())
6046 return Cost;
6047
6048 // If we have fully processed the previous reg, we need to replenish it.
6049 if (SubVecEltsLeft == 0) {
6050 SubVecEltsLeft += CurrVecTy->getNumElements();
6051 // And that's free only for the 0'th subvector of a legalized vector.
6052 if (!Is0thSubVec)
6053 Cost +=
6054 getShuffleCost(Kind: IsLoad ? TTI::ShuffleKind::SK_InsertSubvector
6055 : TTI::ShuffleKind::SK_ExtractSubvector,
6056 DstTy: VTy, SrcTy: VTy, CostKind, Mask: {}, Index: NumEltDone(), SubTp: CurrVecTy);
6057 }
6058
6059 // While we can directly load/store ZMM, YMM, and 64-bit halves of XMM,
6060 // for smaller widths (32/16/8) we have to insert/extract them separately.
6061 // Again, it's free for the 0'th subreg (if op is 32/64 bit wide,
6062 // but let's pretend that it is also true for 16/8 bit wide ops...)
6063 if (CurrOpSizeBytes <= 32 / 8 && !Is0thSubVec && !FoldedInLaneInsert) {
6064 int NumEltDoneInCurrXMM = NumEltDone() % NumEltPerXMM;
6065 assert(NumEltDoneInCurrXMM % CurrNumEltPerOp == 0 && "");
6066 int CoalescedVecEltIdx = NumEltDoneInCurrXMM / CurrNumEltPerOp;
6067 APInt DemandedElts =
6068 APInt::getBitsSet(numBits: CoalescedVecTy->getNumElements(),
6069 loBit: CoalescedVecEltIdx, hiBit: CoalescedVecEltIdx + 1);
6070 assert(DemandedElts.popcount() == 1 && "Inserting single value");
6071 Cost += getScalarizationOverhead(Ty: CoalescedVecTy, DemandedElts, Insert: IsLoad,
6072 Extract: !IsLoad, CostKind);
6073 }
6074
6075 SubVecEltsLeft -= CurrNumEltPerOp;
6076 NumEltRemaining -= CurrNumEltPerOp;
6077 Alignment = commonAlignment(A: Alignment, Offset: CurrOpSizeBytes);
6078 }
6079 }
6080
6081 assert(NumEltRemaining <= 0 && "Should have processed all the elements.");
6082
6083 return Cost;
6084}
6085
6086InstructionCost
6087X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
6088 TTI::TargetCostKind CostKind) const {
6089 switch (MICA.getID()) {
6090 case Intrinsic::masked_scatter:
6091 case Intrinsic::masked_gather:
6092 return getGatherScatterOpCost(MICA, CostKind);
6093 case Intrinsic::masked_load:
6094 case Intrinsic::masked_store:
6095 return getMaskedMemoryOpCost(MICA, CostKind);
6096 case Intrinsic::masked_compressstore:
6097 // Fallback to scalarization for slow compressstore targets (e.g. znver4).
6098 if (ST->isVecCompressStoreSlow())
6099 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6100 break;
6101 }
6102
6103 static const CostKindTblEntry AVX512VBMI2CostTable[] = {
6104 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6105 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6106 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6107
6108 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6109 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6110 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6111
6112 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6113 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6114 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6115
6116 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8bf16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6117 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16bf16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6118 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v32bf16,.Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6119
6120 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6121 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6122 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v64i8, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6123
6124 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6125 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6126 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32i16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6127
6128 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8f16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6129 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16f16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6130 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32f16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6131
6132 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8bf16, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6133 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16bf16,.Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6134 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v32bf16,.Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6135 };
6136
6137 static const CostKindTblEntry AVX512CostTable[] = {
6138 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6139 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6140 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6141 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6142 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6143 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6144
6145 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6146 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6147 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6148 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6149 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6150 { .ISD: Intrinsic::masked_expandload, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 1, .SizeAndLatencyCost: 3 } },
6151
6152 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6153 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6154 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6155 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6156 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16i32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6157 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v16f32, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6158
6159 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v2i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6160 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v2f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6161 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6162 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v4f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 1, .SizeAndLatencyCost: 7 } },
6163 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8i64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6164 { .ISD: Intrinsic::masked_compressstore, .Type: MVT::v8f64, .Cost: { .RecipThroughputCost: 3,.LatencyCost: 12, .CodeSizeCost: 1, .SizeAndLatencyCost: 8 } },
6165 };
6166
6167 std::pair<InstructionCost, MVT> LT =
6168 getTypeLegalizationCost(Ty: MICA.getDataType());
6169
6170 if (ST->hasVBMI2())
6171 if (const auto *Entry =
6172 CostTableLookup(Table: AVX512VBMI2CostTable, ISD: MICA.getID(), Ty: LT.second))
6173 if (auto KindCost = Entry->Cost[CostKind])
6174 return LT.first * *KindCost;
6175
6176 if (ST->hasAVX512())
6177 if (const auto *Entry =
6178 CostTableLookup(Table: AVX512CostTable, ISD: MICA.getID(), Ty: LT.second))
6179 if (auto KindCost = Entry->Cost[CostKind])
6180 return LT.first * *KindCost;
6181
6182 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
6183}
6184
6185InstructionCost
6186X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
6187 TTI::TargetCostKind CostKind) const {
6188 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
6189 : Instruction::Store;
6190 Type *SrcTy = MICA.getDataType();
6191 Align Alignment = MICA.getAlignment();
6192 unsigned AddressSpace = MICA.getAddressSpace();
6193
6194 bool IsLoad = (Instruction::Load == Opcode);
6195 bool IsStore = (Instruction::Store == Opcode);
6196
6197 auto *SrcVTy = dyn_cast<FixedVectorType>(Val: SrcTy);
6198 if (!SrcVTy)
6199 // To calculate scalar take the regular cost, without mask
6200 return getMemoryOpCost(Opcode, Src: SrcTy, Alignment, AddressSpace, CostKind);
6201
6202 unsigned NumElem = SrcVTy->getNumElements();
6203 auto *MaskTy =
6204 FixedVectorType::get(ElementType: Type::getInt8Ty(C&: SrcVTy->getContext()), NumElts: NumElem);
6205 if ((IsLoad && !isLegalMaskedLoad(DataType: SrcVTy, Alignment, AddressSpace)) ||
6206 (IsStore && !isLegalMaskedStore(DataType: SrcVTy, Alignment, AddressSpace))) {
6207 // Scalarization
6208 APInt DemandedElts = APInt::getAllOnes(numBits: NumElem);
6209 InstructionCost MaskSplitCost = getScalarizationOverhead(
6210 Ty: MaskTy, DemandedElts, /*Insert*/ false, /*Extract*/ true, CostKind);
6211 InstructionCost ScalarCompareCost = getCmpSelInstrCost(
6212 Opcode: Instruction::ICmp, ValTy: Type::getInt8Ty(C&: SrcVTy->getContext()), CondTy: nullptr,
6213 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
6214 InstructionCost BranchCost = getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
6215 InstructionCost MaskCmpCost = NumElem * (BranchCost + ScalarCompareCost);
6216 InstructionCost ValueSplitCost = getScalarizationOverhead(
6217 Ty: SrcVTy, DemandedElts, Insert: IsLoad, Extract: IsStore, CostKind);
6218 InstructionCost MemopCost =
6219 NumElem * BaseT::getMemoryOpCost(Opcode, Src: SrcVTy->getScalarType(),
6220 Alignment, AddressSpace, CostKind);
6221 return MemopCost + ValueSplitCost + MaskSplitCost + MaskCmpCost;
6222 }
6223
6224 // Legalize the type.
6225 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcVTy);
6226 auto VT = TLI->getValueType(DL, Ty: SrcVTy);
6227 InstructionCost Cost = 0;
6228 MVT Ty = LT.second;
6229 if (Ty == MVT::i16 || Ty == MVT::i32 || Ty == MVT::i64)
6230 // APX masked load/store for scalar is cheap.
6231 return Cost + LT.first;
6232
6233 if (VT.isSimple() && Ty != VT.getSimpleVT() &&
6234 LT.second.getVectorNumElements() == NumElem)
6235 // Promotion requires extend/truncate for data and a shuffle for mask.
6236 Cost += getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SrcVTy, SrcTy: SrcVTy, CostKind, Mask: {},
6237 Index: 0, SubTp: nullptr) +
6238 getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: MaskTy, SrcTy: MaskTy, CostKind, Mask: {},
6239 Index: 0, SubTp: nullptr);
6240
6241 else if (LT.first * Ty.getVectorNumElements() > NumElem) {
6242 auto *NewMaskTy = FixedVectorType::get(ElementType: MaskTy->getElementType(),
6243 NumElts: (unsigned)LT.first.getValue() *
6244 Ty.getVectorNumElements());
6245 // Expanding requires fill mask with zeroes
6246 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: NewMaskTy, SrcTy: NewMaskTy,
6247 CostKind, Mask: {}, Index: 0, SubTp: MaskTy);
6248 }
6249
6250 // Predicate fanout (see getPredicateFanoutCost): only a variable mask needs
6251 // a per-part sub-mask; a constant mask is folded by codegen and pays nothing.
6252 InstructionCost MaskExpCost = 0;
6253 if (MICA.getVariableMask()) {
6254 auto *MaskVecTy =
6255 FixedVectorType::get(ElementType: Type::getInt1Ty(C&: SrcVTy->getContext()), NumElts: NumElem);
6256 MaskExpCost = getPredicateFanoutCost(TTI: *this, DataVTy: SrcVTy, MaskVTy: MaskVecTy);
6257 }
6258
6259 // Pre-AVX512 - each maskmov load costs 2 + store costs ~8.
6260 if (!ST->hasAVX512())
6261 return Cost + LT.first * (IsLoad ? 2 : 8) + MaskExpCost;
6262
6263 // AVX-512 masked load/store is cheaper.
6264 return Cost + LT.first + MaskExpCost;
6265}
6266
6267InstructionCost X86TTIImpl::getPointersChainCost(
6268 ArrayRef<const Value *> Ptrs, const Value *Base,
6269 const TTI::PointersChainInfo &Info, Type *AccessTy,
6270 const TTI::TargetCostKind CostKind) const {
6271 if (Info.isSameBase() && Info.isKnownStride()) {
6272 // If all the pointers have known stride all the differences are translated
6273 // into constants. X86 memory addressing allows encoding it into
6274 // displacement. So we just need to take the base GEP cost.
6275 if (const auto *BaseGEP = dyn_cast<GetElementPtrInst>(Val: Base)) {
6276 SmallVector<const Value *> Indices(BaseGEP->indices());
6277 return getGEPCost(PointeeType: BaseGEP->getSourceElementType(),
6278 Ptr: BaseGEP->getPointerOperand(), Operands: Indices, CostKind,
6279 AccessType: nullptr);
6280 }
6281 return TTI::TCC_Free;
6282 }
6283 return BaseT::getPointersChainCost(Ptrs, Base, Info, AccessTy, CostKind);
6284}
6285
6286InstructionCost
6287X86TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
6288 const SCEV *Ptr,
6289 TTI::TargetCostKind CostKind) const {
6290 // Address computations in vectorized code with non-consecutive addresses will
6291 // likely result in more instructions compared to scalar code where the
6292 // computation can more often be merged into the index mode. The resulting
6293 // extra micro-ops can significantly decrease throughput.
6294 const unsigned NumVectorInstToHideOverhead = 10;
6295
6296 // Cost modeling of Strided Access Computation is hidden by the indexing
6297 // modes of X86 regardless of the stride value. We dont believe that there
6298 // is a difference between constant strided access in gerenal and constant
6299 // strided value which is less than or equal to 64.
6300 // Even in the case of (loop invariant) stride whose value is not known at
6301 // compile time, the address computation will not incur more than one extra
6302 // ADD instruction.
6303 if (PtrTy->isVectorTy() && SE) {
6304 if (BaseT::isStridedAccess(Ptr) && !BaseT::getConstantStrideStep(SE, Ptr))
6305 return 1;
6306 if (!ST->hasAVX2()) {
6307 // TODO: AVX2 is the current cut-off because we don't have correct
6308 // interleaving costs for prior ISA's.
6309 if (!BaseT::isStridedAccess(Ptr))
6310 return NumVectorInstToHideOverhead;
6311 }
6312 }
6313
6314 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
6315}
6316
6317InstructionCost X86TTIImpl::getPartialReductionCost(
6318 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
6319 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
6320 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
6321 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
6322 auto ExpandCost = [&]() {
6323 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
6324 AccumType, VF, OpAExtend, OpBExtend,
6325 BinOp, CostKind, FMF);
6326 };
6327
6328 // The dot product instructions multiply-accumulate i8 x i8 -> i32,
6329 // i16 x i16 -> i32, bf16 x bf16 -> f32 or f16 x f16 -> f32. Partial
6330 // reductions may also multiply inputs extended from different types, which
6331 // they can't handle.
6332 if (VF.isScalable() || !BinOp || OpAExtend == TTI::PR_None ||
6333 OpBExtend == TTI::PR_None || InputTypeA != InputTypeB)
6334 return ExpandCost();
6335
6336 unsigned Opc;
6337 if (Opcode == Instruction::Add && *BinOp == Instruction::Mul &&
6338 AccumType->isIntegerTy(BitWidth: 32) &&
6339 (InputTypeA->isIntegerTy(BitWidth: 8) || InputTypeA->isIntegerTy(BitWidth: 16))) {
6340 if (OpAExtend != OpBExtend)
6341 Opc = ISD::PARTIAL_REDUCE_SUMLA;
6342 else if (OpAExtend == TTI::PR_SignExtend)
6343 Opc = ISD::PARTIAL_REDUCE_SMLA;
6344 else
6345 Opc = ISD::PARTIAL_REDUCE_UMLA;
6346 } else if (Opcode == Instruction::FAdd && *BinOp == Instruction::FMul &&
6347 AccumType->isFloatTy() &&
6348 (InputTypeA->isBFloatTy() || InputTypeA->isHalfTy())) {
6349 // VDPBF16PS and VDPPHPS, like the expansion, reassociate the additions and
6350 // fuse the multiplications.
6351 if (!FMF || !FMF->allowReassoc() || !FMF->allowContract())
6352 return InstructionCost::getInvalid();
6353 Opc = ISD::PARTIAL_REDUCE_FMLA;
6354 } else {
6355 return ExpandCost();
6356 }
6357
6358 unsigned Ratio =
6359 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
6360 if (!VF.isKnownMultipleOf(RHS: Ratio))
6361 return ExpandCost();
6362
6363 // One dot product per legal accumulator vector. Accumulators narrower than
6364 // a legal vector are widened by expanding the partial reduction instead.
6365 auto *AccVecTy = VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: Ratio));
6366 auto *InputVecTy = VectorType::get(ElementType: InputTypeA, EC: VF);
6367 std::pair<InstructionCost, MVT> AccLT = getTypeLegalizationCost(Ty: AccVecTy);
6368 std::pair<InstructionCost, MVT> InputLT = getTypeLegalizationCost(Ty: InputVecTy);
6369 if (AccLT.second.getFixedSizeInBits() >
6370 AccVecTy->getPrimitiveSizeInBits().getFixedValue() ||
6371 !TLI->isPartialReduceMLALegalOrCustom(Opc, AccVT: AccLT.second, InputVT: InputLT.second))
6372 return ExpandCost();
6373
6374 // Without VNNI, the signed i16 dot product is a PMADDWD and an ADD, which
6375 // cost about the same.
6376 if (Opc == ISD::PARTIAL_REDUCE_SMLA && InputTypeA->isIntegerTy(BitWidth: 16) &&
6377 !ST->hasVNNI() && !ST->hasAVXVNNI())
6378 return 2 * getArithmeticInstrCost(Opcode: Instruction::Add, Ty: AccVecTy, CostKind);
6379
6380 return AccLT.first;
6381}
6382
6383InstructionCost
6384X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
6385 std::optional<FastMathFlags> FMF,
6386 TTI::TargetCostKind CostKind) const {
6387 if (TTI::requiresOrderedReduction(FMF))
6388 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6389
6390 // We use llvm-mca across all supported CPUs to measure the logic cost stats.
6391 // We use the Intel Architecture Code Analyzer(IACA) to measure the throughput
6392 // and make it as the cost. TODO: Update old IACA numbers to llvm-mca.
6393
6394 static const CostKindTblEntry SLMCostTbl[] = {
6395 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6396 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6397 };
6398
6399 static const CostKindTblEntry SSE2CostTbl[] = {
6400 { .ISD: ISD::FADD, .Type: MVT::v2f64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6401 { .ISD: ISD::FADD, .Type: MVT::v2f32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6402 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6403 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // The data reported by the IACA tool is "1.6".
6404 { .ISD: ISD::ADD, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // FIXME: chosen to be less than v4i32
6405 { .ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} }, // The data reported by the IACA tool is "3.3".
6406 { .ISD: ISD::ADD, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // The data reported by the IACA tool is "4.3".
6407 { .ISD: ISD::ADD, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} }, // The data reported by the IACA tool is "4.3".
6408 { .ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // The data reported by the IACA tool is "4.3".
6409 { .ISD: ISD::ADD, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6410 { .ISD: ISD::ADD, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6411 { .ISD: ISD::ADD, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} },
6412 { .ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6413
6414 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6415 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6416 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6417 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6418 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6419 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6420 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6421 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6422 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6423 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6424 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 8, .SizeAndLatencyCost: 8} },
6425 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6426 };
6427
6428 static const CostKindTblEntry AVX1CostTbl[] = {
6429 { .ISD: ISD::FADD, .Type: MVT::v4f64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6430 { .ISD: ISD::FADD, .Type: MVT::v4f32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6431 { .ISD: ISD::FADD, .Type: MVT::v8f32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6432 { .ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 1, .CodeSizeCost: 1, .SizeAndLatencyCost: 1} }, // The data reported by the IACA tool is "1.5".
6433 { .ISD: ISD::ADD, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6434 { .ISD: ISD::ADD, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6435 { .ISD: ISD::ADD, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6436 { .ISD: ISD::ADD, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6437
6438 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6439 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6440 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6441 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6442 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6443 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6444 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6445 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6446 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6447 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6448 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6449 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6450 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6451 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6452 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6453 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6454 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6455 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 10, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6456 };
6457
6458 static const CostKindTblEntry AVX2CostTbl[] = {
6459 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6460 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6461 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6462 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6463 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6464 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6465 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6466 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6467 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6468 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6469 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6470 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6471 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6472 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6473 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6474 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6475 { .ISD: ISD::XOR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 7, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6476 { .ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6477 { .ISD: ISD::XOR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6478 { .ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6479 { .ISD: ISD::XOR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6480 { .ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6481 { .ISD: ISD::XOR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 13,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6482 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6483 };
6484
6485 static const CostKindTblEntry AVX512FCostTbl[] = {
6486 { .ISD: ISD::FADD, .Type: MVT::v8f64, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6487 { .ISD: ISD::FADD, .Type: MVT::v16f32, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6488 { .ISD: ISD::ADD, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6489 { .ISD: ISD::ADD, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 6, .LatencyCost: 6, .CodeSizeCost: 6, .SizeAndLatencyCost: 6} },
6490
6491 { .ISD: ISD::AND, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6492 { .ISD: ISD::AND, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6493 { .ISD: ISD::AND, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6494 { .ISD: ISD::AND, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6495 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6496 { .ISD: ISD::OR, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6497 { .ISD: ISD::OR, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6498 { .ISD: ISD::OR, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6499 { .ISD: ISD::OR, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6500 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6501 { .ISD: ISD::XOR, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6502 { .ISD: ISD::XOR, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6503 { .ISD: ISD::XOR, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 14,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6504 { .ISD: ISD::XOR, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 16,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6505 { .ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6506 };
6507
6508 static const CostKindTblEntry AVX512BWCostTbl[] = {
6509 { .ISD: ISD::ADD, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6510 { .ISD: ISD::ADD, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} },
6511 };
6512
6513 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6514 assert(ISD && "Invalid opcode");
6515
6516 // Before legalizing the type, give a chance to look up illegal narrow types
6517 // in the table.
6518 // FIXME: Is there a better way to do this?
6519 EVT VT = TLI->getValueType(DL, Ty: ValTy);
6520 if (VT.isSimple()) {
6521 MVT MTy = VT.getSimpleVT();
6522 if (ST->useSLMArithCosts())
6523 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
6524 if (auto KindCost = Entry->Cost[CostKind])
6525 return *KindCost;
6526
6527 if (ST->hasBWI())
6528 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6529 if (auto KindCost = Entry->Cost[CostKind])
6530 return *KindCost;
6531
6532 if (ST->hasAVX512())
6533 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6534 if (auto KindCost = Entry->Cost[CostKind])
6535 return *KindCost;
6536
6537 if (ST->hasAVX2())
6538 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6539 if (auto KindCost = Entry->Cost[CostKind])
6540 return *KindCost;
6541
6542 if (ST->hasAVX())
6543 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6544 if (auto KindCost = Entry->Cost[CostKind])
6545 return *KindCost;
6546
6547 if (ST->hasSSE2())
6548 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6549 if (auto KindCost = Entry->Cost[CostKind])
6550 return *KindCost;
6551 }
6552
6553 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6554
6555 MVT MTy = LT.second;
6556
6557 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6558
6559 InstructionCost ArithmeticCost = 0;
6560 if (LT.first != 1 && MTy.isVector() &&
6561 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6562 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6563 auto *SingleOpTy = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6564 NumElts: MTy.getVectorNumElements());
6565 ArithmeticCost = getArithmeticInstrCost(Opcode, Ty: SingleOpTy, CostKind);
6566 ArithmeticCost *= LT.first - 1;
6567 }
6568
6569 // FIXME: These assume a naive kshift+binop lowering, which is probably
6570 // conservative in most cases.
6571 static const CostKindTblEntry AVX512BoolReduction[] = {
6572 { .ISD: ISD::AND, .Type: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6573 { .ISD: ISD::AND, .Type: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6574 { .ISD: ISD::AND, .Type: MVT::v8i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6575 { .ISD: ISD::AND, .Type: MVT::v16i1, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6576 { .ISD: ISD::AND, .Type: MVT::v32i1, .Cost: {.RecipThroughputCost: 11,.LatencyCost: 11,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6577 { .ISD: ISD::AND, .Type: MVT::v64i1, .Cost: {.RecipThroughputCost: 13,.LatencyCost: 13,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6578 { .ISD: ISD::OR, .Type: MVT::v2i1, .Cost: { .RecipThroughputCost: 3, .LatencyCost: 3, .CodeSizeCost: 3, .SizeAndLatencyCost: 3} },
6579 { .ISD: ISD::OR, .Type: MVT::v4i1, .Cost: { .RecipThroughputCost: 5, .LatencyCost: 5, .CodeSizeCost: 5, .SizeAndLatencyCost: 5} },
6580 { .ISD: ISD::OR, .Type: MVT::v8i1, .Cost: { .RecipThroughputCost: 7, .LatencyCost: 7, .CodeSizeCost: 7, .SizeAndLatencyCost: 7} },
6581 { .ISD: ISD::OR, .Type: MVT::v16i1, .Cost: { .RecipThroughputCost: 9, .LatencyCost: 9, .CodeSizeCost: 9, .SizeAndLatencyCost: 9} },
6582 { .ISD: ISD::OR, .Type: MVT::v32i1, .Cost: {.RecipThroughputCost: 11,.LatencyCost: 11,.CodeSizeCost: 11,.SizeAndLatencyCost: 11} },
6583 { .ISD: ISD::OR, .Type: MVT::v64i1, .Cost: {.RecipThroughputCost: 13,.LatencyCost: 13,.CodeSizeCost: 13,.SizeAndLatencyCost: 13} },
6584 };
6585
6586 static const CostKindTblEntry AVX2BoolReduction[] = {
6587 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6588 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6589 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6590 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: { .RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vpmovmskb + cmp
6591 };
6592
6593 static const CostKindTblEntry AVX1BoolReduction[] = {
6594 { .ISD: ISD::AND, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskpd + cmp
6595 { .ISD: ISD::AND, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskps + cmp
6596 { .ISD: ISD::AND, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6597 { .ISD: ISD::AND, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpand + vpmovmskb + cmp
6598 { .ISD: ISD::OR, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskpd + cmp
6599 { .ISD: ISD::OR, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // vmovmskps + cmp
6600 { .ISD: ISD::OR, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6601 { .ISD: ISD::OR, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 4} }, // vextractf128 + vpor + vpmovmskb + cmp
6602 };
6603
6604 static const CostKindTblEntry SSE2BoolReduction[] = {
6605 { .ISD: ISD::AND, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskpd + cmp
6606 { .ISD: ISD::AND, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskps + cmp
6607 { .ISD: ISD::AND, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6608 { .ISD: ISD::AND, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6609 { .ISD: ISD::OR, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskpd + cmp
6610 { .ISD: ISD::OR, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // movmskps + cmp
6611 { .ISD: ISD::OR, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6612 { .ISD: ISD::OR, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 2, .SizeAndLatencyCost: 2} }, // pmovmskb + cmp
6613 };
6614
6615 // Handle bool allof/anyof vXi1 patterns before we check legal types.
6616 if (ValVTy->getElementType()->isIntegerTy(BitWidth: 1)) {
6617 if (ISD == ISD::ADD) {
6618 // vXi1 addition reduction will bitcast to scalar and perform a popcount.
6619 auto *IntTy = IntegerType::getIntNTy(C&: ValVTy->getContext(),
6620 N: ValVTy->getNumElements());
6621 IntrinsicCostAttributes ICA(Intrinsic::ctpop, IntTy, {IntTy});
6622 return getCastInstrCost(Opcode: Instruction::BitCast, Dst: IntTy, Src: ValVTy,
6623 CCH: TargetTransformInfo::CastContextHint::None,
6624 CostKind) +
6625 getIntrinsicInstrCost(ICA, CostKind);
6626 }
6627
6628 if (ST->hasAVX512())
6629 if (const auto *Entry = CostTableLookup(Table: AVX512BoolReduction, ISD, Ty: MTy))
6630 if (auto KindCost = Entry->Cost[CostKind])
6631 return ArithmeticCost + *KindCost;
6632 if (ST->hasAVX2())
6633 if (const auto *Entry = CostTableLookup(Table: AVX2BoolReduction, ISD, Ty: MTy))
6634 if (auto KindCost = Entry->Cost[CostKind])
6635 return ArithmeticCost + *KindCost;
6636 if (ST->hasAVX())
6637 if (const auto *Entry = CostTableLookup(Table: AVX1BoolReduction, ISD, Ty: MTy))
6638 if (auto KindCost = Entry->Cost[CostKind])
6639 return ArithmeticCost + *KindCost;
6640 if (ST->hasSSE2())
6641 if (const auto *Entry = CostTableLookup(Table: SSE2BoolReduction, ISD, Ty: MTy))
6642 if (auto KindCost = Entry->Cost[CostKind])
6643 return ArithmeticCost + *KindCost;
6644
6645 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValVTy, FMF, CostKind);
6646 }
6647
6648 // Special case: vXi8 mul reductions are performed as vXi16.
6649 if (ISD == ISD::MUL && MTy.getScalarType() == MVT::i8) {
6650 auto *WideSclTy = IntegerType::get(C&: ValVTy->getContext(), NumBits: 16);
6651 auto *WideVecTy = FixedVectorType::get(ElementType: WideSclTy, NumElts: ValVTy->getNumElements());
6652 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideVecTy, Src: ValTy,
6653 CCH: TargetTransformInfo::CastContextHint::None,
6654 CostKind) +
6655 getArithmeticReductionCost(Opcode, ValTy: WideVecTy, FMF, CostKind);
6656 }
6657
6658 if (ST->useSLMArithCosts())
6659 if (const auto *Entry = CostTableLookup(Table: SLMCostTbl, ISD, Ty: MTy))
6660 if (auto KindCost = Entry->Cost[CostKind])
6661 return ArithmeticCost + *KindCost;
6662
6663 if (ST->hasBWI())
6664 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6665 if (auto KindCost = Entry->Cost[CostKind])
6666 return ArithmeticCost + *KindCost;
6667
6668 if (ST->hasAVX512())
6669 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6670 if (auto KindCost = Entry->Cost[CostKind])
6671 return ArithmeticCost + *KindCost;
6672
6673 if (ST->hasAVX2())
6674 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6675 if (auto KindCost = Entry->Cost[CostKind])
6676 return ArithmeticCost + *KindCost;
6677
6678 if (ST->hasAVX())
6679 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6680 if (auto KindCost = Entry->Cost[CostKind])
6681 return ArithmeticCost + *KindCost;
6682
6683 if (ST->hasSSE2())
6684 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6685 if (auto KindCost = Entry->Cost[CostKind])
6686 return ArithmeticCost + *KindCost;
6687
6688 unsigned NumVecElts = ValVTy->getNumElements();
6689 unsigned ScalarSize = ValVTy->getScalarSizeInBits();
6690
6691 // Special case power of 2 reductions where the scalar type isn't changed
6692 // by type legalization.
6693 if (!isPowerOf2_32(Value: NumVecElts) || ScalarSize != MTy.getScalarSizeInBits())
6694 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValVTy, FMF, CostKind);
6695
6696 InstructionCost ReductionCost = 0;
6697
6698 auto *Ty = ValVTy;
6699 if (LT.first != 1 && MTy.isVector() &&
6700 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6701 // Type needs to be split. We need LT.first - 1 arithmetic ops.
6702 Ty = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6703 NumElts: MTy.getVectorNumElements());
6704 ReductionCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
6705 ReductionCost *= LT.first - 1;
6706 NumVecElts = MTy.getVectorNumElements();
6707 }
6708
6709 // Now handle reduction with the legal type, taking into account size changes
6710 // at each level.
6711 while (NumVecElts > 1) {
6712 // Determine the size of the remaining vector we need to reduce.
6713 unsigned Size = NumVecElts * ScalarSize;
6714 NumVecElts /= 2;
6715 // If we're reducing from 256/512 bits, use an extract_subvector.
6716 if (Size > 128) {
6717 auto *SubTy = FixedVectorType::get(ElementType: ValVTy->getElementType(), NumElts: NumVecElts);
6718 ReductionCost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty,
6719 CostKind, Mask: {}, Index: NumVecElts, SubTp: SubTy);
6720 Ty = SubTy;
6721 } else if (Size == 128) {
6722 // Reducing from 128 bits is a permute of v2f64/v2i64.
6723 FixedVectorType *ShufTy;
6724 if (ValVTy->isFloatingPointTy())
6725 ShufTy =
6726 FixedVectorType::get(ElementType: Type::getDoubleTy(C&: ValVTy->getContext()), NumElts: 2);
6727 else
6728 ShufTy =
6729 FixedVectorType::get(ElementType: Type::getInt64Ty(C&: ValVTy->getContext()), NumElts: 2);
6730 ReductionCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6731 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6732 } else if (Size == 64) {
6733 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
6734 FixedVectorType *ShufTy;
6735 if (ValVTy->isFloatingPointTy())
6736 ShufTy =
6737 FixedVectorType::get(ElementType: Type::getFloatTy(C&: ValVTy->getContext()), NumElts: 4);
6738 else
6739 ShufTy =
6740 FixedVectorType::get(ElementType: Type::getInt32Ty(C&: ValVTy->getContext()), NumElts: 4);
6741 ReductionCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
6742 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
6743 } else {
6744 // Reducing from smaller size is a shift by immediate.
6745 auto *ShiftTy = FixedVectorType::get(
6746 ElementType: Type::getIntNTy(C&: ValVTy->getContext(), N: Size), NumElts: 128 / Size);
6747 ReductionCost += getArithmeticInstrCost(
6748 Opcode: Instruction::LShr, Ty: ShiftTy, CostKind,
6749 Op1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
6750 Op2Info: {.Kind: TargetTransformInfo::OK_UniformConstantValue, .Properties: TargetTransformInfo::OP_None});
6751 }
6752
6753 // Add the arithmetic op for this level.
6754 ReductionCost += getArithmeticInstrCost(Opcode, Ty, CostKind);
6755 }
6756
6757 // Add the final extract element to the cost.
6758 return ReductionCost + getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty,
6759 CostKind, Index: 0, Op0: nullptr, Op1: nullptr,
6760 VIC: TTI::VectorInstrContext::None);
6761}
6762
6763InstructionCost X86TTIImpl::getMinMaxCost(Intrinsic::ID IID, Type *Ty,
6764 TTI::TargetCostKind CostKind,
6765 FastMathFlags FMF) const {
6766 IntrinsicCostAttributes ICA(IID, Ty, {Ty, Ty}, FMF);
6767 return getIntrinsicInstrCost(ICA, CostKind);
6768}
6769
6770InstructionCost
6771X86TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *ValTy,
6772 FastMathFlags FMF,
6773 TTI::TargetCostKind CostKind) const {
6774 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6775
6776 MVT MTy = LT.second;
6777
6778 ISD::NodeType ISD;
6779 if (ValTy->isIntOrIntVectorTy()) {
6780 ISD = (IID == Intrinsic::umin || IID == Intrinsic::umax) ? ISD::UMIN
6781 : ISD::SMIN;
6782 } else {
6783 assert(ValTy->isFPOrFPVectorTy() &&
6784 "Expected float point or integer vector type.");
6785 ISD = (IID == Intrinsic::minnum || IID == Intrinsic::maxnum)
6786 ? ISD::FMINNUM
6787 : ISD::FMINIMUM;
6788 }
6789
6790 // We use llvm-mca across all supported CPUs to measure the cost stats.
6791 static const CostKindTblEntry SSE2CostTbl[] = {
6792 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6793 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6794 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6795 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6796 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 7,.CodeSizeCost: 11,.SizeAndLatencyCost: 12}},
6797 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 7,.CodeSizeCost: 14,.SizeAndLatencyCost: 15}},
6798 {.ISD: ISD::SMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6799 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6800 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6801 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 8, .SizeAndLatencyCost: 10}},
6802 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}},
6803 {.ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8,.CodeSizeCost: 12,.SizeAndLatencyCost: 14}},
6804 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 5, .SizeAndLatencyCost: 6}},
6805 {.ISD: ISD::UMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6806 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 6,.CodeSizeCost: 12,.SizeAndLatencyCost: 13}},
6807 {.ISD: ISD::UMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6808 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 5, .LatencyCost: 9,.CodeSizeCost: 18,.SizeAndLatencyCost: 19}},
6809 {.ISD: ISD::UMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6810 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 7,.LatencyCost: 13,.CodeSizeCost: 24,.SizeAndLatencyCost: 25}},
6811 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10,.CodeSizeCost: 11,.SizeAndLatencyCost: 11}},
6812 };
6813
6814 static const CostKindTblEntry SSE41CostTbl[] = {
6815 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6816 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 4, .SizeAndLatencyCost: 6}},
6817 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6818 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6819 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6820 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6821 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6822 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6823 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 5, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6824 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 8, .CodeSizeCost: 4, .SizeAndLatencyCost: 5}},
6825 {.ISD: ISD::UMIN, .Type: MVT::v8i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 5, .CodeSizeCost: 2, .SizeAndLatencyCost: 2}},
6826 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 3, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6827 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6828 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 8, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6829 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 7, .SizeAndLatencyCost: 8}},
6830 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 8, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6831 };
6832
6833 static const CostKindTblEntry AVX1CostTbl[] = {
6834 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10}},
6835 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 6,.LatencyCost: 12,.CodeSizeCost: 10,.SizeAndLatencyCost: 13}},
6836 {.ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6837 {.ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 4, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6838 {.ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 15, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6839 {.ISD: ISD::UMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 9, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6840 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6841 {.ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 11, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6842 };
6843
6844 static const CostKindTblEntry AVX2CostTbl[] = {
6845 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 11, .CodeSizeCost: 7,.SizeAndLatencyCost: 10}},
6846 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12,.CodeSizeCost: 10,.SizeAndLatencyCost: 13}},
6847 {.ISD: ISD::SMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6848 {.ISD: ISD::UMIN, .Type: MVT::v2i32, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6849 {.ISD: ISD::UMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6850 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6851 {.ISD: ISD::SMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6852 {.ISD: ISD::UMIN, .Type: MVT::v8i32, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 9, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6853 {.ISD: ISD::SMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6854 {.ISD: ISD::UMIN, .Type: MVT::v4i16, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6855 {.ISD: ISD::SMIN, .Type: MVT::v16i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 15, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6856 {.ISD: ISD::SMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6857 {.ISD: ISD::UMIN, .Type: MVT::v8i8, .Cost: {.RecipThroughputCost: 3, .LatencyCost: 6, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6858 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6859 };
6860
6861 static const CostKindTblEntry AVX512FCostTbl[] = {
6862 {.ISD: ISD::SMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6863 {.ISD: ISD::UMIN, .Type: MVT::v2i64, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6864 {.ISD: ISD::SMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6865 {.ISD: ISD::UMIN, .Type: MVT::v4i64, .Cost: {.RecipThroughputCost: 3,.LatencyCost: 10, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6866 {.ISD: ISD::SMIN, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 16, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6867 {.ISD: ISD::UMIN, .Type: MVT::v8i64, .Cost: {.RecipThroughputCost: 5,.LatencyCost: 16, .CodeSizeCost: 7, .SizeAndLatencyCost: 7}},
6868 {.ISD: ISD::SMIN, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6869 {.ISD: ISD::UMIN, .Type: MVT::v16i32, .Cost: {.RecipThroughputCost: 4,.LatencyCost: 12, .CodeSizeCost: 9, .SizeAndLatencyCost: 9}},
6870 };
6871
6872 static const CostKindTblEntry AVX512BWCostTbl[] = {
6873 {.ISD: ISD::SMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6874 {.ISD: ISD::UMIN, .Type: MVT::v2i16, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6875 {.ISD: ISD::SMIN, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 19, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6876 {.ISD: ISD::UMIN, .Type: MVT::v32i16, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 12, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6877 {.ISD: ISD::SMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6878 {.ISD: ISD::UMIN, .Type: MVT::v2i8, .Cost: {.RecipThroughputCost: 1, .LatencyCost: 2, .CodeSizeCost: 3, .SizeAndLatencyCost: 3}},
6879 {.ISD: ISD::SMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6880 {.ISD: ISD::UMIN, .Type: MVT::v4i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 4, .CodeSizeCost: 5, .SizeAndLatencyCost: 5}},
6881 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 7}},
6882 {.ISD: ISD::UMIN, .Type: MVT::v16i8, .Cost: {.RecipThroughputCost: 2, .LatencyCost: 6, .CodeSizeCost: 4, .SizeAndLatencyCost: 4}},
6883 {.ISD: ISD::SMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 17, .CodeSizeCost: 8, .SizeAndLatencyCost: 9}},
6884 {.ISD: ISD::UMIN, .Type: MVT::v32i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 10, .CodeSizeCost: 6, .SizeAndLatencyCost: 6}},
6885 {.ISD: ISD::SMIN, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 21,.CodeSizeCost: 10,.SizeAndLatencyCost: 11}},
6886 {.ISD: ISD::UMIN, .Type: MVT::v64i8, .Cost: {.RecipThroughputCost: 2,.LatencyCost: 14, .CodeSizeCost: 8, .SizeAndLatencyCost: 8}},
6887 };
6888
6889 // Before legalizing the type, give a chance to look up illegal narrow types
6890 // in the table.
6891 // FIXME: Is there a better way to do this?
6892 EVT VT = TLI->getValueType(DL, Ty: ValTy);
6893 if (VT.isSimple()) {
6894 MVT MTy = VT.getSimpleVT();
6895 if (ST->hasBWI())
6896 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6897 if (auto KindCost = Entry->Cost[CostKind])
6898 return *KindCost;
6899
6900 if (ST->hasAVX512())
6901 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6902 if (auto KindCost = Entry->Cost[CostKind])
6903 return *KindCost;
6904
6905 if (ST->hasAVX2())
6906 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6907 if (auto KindCost = Entry->Cost[CostKind])
6908 return *KindCost;
6909
6910 if (ST->hasAVX())
6911 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6912 if (auto KindCost = Entry->Cost[CostKind])
6913 return *KindCost;
6914
6915 if (ST->hasSSE41())
6916 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
6917 if (auto KindCost = Entry->Cost[CostKind])
6918 return *KindCost;
6919
6920 if (ST->hasSSE2())
6921 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6922 if (auto KindCost = Entry->Cost[CostKind])
6923 return *KindCost;
6924 }
6925
6926 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6927 unsigned NumVecElts = ValVTy->getNumElements();
6928
6929 auto *Ty = ValVTy;
6930 InstructionCost MinMaxCost = 0;
6931 if (LT.first != 1 && MTy.isVector() &&
6932 MTy.getVectorNumElements() < ValVTy->getNumElements()) {
6933 // Type needs to be split. We need LT.first - 1 operations ops.
6934 Ty = FixedVectorType::get(ElementType: ValVTy->getElementType(),
6935 NumElts: MTy.getVectorNumElements());
6936 MinMaxCost = getMinMaxCost(IID, Ty, CostKind, FMF);
6937 MinMaxCost *= LT.first - 1;
6938 NumVecElts = MTy.getVectorNumElements();
6939 }
6940
6941 if (ST->hasBWI())
6942 if (const auto *Entry = CostTableLookup(Table: AVX512BWCostTbl, ISD, Ty: MTy))
6943 if (auto KindCost = Entry->Cost[CostKind])
6944 return MinMaxCost + *KindCost;
6945
6946 if (ST->hasAVX512())
6947 if (const auto *Entry = CostTableLookup(Table: AVX512FCostTbl, ISD, Ty: MTy))
6948 if (auto KindCost = Entry->Cost[CostKind])
6949 return MinMaxCost + *KindCost;
6950
6951 if (ST->hasAVX2())
6952 if (const auto *Entry = CostTableLookup(Table: AVX2CostTbl, ISD, Ty: MTy))
6953 if (auto KindCost = Entry->Cost[CostKind])
6954 return MinMaxCost + *KindCost;
6955
6956 if (ST->hasAVX())
6957 if (const auto *Entry = CostTableLookup(Table: AVX1CostTbl, ISD, Ty: MTy))
6958 if (auto KindCost = Entry->Cost[CostKind])
6959 return MinMaxCost + *KindCost;
6960
6961 if (ST->hasSSE41())
6962 if (const auto *Entry = CostTableLookup(Table: SSE41CostTbl, ISD, Ty: MTy))
6963 if (auto KindCost = Entry->Cost[CostKind])
6964 return MinMaxCost + *KindCost;
6965
6966 if (ST->hasSSE2())
6967 if (const auto *Entry = CostTableLookup(Table: SSE2CostTbl, ISD, Ty: MTy))
6968 if (auto KindCost = Entry->Cost[CostKind])
6969 return MinMaxCost + *KindCost;
6970
6971 unsigned ScalarSize = ValTy->getScalarSizeInBits();
6972
6973 // Special case power of 2 reductions where the scalar type isn't changed
6974 // by type legalization.
6975 if (!isPowerOf2_32(Value: ValVTy->getNumElements()) ||
6976 ScalarSize != MTy.getScalarSizeInBits())
6977 return BaseT::getMinMaxReductionCost(IID, Ty: ValTy, FMF, CostKind);
6978
6979 // Now handle reduction with the legal type, taking into account size changes
6980 // at each level.
6981 while (NumVecElts > 1) {
6982 // Determine the size of the remaining vector we need to reduce.
6983 unsigned Size = NumVecElts * ScalarSize;
6984 NumVecElts /= 2;
6985 // If we're reducing from 256/512 bits, use an extract_subvector.
6986 if (Size > 128) {
6987 auto *SubTy = FixedVectorType::get(ElementType: ValVTy->getElementType(), NumElts: NumVecElts);
6988 MinMaxCost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: Ty, SrcTy: Ty, CostKind,
6989 Mask: {}, Index: NumVecElts, SubTp: SubTy);
6990 Ty = SubTy;
6991 } else if (Size == 128) {
6992 // Reducing from 128 bits is a permute of v2f64/v2i64.
6993 VectorType *ShufTy;
6994 if (ValTy->isFloatingPointTy())
6995 ShufTy =
6996 FixedVectorType::get(ElementType: Type::getDoubleTy(C&: ValTy->getContext()), NumElts: 2);
6997 else
6998 ShufTy = FixedVectorType::get(ElementType: Type::getInt64Ty(C&: ValTy->getContext()), NumElts: 2);
6999 MinMaxCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
7000 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7001 } else if (Size == 64) {
7002 // Reducing from 64 bits is a shuffle of v4f32/v4i32.
7003 FixedVectorType *ShufTy;
7004 if (ValTy->isFloatingPointTy())
7005 ShufTy = FixedVectorType::get(ElementType: Type::getFloatTy(C&: ValTy->getContext()), NumElts: 4);
7006 else
7007 ShufTy = FixedVectorType::get(ElementType: Type::getInt32Ty(C&: ValTy->getContext()), NumElts: 4);
7008 MinMaxCost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: ShufTy, SrcTy: ShufTy,
7009 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7010 } else {
7011 // Reducing from smaller size is a shift by immediate.
7012 auto *ShiftTy = FixedVectorType::get(
7013 ElementType: Type::getIntNTy(C&: ValTy->getContext(), N: Size), NumElts: 128 / Size);
7014 MinMaxCost += getArithmeticInstrCost(
7015 Opcode: Instruction::LShr, Ty: ShiftTy, CostKind: TTI::TCK_RecipThroughput,
7016 Op1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
7017 Op2Info: {.Kind: TargetTransformInfo::OK_UniformConstantValue, .Properties: TargetTransformInfo::OP_None});
7018 }
7019
7020 // Add the arithmetic op for this level.
7021 MinMaxCost += getMinMaxCost(IID, Ty, CostKind, FMF);
7022 }
7023
7024 // Add the final extract element to the cost.
7025 return MinMaxCost + getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty,
7026 CostKind, Index: 0, Op0: nullptr, Op1: nullptr,
7027 VIC: TTI::VectorInstrContext::None);
7028}
7029
7030/// Calculate the cost of materializing a 64-bit value. This helper
7031/// method might only calculate a fraction of a larger immediate. Therefore it
7032/// is valid to return a cost of ZERO.
7033InstructionCost X86TTIImpl::getIntImmCost(int64_t Val) const {
7034 if (Val == 0)
7035 return TTI::TCC_Free;
7036
7037 if (isInt<32>(x: Val))
7038 return TTI::TCC_Basic;
7039
7040 return 2 * TTI::TCC_Basic;
7041}
7042
7043InstructionCost X86TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
7044 TTI::TargetCostKind CostKind) const {
7045 assert(Ty->isIntegerTy());
7046
7047 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7048 if (BitSize == 0)
7049 return ~0U;
7050
7051 // Never hoist constants larger than 128bit, because this might lead to
7052 // incorrect code generation or assertions in codegen.
7053 // Fixme: Create a cost model for types larger than i128 once the codegen
7054 // issues have been fixed.
7055 if (BitSize > 128)
7056 return TTI::TCC_Free;
7057
7058 if (Imm == 0)
7059 return TTI::TCC_Free;
7060
7061 // Sign-extend all constants to a multiple of 64-bit.
7062 APInt ImmVal = Imm;
7063 if (BitSize % 64 != 0)
7064 ImmVal = Imm.sext(width: alignTo(Value: BitSize, Align: 64));
7065
7066 // Split the constant into 64-bit chunks and calculate the cost for each
7067 // chunk.
7068 InstructionCost Cost = 0;
7069 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
7070 APInt Tmp = ImmVal.ashr(ShiftAmt: ShiftVal).sextOrTrunc(width: 64);
7071 int64_t Val = Tmp.getSExtValue();
7072 Cost += getIntImmCost(Val);
7073 }
7074 // We need at least one instruction to materialize the constant.
7075 return std::max<InstructionCost>(a: 1, b: Cost);
7076}
7077
7078InstructionCost X86TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
7079 const APInt &Imm, Type *Ty,
7080 TTI::TargetCostKind CostKind,
7081 Instruction *Inst) const {
7082 assert(Ty->isIntegerTy());
7083
7084 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7085 unsigned ImmBitWidth = Imm.getBitWidth();
7086
7087 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7088 // here, so that constant hoisting will ignore this constant.
7089 if (BitSize == 0)
7090 return TTI::TCC_Free;
7091
7092 unsigned ImmIdx = ~0U;
7093 switch (Opcode) {
7094 default:
7095 return TTI::TCC_Free;
7096 case Instruction::GetElementPtr:
7097 // Always hoist the base address of a GetElementPtr. This prevents the
7098 // creation of new constants for every base constant that gets constant
7099 // folded with the offset.
7100 if (Idx == 0)
7101 return 2 * TTI::TCC_Basic;
7102 return TTI::TCC_Free;
7103 case Instruction::Store:
7104 ImmIdx = 0;
7105 break;
7106 case Instruction::ICmp:
7107 // This is an imperfect hack to prevent constant hoisting of
7108 // compares that might be trying to check if a 64-bit value fits in
7109 // 32-bits. The backend can optimize these cases using a right shift by 32.
7110 // There are other predicates and immediates the backend can use shifts for.
7111 if (Idx == 1 && ImmBitWidth == 64) {
7112 uint64_t ImmVal = Imm.getZExtValue();
7113 if (ImmVal == 0x100000000ULL || ImmVal == 0xffffffff)
7114 return TTI::TCC_Free;
7115
7116 if (auto *Cmp = dyn_cast_or_null<CmpInst>(Val: Inst)) {
7117 if (Cmp->isEquality()) {
7118 KnownBits Known = computeKnownBits(V: Cmp->getOperand(i_nocapture: 0), DL);
7119 if (Known.countMinTrailingZeros() >= 32)
7120 return TTI::TCC_Free;
7121 }
7122 }
7123 }
7124 ImmIdx = 1;
7125 break;
7126 case Instruction::And:
7127 // We support 64-bit ANDs with immediates with 32-bits of leading zeroes
7128 // by using a 32-bit operation with implicit zero extension. Detect such
7129 // immediates here as the normal path expects bit 31 to be sign extended.
7130 if (Idx == 1 && ImmBitWidth == 64 && Imm.isIntN(N: 32))
7131 return TTI::TCC_Free;
7132 // If we have BMI then we can use BEXTR/BZHI to mask out upper i64 bits.
7133 if (Idx == 1 && ImmBitWidth == 64 && ST->is64Bit() && ST->hasBMI() &&
7134 Imm.isMask())
7135 return X86TTIImpl::getIntImmCost(Val: ST->hasBMI2() ? 255 : 65535);
7136 ImmIdx = 1;
7137 break;
7138 case Instruction::Add:
7139 case Instruction::Sub:
7140 // For add/sub, we can use the opposite instruction for INT32_MIN.
7141 if (Idx == 1 && ImmBitWidth == 64 && Imm.getZExtValue() == 0x80000000)
7142 return TTI::TCC_Free;
7143 ImmIdx = 1;
7144 break;
7145 case Instruction::UDiv:
7146 case Instruction::SDiv:
7147 case Instruction::URem:
7148 case Instruction::SRem:
7149 // Division by constant is typically expanded later into a different
7150 // instruction sequence. This completely changes the constants.
7151 // Report them as "free" to stop ConstantHoist from marking them as opaque.
7152 return TTI::TCC_Free;
7153 case Instruction::Mul:
7154 case Instruction::Or:
7155 case Instruction::Xor:
7156 ImmIdx = 1;
7157 break;
7158 // Always return TCC_Free for the shift value of a shift instruction.
7159 case Instruction::Shl:
7160 case Instruction::LShr:
7161 case Instruction::AShr:
7162 if (Idx == 1)
7163 return TTI::TCC_Free;
7164 break;
7165 case Instruction::Trunc:
7166 case Instruction::ZExt:
7167 case Instruction::SExt:
7168 case Instruction::IntToPtr:
7169 case Instruction::PtrToInt:
7170 case Instruction::BitCast:
7171 case Instruction::PHI:
7172 case Instruction::Call:
7173 case Instruction::Select:
7174 case Instruction::Ret:
7175 case Instruction::Load:
7176 break;
7177 }
7178
7179 if (Idx == ImmIdx) {
7180 uint64_t NumConstants = divideCeil(Numerator: BitSize, Denominator: 64);
7181 InstructionCost Cost = X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7182 return (Cost <= NumConstants * TTI::TCC_Basic)
7183 ? static_cast<int>(TTI::TCC_Free)
7184 : Cost;
7185 }
7186
7187 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7188}
7189
7190InstructionCost
7191X86TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
7192 const APInt &Imm, Type *Ty,
7193 TTI::TargetCostKind CostKind) const {
7194 assert(Ty->isIntegerTy());
7195
7196 unsigned BitSize = Ty->getPrimitiveSizeInBits();
7197 // There is no cost model for constants with a bit size of 0. Return TCC_Free
7198 // here, so that constant hoisting will ignore this constant.
7199 if (BitSize == 0)
7200 return TTI::TCC_Free;
7201
7202 switch (IID) {
7203 default:
7204 return TTI::TCC_Free;
7205 case Intrinsic::sadd_with_overflow:
7206 case Intrinsic::uadd_with_overflow:
7207 case Intrinsic::ssub_with_overflow:
7208 case Intrinsic::usub_with_overflow:
7209 case Intrinsic::smul_with_overflow:
7210 case Intrinsic::umul_with_overflow:
7211 if ((Idx == 1) && Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 32))
7212 return TTI::TCC_Free;
7213 break;
7214 case Intrinsic::experimental_stackmap:
7215 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 64)))
7216 return TTI::TCC_Free;
7217 break;
7218 case Intrinsic::experimental_patchpoint_void:
7219 case Intrinsic::experimental_patchpoint:
7220 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && Imm.isSignedIntN(N: 64)))
7221 return TTI::TCC_Free;
7222 break;
7223 }
7224 return X86TTIImpl::getIntImmCost(Imm, Ty, CostKind);
7225}
7226
7227InstructionCost X86TTIImpl::getCFInstrCost(unsigned Opcode,
7228 TTI::TargetCostKind CostKind,
7229 const Instruction *I) const {
7230 if (CostKind != TTI::TCK_RecipThroughput)
7231 return Opcode == Instruction::PHI ? TTI::TCC_Free : TTI::TCC_Basic;
7232 // Branches are assumed to be predicted.
7233 return TTI::TCC_Free;
7234}
7235
7236int X86TTIImpl::getGatherOverhead() const {
7237 // Some CPUs have more overhead for gather. The specified overhead is relative
7238 // to the Load operation. "2" is the number provided by Intel architects. This
7239 // parameter is used for cost estimation of Gather Op and comparison with
7240 // other alternatives.
7241 // TODO: Remove the explicit hasAVX512()?, That would mean we would only
7242 // enable gather with a -march.
7243 if (ST->hasAVX512() || (ST->hasAVX2() && ST->hasFastGather()))
7244 return 2;
7245
7246 return 1024;
7247}
7248
7249int X86TTIImpl::getScatterOverhead() const {
7250 if (ST->hasAVX512())
7251 return 2;
7252
7253 return 1024;
7254}
7255
7256// Return an average cost of Gather / Scatter instruction, maybe improved later.
7257InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode,
7258 TTI::TargetCostKind CostKind,
7259 Type *SrcVTy, const Value *Ptr,
7260 Align Alignment,
7261 unsigned AddressSpace) const {
7262
7263 assert(isa<VectorType>(SrcVTy) && "Unexpected type in getGSVectorCost");
7264 unsigned VF = cast<FixedVectorType>(Val: SrcVTy)->getNumElements();
7265
7266 // Try to reduce index size from 64 bit (default for GEP)
7267 // to 32. It is essential for VF 16. If the index can't be reduced to 32, the
7268 // operation will use 16 x 64 indices which do not fit in a zmm and needs
7269 // to split. Also check that the base pointer is the same for all lanes,
7270 // and that there's at most one variable index.
7271 auto getIndexSizeInBits = [](const Value *Ptr, const DataLayout &DL) {
7272 unsigned IndexSize = DL.getPointerSizeInBits();
7273 const GetElementPtrInst *GEP = dyn_cast_or_null<GetElementPtrInst>(Val: Ptr);
7274 if (IndexSize < 64 || !GEP)
7275 return IndexSize;
7276
7277 unsigned NumOfVarIndices = 0;
7278 const Value *Ptrs = GEP->getPointerOperand();
7279 if (Ptrs->getType()->isVectorTy() && !getSplatValue(V: Ptrs))
7280 return IndexSize;
7281 for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I) {
7282 if (isa<Constant>(Val: GEP->getOperand(i_nocapture: I)))
7283 continue;
7284 Type *IndxTy = GEP->getOperand(i_nocapture: I)->getType();
7285 if (auto *IndexVTy = dyn_cast<VectorType>(Val: IndxTy))
7286 IndxTy = IndexVTy->getElementType();
7287 if ((IndxTy->getPrimitiveSizeInBits() == 64 &&
7288 !isa<SExtInst>(Val: GEP->getOperand(i_nocapture: I))) ||
7289 ++NumOfVarIndices > 1)
7290 return IndexSize; // 64
7291 }
7292 return (unsigned)32;
7293 };
7294
7295 // Trying to reduce IndexSize to 32 bits for vector 16.
7296 // By default the IndexSize is equal to pointer size.
7297 unsigned IndexSize = (ST->hasAVX512() && VF >= 16)
7298 ? getIndexSizeInBits(Ptr, DL)
7299 : DL.getPointerSizeInBits();
7300
7301 auto *IndexVTy = FixedVectorType::get(
7302 ElementType: IntegerType::get(C&: SrcVTy->getContext(), NumBits: IndexSize), NumElts: VF);
7303 std::pair<InstructionCost, MVT> IdxsLT = getTypeLegalizationCost(Ty: IndexVTy);
7304 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: SrcVTy);
7305 InstructionCost::CostType SplitFactor =
7306 std::max(a: IdxsLT.first, b: SrcLT.first).getValue();
7307 if (SplitFactor > 1) {
7308 // Handle splitting of vector of pointers
7309 auto *SplitSrcTy =
7310 FixedVectorType::get(ElementType: SrcVTy->getScalarType(), NumElts: VF / SplitFactor);
7311 return SplitFactor * getGSVectorCost(Opcode, CostKind, SrcVTy: SplitSrcTy, Ptr,
7312 Alignment, AddressSpace);
7313 }
7314
7315 // If we didn't split, this will be a single gather/scatter instruction.
7316 if (CostKind == TTI::TCK_CodeSize)
7317 return 1;
7318
7319 // The gather / scatter cost is given by Intel architects. It is a rough
7320 // number since we are looking at one instruction in a time.
7321 const int GSOverhead = (Opcode == Instruction::Load) ? getGatherOverhead()
7322 : getScatterOverhead();
7323 return GSOverhead + VF * getMemoryOpCost(Opcode, Src: SrcVTy->getScalarType(),
7324 Alignment, AddressSpace, CostKind);
7325}
7326
7327/// Calculate the cost of Gather / Scatter operation
7328InstructionCost
7329X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
7330 TTI::TargetCostKind CostKind) const {
7331 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
7332 MICA.getID() == Intrinsic::vp_gather;
7333 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
7334 Type *SrcVTy = MICA.getDataType();
7335 const Value *Ptr = MICA.getPointer();
7336 Align Alignment = MICA.getAlignment();
7337 if ((Opcode == Instruction::Load &&
7338 (!isLegalMaskedGather(DataType: SrcVTy, Alignment: Align(Alignment)) ||
7339 forceScalarizeMaskedGather(VTy: cast<VectorType>(Val: SrcVTy),
7340 Alignment: Align(Alignment)))) ||
7341 (Opcode == Instruction::Store &&
7342 (!isLegalMaskedScatter(DataType: SrcVTy, Alignment: Align(Alignment)) ||
7343 forceScalarizeMaskedScatter(VTy: cast<VectorType>(Val: SrcVTy),
7344 Alignment: Align(Alignment)))))
7345 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
7346
7347 assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
7348 unsigned AddressSpace = MICA.getAddressSpace();
7349 InstructionCost Cost =
7350 getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment, AddressSpace);
7351
7352 // Predicate fanout (see getPredicateFanoutCost); variable mask only.
7353 if (MICA.getVariableMask()) {
7354 auto *MaskVecTy =
7355 FixedVectorType::get(ElementType: Type::getInt1Ty(C&: SrcVTy->getContext()),
7356 NumElts: cast<FixedVectorType>(Val: SrcVTy)->getNumElements());
7357 Cost += getPredicateFanoutCost(TTI: *this, DataVTy: SrcVTy, MaskVTy: MaskVecTy);
7358 }
7359 return Cost;
7360}
7361
7362bool X86TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
7363 const TargetTransformInfo::LSRCost &C2) const {
7364 // X86 specific here are "instruction number 1st priority".
7365 return std::tie(args: C1.Insns, args: C1.NumRegs, args: C1.AddRecCost, args: C1.NumIVMuls,
7366 args: C1.NumBaseAdds, args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
7367 std::tie(args: C2.Insns, args: C2.NumRegs, args: C2.AddRecCost, args: C2.NumIVMuls,
7368 args: C2.NumBaseAdds, args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
7369}
7370
7371bool X86TTIImpl::canMacroFuseCmp() const {
7372 return ST->hasMacroFusion() || ST->hasBranchFusion();
7373}
7374
7375static bool isLegalMaskedLoadStore(Type *ScalarTy, const X86Subtarget *ST) {
7376 if (!ST->hasAVX())
7377 return false;
7378
7379 if (ScalarTy->isPointerTy())
7380 return true;
7381
7382 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7383 return true;
7384
7385 if ((ScalarTy->isHalfTy() || ScalarTy->isBFloatTy()) && ST->hasBWI())
7386 return true;
7387
7388 if (!ScalarTy->isIntegerTy())
7389 return false;
7390
7391 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7392 return IntWidth == 32 || IntWidth == 64 ||
7393 ((IntWidth == 8 || IntWidth == 16) && ST->hasBWI());
7394}
7395
7396bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
7397 unsigned AddressSpace,
7398 TTI::MaskKind MaskKind) const {
7399 Type *ScalarTy = DataTy->getScalarType();
7400
7401 // The backend can't handle a single element vector w/o CFCMOV.
7402 if (isa<VectorType>(Val: DataTy) &&
7403 cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7404 return ST->hasCF() &&
7405 hasConditionalLoadStoreForType(Ty: ScalarTy, /*IsStore=*/false);
7406
7407 return isLegalMaskedLoadStore(ScalarTy, ST);
7408}
7409
7410bool X86TTIImpl::isLegalMaskedStore(Type *DataTy, Align Alignment,
7411 unsigned AddressSpace,
7412 TTI::MaskKind MaskKind) const {
7413 Type *ScalarTy = DataTy->getScalarType();
7414
7415 // The backend can't handle a single element vector w/o CFCMOV.
7416 if (isa<VectorType>(Val: DataTy) &&
7417 cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7418 return ST->hasCF() &&
7419 hasConditionalLoadStoreForType(Ty: ScalarTy, /*IsStore=*/true);
7420
7421 return isLegalMaskedLoadStore(ScalarTy, ST);
7422}
7423
7424bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) const {
7425 unsigned DataSize = DL.getTypeStoreSize(Ty: DataType);
7426 // The only supported nontemporal loads are for aligned vectors of 16 or 32
7427 // bytes. Note that 32-byte nontemporal vector loads are supported by AVX2
7428 // (the equivalent stores only require AVX).
7429 if (Alignment >= DataSize && (DataSize == 16 || DataSize == 32))
7430 return DataSize == 16 ? ST->hasSSE1() : ST->hasAVX2();
7431
7432 return false;
7433}
7434
7435bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) const {
7436 unsigned DataSize = DL.getTypeStoreSize(Ty: DataType);
7437
7438 // SSE4A supports nontemporal stores of float and double at arbitrary
7439 // alignment.
7440 if (ST->hasSSE4A() && (DataType->isFloatTy() || DataType->isDoubleTy()))
7441 return true;
7442
7443 // Besides the SSE4A subtarget exception above, only aligned stores are
7444 // available nontemporaly on any other subtarget. And only stores with a size
7445 // of 4..32 bytes (powers of 2, only) are permitted.
7446 if (Alignment < DataSize || DataSize < 4 || DataSize > 32 ||
7447 !isPowerOf2_32(Value: DataSize))
7448 return false;
7449
7450 // 32-byte vector nontemporal stores are supported by AVX (the equivalent
7451 // loads require AVX2).
7452 if (DataSize == 32)
7453 return ST->hasAVX();
7454 if (DataSize == 16)
7455 return ST->hasSSE1();
7456 return true;
7457}
7458
7459bool X86TTIImpl::isLegalBroadcastLoad(Type *ElementTy,
7460 ElementCount NumElements) const {
7461 // movddup
7462 return ST->hasSSE3() && !NumElements.isScalable() &&
7463 NumElements.getFixedValue() == 2 &&
7464 ElementTy == Type::getDoubleTy(C&: ElementTy->getContext());
7465}
7466
7467bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy, Align Alignment) const {
7468 if (!isa<VectorType>(Val: DataTy))
7469 return false;
7470
7471 if (!ST->hasAVX512())
7472 return false;
7473
7474 // The backend can't handle a single element vector.
7475 if (cast<FixedVectorType>(Val: DataTy)->getNumElements() == 1)
7476 return false;
7477
7478 Type *ScalarTy = cast<VectorType>(Val: DataTy)->getElementType();
7479
7480 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7481 return true;
7482
7483 if (ScalarTy->isHalfTy() || ScalarTy->isBFloatTy())
7484 return ST->hasVBMI2();
7485
7486 if (!ScalarTy->isIntegerTy())
7487 return false;
7488
7489 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7490 return IntWidth == 32 || IntWidth == 64 ||
7491 ((IntWidth == 8 || IntWidth == 16) && ST->hasVBMI2());
7492}
7493
7494bool X86TTIImpl::isLegalMaskedCompressStore(Type *DataTy,
7495 Align Alignment) const {
7496 return isLegalMaskedExpandLoad(DataTy, Alignment);
7497}
7498
7499bool X86TTIImpl::supportsGather() const {
7500 // Some CPUs have better gather performance than others.
7501 // TODO: Remove the explicit ST->hasAVX512()?, That would mean we would only
7502 // enable gather with a -march.
7503 return ST->hasAVX512() || (ST->hasFastGather() && ST->hasAVX2());
7504}
7505
7506bool X86TTIImpl::forceScalarizeMaskedGather(VectorType *VTy,
7507 Align Alignment) const {
7508 // Gather / Scatter for vector 2 is not profitable on KNL / SKX
7509 // Vector-4 of gather/scatter instruction does not exist on KNL. We can extend
7510 // it to 8 elements, but zeroing upper bits of the mask vector will add more
7511 // instructions. Right now we give the scalar cost of vector-4 for KNL. TODO:
7512 // Check, maybe the gather/scatter instruction is better in the VariableMask
7513 // case.
7514 unsigned NumElts = cast<FixedVectorType>(Val: VTy)->getNumElements();
7515 return NumElts == 1 ||
7516 (ST->hasAVX512() && (NumElts == 2 || (NumElts == 4 && !ST->hasVLX())));
7517}
7518
7519bool X86TTIImpl::isLegalMaskedGatherScatter(Type *DataTy,
7520 Align Alignment) const {
7521 Type *ScalarTy = DataTy->getScalarType();
7522 if (ScalarTy->isPointerTy())
7523 return true;
7524
7525 if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
7526 return true;
7527
7528 if (!ScalarTy->isIntegerTy())
7529 return false;
7530
7531 unsigned IntWidth = ScalarTy->getIntegerBitWidth();
7532 return IntWidth == 32 || IntWidth == 64;
7533}
7534
7535bool X86TTIImpl::isLegalMaskedGather(Type *DataTy, Align Alignment) const {
7536 if (!supportsGather() || !ST->preferGather())
7537 return false;
7538 return isLegalMaskedGatherScatter(DataTy, Alignment);
7539}
7540
7541bool X86TTIImpl::isLegalAltInstr(VectorType *VecTy, unsigned Opcode0,
7542 unsigned Opcode1,
7543 const SmallBitVector &OpcodeMask,
7544 ArrayRef<const Value *> Scalars) const {
7545 // ADDSUBPS 4xf32 SSE3
7546 // VADDSUBPS 4xf32 AVX
7547 // VADDSUBPS 8xf32 AVX2
7548 // ADDSUBPD 2xf64 SSE3
7549 // VADDSUBPD 2xf64 AVX
7550 // VADDSUBPD 4xf64 AVX2
7551
7552 unsigned NumElements = cast<FixedVectorType>(Val: VecTy)->getNumElements();
7553 assert(OpcodeMask.size() == NumElements && "Mask and VecTy are incompatible");
7554 if (!isPowerOf2_32(Value: NumElements))
7555 return false;
7556 // Check the opcode pattern. We apply the mask on the opcode arguments and
7557 // then check if it is what we expect.
7558 auto IsLaneOrder = [&](unsigned EvenOpc, unsigned OddOpc) {
7559 return all_of(Range: seq<unsigned>(Begin: 0, End: NumElements), P: [&](unsigned Lane) {
7560 return (OpcodeMask.test(Idx: Lane) ? Opcode1 : Opcode0) ==
7561 (Lane % 2 == 0 ? EvenOpc : OddOpc);
7562 });
7563 };
7564 // We expect FSub for even lanes and FAdd for odd lanes.
7565 const bool IsAddSub = IsLaneOrder(Instruction::FSub, Instruction::FAdd);
7566 // Now check that the pattern is supported by the target ISA.
7567 Type *ElemTy = cast<VectorType>(Val: VecTy)->getElementType();
7568 if (IsAddSub && ST->hasSSE3() &&
7569 ((ElemTy->isFloatTy() && NumElements % 4 == 0) ||
7570 (ElemTy->isDoubleTy() && NumElements % 2 == 0)))
7571 return true;
7572 // The multiplication is fused into every lane: fmaddsub (the even lanes
7573 // subtract and the odd lanes add the product) or fmsubadd (the other way
7574 // round). The subtracted product is the first operand of the subtraction.
7575 using namespace PatternMatch;
7576 auto IsFusedWithFMul = [](const Value *V) {
7577 const auto *I = dyn_cast<Instruction>(Val: V);
7578 if (!I || !I->hasAllowContract())
7579 return false;
7580 auto IsFMul = [&](const Value *Op) {
7581 return match(V: Op,
7582 P: m_OneUse(SubPattern: m_AllowContract(SubPattern: m_FMul(L: m_Value(), R: m_Value())))) &&
7583 cast<Instruction>(Val: Op)->getParent() == I->getParent();
7584 };
7585 return IsFMul(I->getOperand(i: 0)) ||
7586 (I->getOpcode() == Instruction::FAdd && IsFMul(I->getOperand(i: 1)));
7587 };
7588 return ST->hasFMA() && !Scalars.empty() &&
7589 (ElemTy->isFloatTy() || ElemTy->isDoubleTy()) &&
7590 (IsAddSub || IsLaneOrder(Instruction::FAdd, Instruction::FSub)) &&
7591 all_of(Range&: Scalars, P: IsFusedWithFMul);
7592}
7593
7594bool X86TTIImpl::isLegalMaskedScatter(Type *DataType, Align Alignment) const {
7595 // AVX2 doesn't support scatter
7596 if (!ST->hasAVX512() || !ST->preferScatter())
7597 return false;
7598 return isLegalMaskedGatherScatter(DataTy: DataType, Alignment);
7599}
7600
7601bool X86TTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) const {
7602 EVT VT = TLI->getValueType(DL, Ty: DataType);
7603 return TLI->isOperationLegal(Op: IsSigned ? ISD::SDIVREM : ISD::UDIVREM, VT);
7604}
7605
7606bool X86TTIImpl::isExpensiveToSpeculativelyExecute(const Instruction *I) const {
7607 // FDIV is always expensive, even if it has a very low uop count.
7608 // TODO: Still necessary for recent CPUs with low latency/throughput fdiv?
7609 if (I->getOpcode() == Instruction::FDiv)
7610 return true;
7611
7612 return BaseT::isExpensiveToSpeculativelyExecute(I);
7613}
7614
7615bool X86TTIImpl::isFCmpOrdCheaperThanFCmpZero(Type *Ty) const { return false; }
7616
7617bool X86TTIImpl::areInlineCompatible(const Function *Caller,
7618 const Function *Callee) const {
7619 const TargetMachine &TM = getTLI()->getTargetMachine();
7620
7621 // Work this as a subsetting of subtarget features.
7622 const X86Subtarget &CallerSubtarget = TM.getSubtarget<X86Subtarget>(F: *Caller);
7623 const X86Subtarget &CalleeSubtarget = TM.getSubtarget<X86Subtarget>(F: *Callee);
7624 const FeatureBitset &CallerBits = CallerSubtarget.getFeatureBits();
7625 const FeatureBitset &CalleeBits = CalleeSubtarget.getFeatureBits();
7626
7627 // Check whether callee features are a subset of caller features
7628 // (apart from the ignore list).
7629 const FeatureBitset &InlineIgnoreFeatures =
7630 CallerSubtarget.getInlineIgnoreFeatures();
7631 FeatureBitset RealCallerBits = CallerBits & ~InlineIgnoreFeatures;
7632 FeatureBitset RealCalleeBits = CalleeBits & ~InlineIgnoreFeatures;
7633 if ((RealCallerBits & RealCalleeBits) != RealCalleeBits)
7634 return false;
7635
7636 // If the features are not exactly the same (or there is a difference in
7637 // AVX512 register usage), we need to additionally check for calls
7638 // that may become ABI-incompatible as a result of inlining.
7639 if (RealCallerBits == RealCalleeBits &&
7640 CallerSubtarget.useAVX512Regs() == CalleeSubtarget.useAVX512Regs())
7641 return true;
7642
7643 for (const Instruction &I : instructions(F: Callee)) {
7644 if (const auto *CB = dyn_cast<CallBase>(Val: &I)) {
7645 // Having more target features is fine for inline ASM and intrinsics.
7646 if (CB->isInlineAsm() || CB->getIntrinsicID() != Intrinsic::not_intrinsic)
7647 continue;
7648
7649 SmallVector<Type *, 8> Types;
7650 for (Value *Arg : CB->args())
7651 Types.push_back(Elt: Arg->getType());
7652 if (!CB->getType()->isVoidTy())
7653 Types.push_back(Elt: CB->getType());
7654
7655 // Simple types are always ABI compatible.
7656 auto IsSimpleTy = [](Type *Ty) {
7657 return !Ty->isVectorTy() && !Ty->isAggregateType();
7658 };
7659 if (all_of(Range&: Types, P: IsSimpleTy))
7660 continue;
7661
7662 // Do a precise compatibility check.
7663 if (!areTypesABICompatible(Caller, Callee, Type: Types))
7664 return false;
7665 }
7666 }
7667 return true;
7668}
7669
7670bool X86TTIImpl::areTypesABICompatible(const Function *Caller,
7671 const Function *Callee,
7672 ArrayRef<Type *> Types) const {
7673 const TargetMachine &TM = getTLI()->getTargetMachine();
7674 const TargetLowering *CallerTLI =
7675 TM.getSubtargetImpl(*Caller)->getTargetLowering();
7676 const TargetLowering *CalleeTLI =
7677 TM.getSubtargetImpl(*Callee)->getTargetLowering();
7678
7679 LLVMContext &Ctx = Caller->getContext();
7680 const DataLayout &DL = Caller->getDataLayout();
7681 CallingConv::ID CC = Callee->getCallingConv();
7682 return all_of(Range&: Types, P: [&](Type *Ty) {
7683 SmallVector<EVT> VTs;
7684 ComputeValueVTs(TLI: *CallerTLI, DL, Ty, ValueVTs&: VTs);
7685 return all_of(Range&: VTs, P: [&](EVT VT) {
7686 return CallerTLI->getRegisterTypeForCallingConv(Context&: Ctx, CC, VT) ==
7687 CalleeTLI->getRegisterTypeForCallingConv(Context&: Ctx, CC, VT);
7688 });
7689 });
7690}
7691
7692X86TTIImpl::TTI::MemCmpExpansionOptions
7693X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
7694 TTI::MemCmpExpansionOptions Options;
7695 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
7696 Options.NumLoadsPerBlock = IsZeroCmp ? 2 : 1;
7697 // All GPR and vector loads can be unaligned.
7698 Options.AllowOverlappingLoads = true;
7699 if (IsZeroCmp) {
7700 // Only enable vector loads for equality comparison. Right now the vector
7701 // version is not as fast for three way compare (see #33329).
7702 const unsigned PreferredWidth = ST->getPreferVectorWidth();
7703 if (PreferredWidth >= 512 && ST->hasAVX512())
7704 Options.LoadSizes.push_back(Elt: 64);
7705 if (PreferredWidth >= 256 && ST->hasAVX()) Options.LoadSizes.push_back(Elt: 32);
7706 if (PreferredWidth >= 128 && ST->hasSSE2()) Options.LoadSizes.push_back(Elt: 16);
7707 }
7708 if (ST->is64Bit()) {
7709 Options.LoadSizes.push_back(Elt: 8);
7710 }
7711 Options.LoadSizes.push_back(Elt: 4);
7712 Options.LoadSizes.push_back(Elt: 2);
7713 Options.LoadSizes.push_back(Elt: 1);
7714 return Options;
7715}
7716
7717bool X86TTIImpl::prefersVectorizedAddressing() const {
7718 return supportsGather();
7719}
7720
7721bool X86TTIImpl::supportsEfficientVectorElementLoadStore() const {
7722 return false;
7723}
7724
7725bool X86TTIImpl::enableInterleavedAccessVectorization() const {
7726 // TODO: We expect this to be beneficial regardless of arch,
7727 // but there are currently some unexplained performance artifacts on Atom.
7728 // As a temporary solution, disable on Atom.
7729 return !(ST->isAtom());
7730}
7731
7732bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
7733 switch (II->getIntrinsicID()) {
7734 default:
7735 return true;
7736 case Intrinsic::vector_reduce_and:
7737 case Intrinsic::vector_reduce_or:
7738 case Intrinsic::vector_reduce_xor:
7739 case Intrinsic::vector_reduce_mul:
7740 case Intrinsic::vector_reduce_smax:
7741 case Intrinsic::vector_reduce_smin:
7742 case Intrinsic::vector_reduce_umax:
7743 case Intrinsic::vector_reduce_umin:
7744 return false;
7745 }
7746}
7747
7748// Get estimation for interleaved load/store operations and strided load.
7749// \p Indices contains indices for strided load.
7750// \p Factor - the factor of interleaving.
7751// AVX-512 provides 3-src shuffles that significantly reduces the cost.
7752InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512(
7753 unsigned Opcode, FixedVectorType *VecTy, unsigned Factor,
7754 ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace,
7755 TTI::TargetCostKind CostKind, bool UseMaskForCond,
7756 bool UseMaskForGaps) const {
7757 // VecTy for interleave memop is <VF*Factor x Elt>.
7758 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7759 // VecTy = <12 x i32>.
7760
7761 // Calculate the number of memory operations (NumOfMemOps), required
7762 // for load/store the VecTy.
7763 MVT LegalVT = getTypeLegalizationCost(Ty: VecTy).second;
7764 unsigned VecTySize = DL.getTypeStoreSize(Ty: VecTy);
7765 unsigned LegalVTSize = LegalVT.getStoreSize();
7766 unsigned NumOfMemOps = (VecTySize + LegalVTSize - 1) / LegalVTSize;
7767
7768 // Get the cost of one memory operation.
7769 auto *SingleMemOpTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
7770 NumElts: LegalVT.getVectorNumElements());
7771 InstructionCost MemOpCost;
7772 bool UseMaskedMemOp = UseMaskForCond || UseMaskForGaps;
7773 if (UseMaskedMemOp) {
7774 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
7775 : Intrinsic::masked_store;
7776 MemOpCost = getMaskedMemoryOpCost(
7777 MICA: {IID, SingleMemOpTy, Alignment, AddressSpace}, CostKind);
7778 } else
7779 MemOpCost = getMemoryOpCost(Opcode, Src: SingleMemOpTy, Alignment, AddressSpace,
7780 CostKind);
7781
7782 unsigned VF = VecTy->getNumElements() / Factor;
7783 MVT VT =
7784 MVT::getVectorVT(VT: TLI->getSimpleValueType(DL, Ty: VecTy->getScalarType()), NumElements: VF);
7785
7786 InstructionCost MaskCost;
7787 if (UseMaskedMemOp) {
7788 APInt DemandedLoadStoreElts = APInt::getZero(numBits: VecTy->getNumElements());
7789 for (unsigned Index : Indices) {
7790 assert(Index < Factor && "Invalid index for interleaved memory op");
7791 for (unsigned Elm = 0; Elm < VF; Elm++)
7792 DemandedLoadStoreElts.setBit(Index + Elm * Factor);
7793 }
7794
7795 Type *I1Type = Type::getInt1Ty(C&: VecTy->getContext());
7796
7797 MaskCost = getReplicationShuffleCost(
7798 EltTy: I1Type, ReplicationFactor: Factor, VF,
7799 DemandedDstElts: UseMaskForGaps ? DemandedLoadStoreElts
7800 : APInt::getAllOnes(numBits: VecTy->getNumElements()),
7801 CostKind);
7802
7803 // The Gaps mask is invariant and created outside the loop, therefore the
7804 // cost of creating it is not accounted for here. However if we have both
7805 // a MaskForGaps and some other mask that guards the execution of the
7806 // memory access, we need to account for the cost of And-ing the two masks
7807 // inside the loop.
7808 if (UseMaskForGaps) {
7809 auto *MaskVT = FixedVectorType::get(ElementType: I1Type, NumElts: VecTy->getNumElements());
7810 MaskCost += getArithmeticInstrCost(Opcode: BinaryOperator::And, Ty: MaskVT, CostKind);
7811 }
7812 }
7813
7814 if (Opcode == Instruction::Load) {
7815 // The tables (AVX512InterleavedLoadTbl and AVX512InterleavedStoreTbl)
7816 // contain the cost of the optimized shuffle sequence that the
7817 // X86InterleavedAccess pass will generate.
7818 // The cost of loads and stores are computed separately from the table.
7819
7820 // X86InterleavedAccess support only the following interleaved-access group.
7821 static const CostTblEntry AVX512InterleavedLoadTbl[] = {
7822 {.ISD: 3, .Type: MVT::v16i8, .Cost: 12}, //(load 48i8 and) deinterleave into 3 x 16i8
7823 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, //(load 96i8 and) deinterleave into 3 x 32i8
7824 {.ISD: 3, .Type: MVT::v64i8, .Cost: 22}, //(load 96i8 and) deinterleave into 3 x 32i8
7825 };
7826
7827 if (const auto *Entry =
7828 CostTableLookup(Table: AVX512InterleavedLoadTbl, ISD: Factor, Ty: VT))
7829 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7830 //If an entry does not exist, fallback to the default implementation.
7831
7832 // Kind of shuffle depends on number of loaded values.
7833 // If we load the entire data in one register, we can use a 1-src shuffle.
7834 // Otherwise, we'll merge 2 sources in each operation.
7835 TTI::ShuffleKind ShuffleKind =
7836 (NumOfMemOps > 1) ? TTI::SK_PermuteTwoSrc : TTI::SK_PermuteSingleSrc;
7837
7838 InstructionCost ShuffleCost = getShuffleCost(
7839 Kind: ShuffleKind, DstTy: SingleMemOpTy, SrcTy: SingleMemOpTy, CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7840
7841 unsigned NumOfLoadsInInterleaveGrp =
7842 Indices.size() ? Indices.size() : Factor;
7843 auto *ResultTy = FixedVectorType::get(ElementType: VecTy->getElementType(),
7844 NumElts: VecTy->getNumElements() / Factor);
7845 InstructionCost NumOfResults =
7846 getTypeLegalizationCost(Ty: ResultTy).first * NumOfLoadsInInterleaveGrp;
7847
7848 // About a half of the loads may be folded in shuffles when we have only
7849 // one result. If we have more than one result, or the loads are masked,
7850 // we do not fold loads at all.
7851 unsigned NumOfUnfoldedLoads =
7852 UseMaskedMemOp || NumOfResults > 1 ? NumOfMemOps : NumOfMemOps / 2;
7853
7854 // Get a number of shuffle operations per result.
7855 unsigned NumOfShufflesPerResult =
7856 std::max(a: (unsigned)1, b: (unsigned)(NumOfMemOps - 1));
7857
7858 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7859 // When we have more than one destination, we need additional instructions
7860 // to keep sources.
7861 InstructionCost NumOfMoves = 0;
7862 if (NumOfResults > 1 && ShuffleKind == TTI::SK_PermuteTwoSrc)
7863 NumOfMoves = NumOfResults * NumOfShufflesPerResult / 2;
7864
7865 InstructionCost Cost = NumOfResults * NumOfShufflesPerResult * ShuffleCost +
7866 MaskCost + NumOfUnfoldedLoads * MemOpCost +
7867 NumOfMoves;
7868
7869 return Cost;
7870 }
7871
7872 // Store.
7873 assert(Opcode == Instruction::Store &&
7874 "Expected Store Instruction at this point");
7875 // X86InterleavedAccess support only the following interleaved-access group.
7876 static const CostTblEntry AVX512InterleavedStoreTbl[] = {
7877 {.ISD: 3, .Type: MVT::v16i8, .Cost: 12}, // interleave 3 x 16i8 into 48i8 (and store)
7878 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, // interleave 3 x 32i8 into 96i8 (and store)
7879 {.ISD: 3, .Type: MVT::v64i8, .Cost: 26}, // interleave 3 x 64i8 into 96i8 (and store)
7880
7881 {.ISD: 4, .Type: MVT::v8i8, .Cost: 10}, // interleave 4 x 8i8 into 32i8 (and store)
7882 {.ISD: 4, .Type: MVT::v16i8, .Cost: 11}, // interleave 4 x 16i8 into 64i8 (and store)
7883 {.ISD: 4, .Type: MVT::v32i8, .Cost: 14}, // interleave 4 x 32i8 into 128i8 (and store)
7884 {.ISD: 4, .Type: MVT::v64i8, .Cost: 24} // interleave 4 x 32i8 into 256i8 (and store)
7885 };
7886
7887 if (const auto *Entry =
7888 CostTableLookup(Table: AVX512InterleavedStoreTbl, ISD: Factor, Ty: VT))
7889 return MaskCost + NumOfMemOps * MemOpCost + Entry->Cost;
7890 //If an entry does not exist, fallback to the default implementation.
7891
7892 // There is no strided stores meanwhile. And store can't be folded in
7893 // shuffle.
7894 unsigned NumOfSources = Factor; // The number of values to be merged.
7895 InstructionCost ShuffleCost =
7896 getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleMemOpTy, SrcTy: SingleMemOpTy,
7897 CostKind, Mask: {}, Index: 0, SubTp: nullptr);
7898 unsigned NumOfShufflesPerStore = NumOfSources - 1;
7899
7900 // The SK_MergeTwoSrc shuffle clobbers one of src operands.
7901 // We need additional instructions to keep sources.
7902 unsigned NumOfMoves = NumOfMemOps * NumOfShufflesPerStore / 2;
7903 InstructionCost Cost =
7904 MaskCost +
7905 NumOfMemOps * (MemOpCost + NumOfShufflesPerStore * ShuffleCost) +
7906 NumOfMoves;
7907 return Cost;
7908}
7909
7910InstructionCost X86TTIImpl::getInterleavedMemoryOpCost(
7911 unsigned Opcode, Type *BaseTy, unsigned Factor, ArrayRef<unsigned> Indices,
7912 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
7913 bool UseMaskForCond, bool UseMaskForGaps) const {
7914 auto *VecTy = cast<FixedVectorType>(Val: BaseTy);
7915
7916 auto isSupportedOnAVX512 = [&](Type *VecTy) {
7917 Type *EltTy = cast<VectorType>(Val: VecTy)->getElementType();
7918 if (EltTy->isFloatTy() || EltTy->isDoubleTy() || EltTy->isIntegerTy(BitWidth: 64) ||
7919 EltTy->isIntegerTy(BitWidth: 32) || EltTy->isPointerTy())
7920 return true;
7921 if (EltTy->isIntegerTy(BitWidth: 16) || EltTy->isIntegerTy(BitWidth: 8) || EltTy->isHalfTy())
7922 return ST->hasBWI();
7923 if (EltTy->isBFloatTy())
7924 return ST->hasBF16();
7925 return false;
7926 };
7927 if (ST->hasAVX512() && isSupportedOnAVX512(VecTy))
7928 return getInterleavedMemoryOpCostAVX512(
7929 Opcode, VecTy, Factor, Indices, Alignment,
7930 AddressSpace, CostKind, UseMaskForCond, UseMaskForGaps);
7931
7932 if (UseMaskForCond || UseMaskForGaps)
7933 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7934 Alignment, AddressSpace, CostKind,
7935 UseMaskForCond, UseMaskForGaps);
7936
7937 // Get estimation for interleaved load/store operations for SSE-AVX2.
7938 // As opposed to AVX-512, SSE-AVX2 do not have generic shuffles that allow
7939 // computing the cost using a generic formula as a function of generic
7940 // shuffles. We therefore use a lookup table instead, filled according to
7941 // the instruction sequences that codegen currently generates.
7942
7943 // VecTy for interleave memop is <VF*Factor x Elt>.
7944 // So, for VF=4, Interleave Factor = 3, Element type = i32 we have
7945 // VecTy = <12 x i32>.
7946 MVT LegalVT = getTypeLegalizationCost(Ty: VecTy).second;
7947
7948 // This function can be called with VecTy=<6xi128>, Factor=3, in which case
7949 // the VF=2, while v2i128 is an unsupported MVT vector type
7950 // (see MachineValueType.h::getVectorVT()).
7951 if (!LegalVT.isVector())
7952 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7953 Alignment, AddressSpace, CostKind);
7954
7955 unsigned VF = VecTy->getNumElements() / Factor;
7956 Type *ScalarTy = VecTy->getElementType();
7957 // Deduplicate entries, model floats/pointers as appropriately-sized integers.
7958 if (!ScalarTy->isIntegerTy())
7959 ScalarTy =
7960 Type::getIntNTy(C&: ScalarTy->getContext(), N: DL.getTypeSizeInBits(Ty: ScalarTy));
7961
7962 // Get the cost of all the memory operations.
7963 // FIXME: discount dead loads.
7964 InstructionCost MemOpCosts =
7965 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
7966
7967 auto *VT = FixedVectorType::get(ElementType: ScalarTy, NumElts: VF);
7968 EVT ETy = TLI->getValueType(DL, Ty: VT);
7969 if (!ETy.isSimple())
7970 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
7971 Alignment, AddressSpace, CostKind);
7972
7973 // TODO: Complete for other data-types and strides.
7974 // Each combination of Stride, element bit width and VF results in a different
7975 // sequence; The cost tables are therefore accessed with:
7976 // Factor (stride) and VectorType=VFxiN.
7977 // The Cost accounts only for the shuffle sequence;
7978 // The cost of the loads/stores is accounted for separately.
7979 //
7980 static const CostTblEntry AVX2InterleavedLoadTbl[] = {
7981 {.ISD: 2, .Type: MVT::v2i8, .Cost: 2}, // (load 4i8 and) deinterleave into 2 x 2i8
7982 {.ISD: 2, .Type: MVT::v4i8, .Cost: 2}, // (load 8i8 and) deinterleave into 2 x 4i8
7983 {.ISD: 2, .Type: MVT::v8i8, .Cost: 2}, // (load 16i8 and) deinterleave into 2 x 8i8
7984 {.ISD: 2, .Type: MVT::v16i8, .Cost: 4}, // (load 32i8 and) deinterleave into 2 x 16i8
7985 {.ISD: 2, .Type: MVT::v32i8, .Cost: 6}, // (load 64i8 and) deinterleave into 2 x 32i8
7986
7987 {.ISD: 2, .Type: MVT::v8i16, .Cost: 6}, // (load 16i16 and) deinterleave into 2 x 8i16
7988 {.ISD: 2, .Type: MVT::v16i16, .Cost: 9}, // (load 32i16 and) deinterleave into 2 x 16i16
7989 {.ISD: 2, .Type: MVT::v32i16, .Cost: 18}, // (load 64i16 and) deinterleave into 2 x 32i16
7990
7991 {.ISD: 2, .Type: MVT::v8i32, .Cost: 4}, // (load 16i32 and) deinterleave into 2 x 8i32
7992 {.ISD: 2, .Type: MVT::v16i32, .Cost: 8}, // (load 32i32 and) deinterleave into 2 x 16i32
7993 {.ISD: 2, .Type: MVT::v32i32, .Cost: 16}, // (load 64i32 and) deinterleave into 2 x 32i32
7994
7995 {.ISD: 2, .Type: MVT::v4i64, .Cost: 4}, // (load 8i64 and) deinterleave into 2 x 4i64
7996 {.ISD: 2, .Type: MVT::v8i64, .Cost: 8}, // (load 16i64 and) deinterleave into 2 x 8i64
7997 {.ISD: 2, .Type: MVT::v16i64, .Cost: 16}, // (load 32i64 and) deinterleave into 2 x 16i64
7998 {.ISD: 2, .Type: MVT::v32i64, .Cost: 32}, // (load 64i64 and) deinterleave into 2 x 32i64
7999
8000 {.ISD: 3, .Type: MVT::v2i8, .Cost: 3}, // (load 6i8 and) deinterleave into 3 x 2i8
8001 {.ISD: 3, .Type: MVT::v4i8, .Cost: 3}, // (load 12i8 and) deinterleave into 3 x 4i8
8002 {.ISD: 3, .Type: MVT::v8i8, .Cost: 6}, // (load 24i8 and) deinterleave into 3 x 8i8
8003 {.ISD: 3, .Type: MVT::v16i8, .Cost: 11}, // (load 48i8 and) deinterleave into 3 x 16i8
8004 {.ISD: 3, .Type: MVT::v32i8, .Cost: 14}, // (load 96i8 and) deinterleave into 3 x 32i8
8005
8006 {.ISD: 3, .Type: MVT::v2i16, .Cost: 5}, // (load 6i16 and) deinterleave into 3 x 2i16
8007 {.ISD: 3, .Type: MVT::v4i16, .Cost: 7}, // (load 12i16 and) deinterleave into 3 x 4i16
8008 {.ISD: 3, .Type: MVT::v8i16, .Cost: 9}, // (load 24i16 and) deinterleave into 3 x 8i16
8009 {.ISD: 3, .Type: MVT::v16i16, .Cost: 28}, // (load 48i16 and) deinterleave into 3 x 16i16
8010 {.ISD: 3, .Type: MVT::v32i16, .Cost: 56}, // (load 96i16 and) deinterleave into 3 x 32i16
8011
8012 {.ISD: 3, .Type: MVT::v2i32, .Cost: 3}, // (load 6i32 and) deinterleave into 3 x 2i32
8013 {.ISD: 3, .Type: MVT::v4i32, .Cost: 3}, // (load 12i32 and) deinterleave into 3 x 4i32
8014 {.ISD: 3, .Type: MVT::v8i32, .Cost: 7}, // (load 24i32 and) deinterleave into 3 x 8i32
8015 {.ISD: 3, .Type: MVT::v16i32, .Cost: 14}, // (load 48i32 and) deinterleave into 3 x 16i32
8016 {.ISD: 3, .Type: MVT::v32i32, .Cost: 32}, // (load 96i32 and) deinterleave into 3 x 32i32
8017
8018 {.ISD: 3, .Type: MVT::v2i64, .Cost: 1}, // (load 6i64 and) deinterleave into 3 x 2i64
8019 {.ISD: 3, .Type: MVT::v4i64, .Cost: 5}, // (load 12i64 and) deinterleave into 3 x 4i64
8020 {.ISD: 3, .Type: MVT::v8i64, .Cost: 10}, // (load 24i64 and) deinterleave into 3 x 8i64
8021 {.ISD: 3, .Type: MVT::v16i64, .Cost: 20}, // (load 48i64 and) deinterleave into 3 x 16i64
8022
8023 {.ISD: 4, .Type: MVT::v2i8, .Cost: 4}, // (load 8i8 and) deinterleave into 4 x 2i8
8024 {.ISD: 4, .Type: MVT::v4i8, .Cost: 4}, // (load 16i8 and) deinterleave into 4 x 4i8
8025 {.ISD: 4, .Type: MVT::v8i8, .Cost: 12}, // (load 32i8 and) deinterleave into 4 x 8i8
8026 {.ISD: 4, .Type: MVT::v16i8, .Cost: 24}, // (load 64i8 and) deinterleave into 4 x 16i8
8027 {.ISD: 4, .Type: MVT::v32i8, .Cost: 56}, // (load 128i8 and) deinterleave into 4 x 32i8
8028
8029 {.ISD: 4, .Type: MVT::v2i16, .Cost: 6}, // (load 8i16 and) deinterleave into 4 x 2i16
8030 {.ISD: 4, .Type: MVT::v4i16, .Cost: 17}, // (load 16i16 and) deinterleave into 4 x 4i16
8031 {.ISD: 4, .Type: MVT::v8i16, .Cost: 33}, // (load 32i16 and) deinterleave into 4 x 8i16
8032 {.ISD: 4, .Type: MVT::v16i16, .Cost: 75}, // (load 64i16 and) deinterleave into 4 x 16i16
8033 {.ISD: 4, .Type: MVT::v32i16, .Cost: 150}, // (load 128i16 and) deinterleave into 4 x 32i16
8034
8035 {.ISD: 4, .Type: MVT::v2i32, .Cost: 4}, // (load 8i32 and) deinterleave into 4 x 2i32
8036 {.ISD: 4, .Type: MVT::v4i32, .Cost: 8}, // (load 16i32 and) deinterleave into 4 x 4i32
8037 {.ISD: 4, .Type: MVT::v8i32, .Cost: 16}, // (load 32i32 and) deinterleave into 4 x 8i32
8038 {.ISD: 4, .Type: MVT::v16i32, .Cost: 32}, // (load 64i32 and) deinterleave into 4 x 16i32
8039 {.ISD: 4, .Type: MVT::v32i32, .Cost: 68}, // (load 128i32 and) deinterleave into 4 x 32i32
8040
8041 {.ISD: 4, .Type: MVT::v2i64, .Cost: 6}, // (load 8i64 and) deinterleave into 4 x 2i64
8042 {.ISD: 4, .Type: MVT::v4i64, .Cost: 8}, // (load 16i64 and) deinterleave into 4 x 4i64
8043 {.ISD: 4, .Type: MVT::v8i64, .Cost: 20}, // (load 32i64 and) deinterleave into 4 x 8i64
8044 {.ISD: 4, .Type: MVT::v16i64, .Cost: 40}, // (load 64i64 and) deinterleave into 4 x 16i64
8045
8046 {.ISD: 6, .Type: MVT::v2i8, .Cost: 6}, // (load 12i8 and) deinterleave into 6 x 2i8
8047 {.ISD: 6, .Type: MVT::v4i8, .Cost: 14}, // (load 24i8 and) deinterleave into 6 x 4i8
8048 {.ISD: 6, .Type: MVT::v8i8, .Cost: 18}, // (load 48i8 and) deinterleave into 6 x 8i8
8049 {.ISD: 6, .Type: MVT::v16i8, .Cost: 43}, // (load 96i8 and) deinterleave into 6 x 16i8
8050 {.ISD: 6, .Type: MVT::v32i8, .Cost: 82}, // (load 192i8 and) deinterleave into 6 x 32i8
8051
8052 {.ISD: 6, .Type: MVT::v2i16, .Cost: 13}, // (load 12i16 and) deinterleave into 6 x 2i16
8053 {.ISD: 6, .Type: MVT::v4i16, .Cost: 9}, // (load 24i16 and) deinterleave into 6 x 4i16
8054 {.ISD: 6, .Type: MVT::v8i16, .Cost: 39}, // (load 48i16 and) deinterleave into 6 x 8i16
8055 {.ISD: 6, .Type: MVT::v16i16, .Cost: 106}, // (load 96i16 and) deinterleave into 6 x 16i16
8056 {.ISD: 6, .Type: MVT::v32i16, .Cost: 212}, // (load 192i16 and) deinterleave into 6 x 32i16
8057
8058 {.ISD: 6, .Type: MVT::v2i32, .Cost: 6}, // (load 12i32 and) deinterleave into 6 x 2i32
8059 {.ISD: 6, .Type: MVT::v4i32, .Cost: 15}, // (load 24i32 and) deinterleave into 6 x 4i32
8060 {.ISD: 6, .Type: MVT::v8i32, .Cost: 31}, // (load 48i32 and) deinterleave into 6 x 8i32
8061 {.ISD: 6, .Type: MVT::v16i32, .Cost: 64}, // (load 96i32 and) deinterleave into 6 x 16i32
8062
8063 {.ISD: 6, .Type: MVT::v2i64, .Cost: 6}, // (load 12i64 and) deinterleave into 6 x 2i64
8064 {.ISD: 6, .Type: MVT::v4i64, .Cost: 18}, // (load 24i64 and) deinterleave into 6 x 4i64
8065 {.ISD: 6, .Type: MVT::v8i64, .Cost: 36}, // (load 48i64 and) deinterleave into 6 x 8i64
8066
8067 {.ISD: 8, .Type: MVT::v8i32, .Cost: 40} // (load 64i32 and) deinterleave into 8 x 8i32
8068 };
8069
8070 static const CostTblEntry SSSE3InterleavedLoadTbl[] = {
8071 {.ISD: 2, .Type: MVT::v4i16, .Cost: 2}, // (load 8i16 and) deinterleave into 2 x 4i16
8072 };
8073
8074 static const CostTblEntry SSE2InterleavedLoadTbl[] = {
8075 {.ISD: 2, .Type: MVT::v2i16, .Cost: 2}, // (load 4i16 and) deinterleave into 2 x 2i16
8076 {.ISD: 2, .Type: MVT::v4i16, .Cost: 7}, // (load 8i16 and) deinterleave into 2 x 4i16
8077
8078 {.ISD: 2, .Type: MVT::v2i32, .Cost: 2}, // (load 4i32 and) deinterleave into 2 x 2i32
8079 {.ISD: 2, .Type: MVT::v4i32, .Cost: 2}, // (load 8i32 and) deinterleave into 2 x 4i32
8080
8081 {.ISD: 2, .Type: MVT::v2i64, .Cost: 2}, // (load 4i64 and) deinterleave into 2 x 2i64
8082 };
8083
8084 static const CostTblEntry AVX2InterleavedStoreTbl[] = {
8085 {.ISD: 2, .Type: MVT::v16i8, .Cost: 3}, // interleave 2 x 16i8 into 32i8 (and store)
8086 {.ISD: 2, .Type: MVT::v32i8, .Cost: 4}, // interleave 2 x 32i8 into 64i8 (and store)
8087
8088 {.ISD: 2, .Type: MVT::v8i16, .Cost: 3}, // interleave 2 x 8i16 into 16i16 (and store)
8089 {.ISD: 2, .Type: MVT::v16i16, .Cost: 4}, // interleave 2 x 16i16 into 32i16 (and store)
8090 {.ISD: 2, .Type: MVT::v32i16, .Cost: 8}, // interleave 2 x 32i16 into 64i16 (and store)
8091
8092 {.ISD: 2, .Type: MVT::v4i32, .Cost: 2}, // interleave 2 x 4i32 into 8i32 (and store)
8093 {.ISD: 2, .Type: MVT::v8i32, .Cost: 4}, // interleave 2 x 8i32 into 16i32 (and store)
8094 {.ISD: 2, .Type: MVT::v16i32, .Cost: 8}, // interleave 2 x 16i32 into 32i32 (and store)
8095 {.ISD: 2, .Type: MVT::v32i32, .Cost: 16}, // interleave 2 x 32i32 into 64i32 (and store)
8096
8097 {.ISD: 2, .Type: MVT::v2i64, .Cost: 2}, // interleave 2 x 2i64 into 4i64 (and store)
8098 {.ISD: 2, .Type: MVT::v4i64, .Cost: 4}, // interleave 2 x 4i64 into 8i64 (and store)
8099 {.ISD: 2, .Type: MVT::v8i64, .Cost: 8}, // interleave 2 x 8i64 into 16i64 (and store)
8100 {.ISD: 2, .Type: MVT::v16i64, .Cost: 16}, // interleave 2 x 16i64 into 32i64 (and store)
8101 {.ISD: 2, .Type: MVT::v32i64, .Cost: 32}, // interleave 2 x 32i64 into 64i64 (and store)
8102
8103 {.ISD: 3, .Type: MVT::v2i8, .Cost: 4}, // interleave 3 x 2i8 into 6i8 (and store)
8104 {.ISD: 3, .Type: MVT::v4i8, .Cost: 4}, // interleave 3 x 4i8 into 12i8 (and store)
8105 {.ISD: 3, .Type: MVT::v8i8, .Cost: 6}, // interleave 3 x 8i8 into 24i8 (and store)
8106 {.ISD: 3, .Type: MVT::v16i8, .Cost: 11}, // interleave 3 x 16i8 into 48i8 (and store)
8107 {.ISD: 3, .Type: MVT::v32i8, .Cost: 13}, // interleave 3 x 32i8 into 96i8 (and store)
8108
8109 {.ISD: 3, .Type: MVT::v2i16, .Cost: 4}, // interleave 3 x 2i16 into 6i16 (and store)
8110 {.ISD: 3, .Type: MVT::v4i16, .Cost: 6}, // interleave 3 x 4i16 into 12i16 (and store)
8111 {.ISD: 3, .Type: MVT::v8i16, .Cost: 12}, // interleave 3 x 8i16 into 24i16 (and store)
8112 {.ISD: 3, .Type: MVT::v16i16, .Cost: 27}, // interleave 3 x 16i16 into 48i16 (and store)
8113 {.ISD: 3, .Type: MVT::v32i16, .Cost: 54}, // interleave 3 x 32i16 into 96i16 (and store)
8114
8115 {.ISD: 3, .Type: MVT::v2i32, .Cost: 4}, // interleave 3 x 2i32 into 6i32 (and store)
8116 {.ISD: 3, .Type: MVT::v4i32, .Cost: 5}, // interleave 3 x 4i32 into 12i32 (and store)
8117 {.ISD: 3, .Type: MVT::v8i32, .Cost: 11}, // interleave 3 x 8i32 into 24i32 (and store)
8118 {.ISD: 3, .Type: MVT::v16i32, .Cost: 22}, // interleave 3 x 16i32 into 48i32 (and store)
8119 {.ISD: 3, .Type: MVT::v32i32, .Cost: 48}, // interleave 3 x 32i32 into 96i32 (and store)
8120
8121 {.ISD: 3, .Type: MVT::v2i64, .Cost: 4}, // interleave 3 x 2i64 into 6i64 (and store)
8122 {.ISD: 3, .Type: MVT::v4i64, .Cost: 6}, // interleave 3 x 4i64 into 12i64 (and store)
8123 {.ISD: 3, .Type: MVT::v8i64, .Cost: 12}, // interleave 3 x 8i64 into 24i64 (and store)
8124 {.ISD: 3, .Type: MVT::v16i64, .Cost: 24}, // interleave 3 x 16i64 into 48i64 (and store)
8125
8126 {.ISD: 4, .Type: MVT::v2i8, .Cost: 4}, // interleave 4 x 2i8 into 8i8 (and store)
8127 {.ISD: 4, .Type: MVT::v4i8, .Cost: 4}, // interleave 4 x 4i8 into 16i8 (and store)
8128 {.ISD: 4, .Type: MVT::v8i8, .Cost: 4}, // interleave 4 x 8i8 into 32i8 (and store)
8129 {.ISD: 4, .Type: MVT::v16i8, .Cost: 8}, // interleave 4 x 16i8 into 64i8 (and store)
8130 {.ISD: 4, .Type: MVT::v32i8, .Cost: 12}, // interleave 4 x 32i8 into 128i8 (and store)
8131
8132 {.ISD: 4, .Type: MVT::v2i16, .Cost: 2}, // interleave 4 x 2i16 into 8i16 (and store)
8133 {.ISD: 4, .Type: MVT::v4i16, .Cost: 6}, // interleave 4 x 4i16 into 16i16 (and store)
8134 {.ISD: 4, .Type: MVT::v8i16, .Cost: 10}, // interleave 4 x 8i16 into 32i16 (and store)
8135 {.ISD: 4, .Type: MVT::v16i16, .Cost: 32}, // interleave 4 x 16i16 into 64i16 (and store)
8136 {.ISD: 4, .Type: MVT::v32i16, .Cost: 64}, // interleave 4 x 32i16 into 128i16 (and store)
8137
8138 {.ISD: 4, .Type: MVT::v2i32, .Cost: 5}, // interleave 4 x 2i32 into 8i32 (and store)
8139 {.ISD: 4, .Type: MVT::v4i32, .Cost: 6}, // interleave 4 x 4i32 into 16i32 (and store)
8140 {.ISD: 4, .Type: MVT::v8i32, .Cost: 16}, // interleave 4 x 8i32 into 32i32 (and store)
8141 {.ISD: 4, .Type: MVT::v16i32, .Cost: 32}, // interleave 4 x 16i32 into 64i32 (and store)
8142 {.ISD: 4, .Type: MVT::v32i32, .Cost: 64}, // interleave 4 x 32i32 into 128i32 (and store)
8143
8144 {.ISD: 4, .Type: MVT::v2i64, .Cost: 6}, // interleave 4 x 2i64 into 8i64 (and store)
8145 {.ISD: 4, .Type: MVT::v4i64, .Cost: 8}, // interleave 4 x 4i64 into 16i64 (and store)
8146 {.ISD: 4, .Type: MVT::v8i64, .Cost: 20}, // interleave 4 x 8i64 into 32i64 (and store)
8147 {.ISD: 4, .Type: MVT::v16i64, .Cost: 40}, // interleave 4 x 16i64 into 64i64 (and store)
8148
8149 {.ISD: 6, .Type: MVT::v2i8, .Cost: 7}, // interleave 6 x 2i8 into 12i8 (and store)
8150 {.ISD: 6, .Type: MVT::v4i8, .Cost: 9}, // interleave 6 x 4i8 into 24i8 (and store)
8151 {.ISD: 6, .Type: MVT::v8i8, .Cost: 16}, // interleave 6 x 8i8 into 48i8 (and store)
8152 {.ISD: 6, .Type: MVT::v16i8, .Cost: 27}, // interleave 6 x 16i8 into 96i8 (and store)
8153 {.ISD: 6, .Type: MVT::v32i8, .Cost: 90}, // interleave 6 x 32i8 into 192i8 (and store)
8154
8155 {.ISD: 6, .Type: MVT::v2i16, .Cost: 10}, // interleave 6 x 2i16 into 12i16 (and store)
8156 {.ISD: 6, .Type: MVT::v4i16, .Cost: 15}, // interleave 6 x 4i16 into 24i16 (and store)
8157 {.ISD: 6, .Type: MVT::v8i16, .Cost: 21}, // interleave 6 x 8i16 into 48i16 (and store)
8158 {.ISD: 6, .Type: MVT::v16i16, .Cost: 58}, // interleave 6 x 16i16 into 96i16 (and store)
8159 {.ISD: 6, .Type: MVT::v32i16, .Cost: 90}, // interleave 6 x 32i16 into 192i16 (and store)
8160
8161 {.ISD: 6, .Type: MVT::v2i32, .Cost: 9}, // interleave 6 x 2i32 into 12i32 (and store)
8162 {.ISD: 6, .Type: MVT::v4i32, .Cost: 12}, // interleave 6 x 4i32 into 24i32 (and store)
8163 {.ISD: 6, .Type: MVT::v8i32, .Cost: 33}, // interleave 6 x 8i32 into 48i32 (and store)
8164 {.ISD: 6, .Type: MVT::v16i32, .Cost: 66}, // interleave 6 x 16i32 into 96i32 (and store)
8165
8166 {.ISD: 6, .Type: MVT::v2i64, .Cost: 8}, // interleave 6 x 2i64 into 12i64 (and store)
8167 {.ISD: 6, .Type: MVT::v4i64, .Cost: 15}, // interleave 6 x 4i64 into 24i64 (and store)
8168 {.ISD: 6, .Type: MVT::v8i64, .Cost: 30}, // interleave 6 x 8i64 into 48i64 (and store)
8169 };
8170
8171 static const CostTblEntry SSE2InterleavedStoreTbl[] = {
8172 {.ISD: 2, .Type: MVT::v2i8, .Cost: 1}, // interleave 2 x 2i8 into 4i8 (and store)
8173 {.ISD: 2, .Type: MVT::v4i8, .Cost: 1}, // interleave 2 x 4i8 into 8i8 (and store)
8174 {.ISD: 2, .Type: MVT::v8i8, .Cost: 1}, // interleave 2 x 8i8 into 16i8 (and store)
8175
8176 {.ISD: 2, .Type: MVT::v2i16, .Cost: 1}, // interleave 2 x 2i16 into 4i16 (and store)
8177 {.ISD: 2, .Type: MVT::v4i16, .Cost: 1}, // interleave 2 x 4i16 into 8i16 (and store)
8178
8179 {.ISD: 2, .Type: MVT::v2i32, .Cost: 1}, // interleave 2 x 2i32 into 4i32 (and store)
8180 };
8181
8182 if (Opcode == Instruction::Load) {
8183 auto GetDiscountedCost = [Factor, NumMembers = Indices.size(),
8184 MemOpCosts](const CostTblEntry *Entry) {
8185 // NOTE: this is just an approximation!
8186 // It can over/under -estimate the cost!
8187 return MemOpCosts + divideCeil(Numerator: NumMembers * Entry->Cost, Denominator: Factor);
8188 };
8189
8190 if (ST->hasAVX2())
8191 if (const auto *Entry = CostTableLookup(Table: AVX2InterleavedLoadTbl, ISD: Factor,
8192 Ty: ETy.getSimpleVT()))
8193 return GetDiscountedCost(Entry);
8194
8195 if (ST->hasSSSE3())
8196 if (const auto *Entry = CostTableLookup(Table: SSSE3InterleavedLoadTbl, ISD: Factor,
8197 Ty: ETy.getSimpleVT()))
8198 return GetDiscountedCost(Entry);
8199
8200 if (ST->hasSSE2())
8201 if (const auto *Entry = CostTableLookup(Table: SSE2InterleavedLoadTbl, ISD: Factor,
8202 Ty: ETy.getSimpleVT()))
8203 return GetDiscountedCost(Entry);
8204 } else {
8205 assert(Opcode == Instruction::Store &&
8206 "Expected Store Instruction at this point");
8207 assert((!Indices.size() || Indices.size() == Factor) &&
8208 "Interleaved store only supports fully-interleaved groups.");
8209 if (ST->hasAVX2())
8210 if (const auto *Entry = CostTableLookup(Table: AVX2InterleavedStoreTbl, ISD: Factor,
8211 Ty: ETy.getSimpleVT()))
8212 return MemOpCosts + Entry->Cost;
8213
8214 if (ST->hasSSE2())
8215 if (const auto *Entry = CostTableLookup(Table: SSE2InterleavedStoreTbl, ISD: Factor,
8216 Ty: ETy.getSimpleVT()))
8217 return MemOpCosts + Entry->Cost;
8218 }
8219
8220 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
8221 Alignment, AddressSpace, CostKind,
8222 UseMaskForCond, UseMaskForGaps);
8223}
8224
8225InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
8226 StackOffset BaseOffset,
8227 bool HasBaseReg, int64_t Scale,
8228 unsigned AddrSpace) const {
8229 // Scaling factors are not free at all.
8230 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
8231 // will take 2 allocations in the out of order engine instead of 1
8232 // for plain addressing mode, i.e. inst (reg1).
8233 // E.g.,
8234 // vaddps (%rsi,%rdx), %ymm0, %ymm1
8235 // Requires two allocations (one for the load, one for the computation)
8236 // whereas:
8237 // vaddps (%rsi), %ymm0, %ymm1
8238 // Requires just 1 allocation, i.e., freeing allocations for other operations
8239 // and having less micro operations to execute.
8240 //
8241 // For some X86 architectures, this is even worse because for instance for
8242 // stores, the complex addressing mode forces the instruction to use the
8243 // "load" ports instead of the dedicated "store" port.
8244 // E.g., on Haswell:
8245 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
8246 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
8247 TargetLoweringBase::AddrMode AM;
8248 AM.BaseGV = BaseGV;
8249 AM.BaseOffs = BaseOffset.getFixed();
8250 AM.HasBaseReg = HasBaseReg;
8251 AM.Scale = Scale;
8252 AM.ScalableOffset = BaseOffset.getScalable();
8253 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace))
8254 // Scale represents reg2 * scale, thus account for 1
8255 // as soon as we use a second register.
8256 return AM.Scale != 0;
8257 return InstructionCost::getInvalid();
8258}
8259
8260InstructionCost X86TTIImpl::getBranchMispredictPenalty() const {
8261 // TODO: Hook MispredictPenalty of SchedMachineModel into this.
8262 return 14;
8263}
8264
8265bool X86TTIImpl::isVectorShiftByScalarCheap(Type *Ty) const {
8266 unsigned Bits = Ty->getScalarSizeInBits();
8267
8268 // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
8269 // Splitting for v32i8/v16i16 on XOP+AVX2 targets is still preferred.
8270 if (ST->hasXOP() && (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
8271 return false;
8272
8273 // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
8274 // shifts just as cheap as scalar ones.
8275 if (ST->hasAVX2() && (Bits == 32 || Bits == 64))
8276 return false;
8277
8278 // AVX512BW has shifts such as vpsllvw.
8279 if (ST->hasBWI() && Bits == 16)
8280 return false;
8281
8282 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
8283 // fully general vector.
8284 return true;
8285}
8286
8287unsigned X86TTIImpl::getStoreMinimumVF(unsigned VF, Type *ScalarMemTy,
8288 Type *ScalarValTy, Align Alignment,
8289 unsigned AddrSpace) const {
8290 if (ST->hasF16C() && ScalarMemTy->isHalfTy()) {
8291 return 4;
8292 }
8293 return BaseT::getStoreMinimumVF(VF, ScalarMemTy, ScalarValTy, Alignment,
8294 AddrSpace);
8295}
8296
8297bool X86TTIImpl::isProfitableToSinkOperands(Instruction *I,
8298 SmallVectorImpl<Use *> &Ops) const {
8299 using namespace llvm::PatternMatch;
8300
8301 if (I->getOpcode() == Instruction::And &&
8302 (ST->hasBMI() || (I->getType()->isVectorTy() && ST->hasSSE2()))) {
8303 for (auto &Op : I->operands()) {
8304 // (and X, (not Y)) -> (andn X, Y)
8305 if (match(V: Op.get(), P: m_Not(V: m_Value())) && !I->getType()->isIntegerTy(BitWidth: 8)) {
8306 Ops.push_back(Elt: &Op);
8307 return true;
8308 }
8309 // (and X, (splat (not Y))) -> (andn X, (splat Y))
8310 if (match(V: Op.get(),
8311 P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()), Idx: m_ZeroInt()),
8312 v2: m_Value(), mask: m_ZeroMask()))) {
8313 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
8314 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
8315 Ops.push_back(Elt: &Not);
8316 Ops.push_back(Elt: &InsertElt);
8317 Ops.push_back(Elt: &Op);
8318 return true;
8319 }
8320 }
8321 }
8322
8323 FixedVectorType *VTy = dyn_cast<FixedVectorType>(Val: I->getType());
8324 if (!VTy)
8325 return false;
8326
8327 if (I->getOpcode() == Instruction::Mul &&
8328 VTy->getElementType()->isIntegerTy(BitWidth: 64)) {
8329 for (auto &Op : I->operands()) {
8330 // Make sure we are not already sinking this operand
8331 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
8332 continue;
8333
8334 // Look for PMULDQ pattern where the input is a sext_inreg from vXi32 or
8335 // the PMULUDQ pattern where the input is a zext_inreg from vXi32.
8336 if (ST->hasSSE41() &&
8337 match(V: Op.get(), P: m_AShr(L: m_Shl(L: m_Value(), R: m_SpecificInt(V: 32)),
8338 R: m_SpecificInt(V: 32)))) {
8339 Ops.push_back(Elt: &cast<Instruction>(Val&: Op)->getOperandUse(i: 0));
8340 Ops.push_back(Elt: &Op);
8341 } else if (ST->hasSSE2() &&
8342 match(V: Op.get(),
8343 P: m_And(L: m_Value(), R: m_SpecificInt(UINT64_C(0xffffffff))))) {
8344 Ops.push_back(Elt: &Op);
8345 }
8346 }
8347
8348 return !Ops.empty();
8349 }
8350
8351 // A uniform shift amount in a vector shift or funnel shift may be much
8352 // cheaper than a generic variable vector shift, so make that pattern visible
8353 // to SDAG by sinking the shuffle instruction next to the shift.
8354 int ShiftAmountOpNum = -1;
8355 if (I->isShift())
8356 ShiftAmountOpNum = 1;
8357 else if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
8358 if (II->getIntrinsicID() == Intrinsic::fshl ||
8359 II->getIntrinsicID() == Intrinsic::fshr)
8360 ShiftAmountOpNum = 2;
8361 }
8362
8363 if (ShiftAmountOpNum == -1)
8364 return false;
8365
8366 auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: I->getOperand(i: ShiftAmountOpNum));
8367 if (Shuf && getSplatIndex(Mask: Shuf->getShuffleMask()) >= 0 &&
8368 isVectorShiftByScalarCheap(Ty: I->getType())) {
8369 Ops.push_back(Elt: &I->getOperandUse(i: ShiftAmountOpNum));
8370 return true;
8371 }
8372
8373 return false;
8374}
8375
8376bool X86TTIImpl::useFastCCForInternalCall(Function &F) const {
8377 bool HasEGPR = ST->hasEGPR();
8378 bool UseAPXFastCC = ST->useAPXFastCC();
8379 const TargetMachine &TM = getTLI()->getTargetMachine();
8380
8381 for (User *U : F.users()) {
8382 CallBase *CB = dyn_cast<CallBase>(Val: U);
8383 if (!CB || CB->getCalledOperand() != &F)
8384 continue;
8385 Function *CallerFunc = CB->getFunction();
8386 const X86Subtarget &CallerST = TM.getSubtarget<X86Subtarget>(F: *CallerFunc);
8387 if (CallerST.hasEGPR() != HasEGPR ||
8388 CallerST.useAPXFastCC() != UseAPXFastCC)
8389 return false;
8390 }
8391
8392 return true;
8393}
8394