1//===-- WebAssemblyTargetTransformInfo.cpp - WebAssembly-specific TTI -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines the WebAssembly-specific TargetTransformInfo
11/// implementation.
12///
13//===----------------------------------------------------------------------===//
14
15#include "WebAssemblyTargetTransformInfo.h"
16#include "llvm/IR/IntrinsicInst.h"
17#include "llvm/IR/IntrinsicsWebAssembly.h"
18#include "llvm/Transforms/InstCombine/InstCombiner.h"
19
20#include "llvm/CodeGen/CostTable.h"
21using namespace llvm;
22
23#define DEBUG_TYPE "wasmtti"
24
25TargetTransformInfo::PopcntSupportKind
26WebAssemblyTTIImpl::getPopcntSupport(unsigned TyWidth) const {
27 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
28 return TargetTransformInfo::PSK_FastHardware;
29}
30
31unsigned WebAssemblyTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
32 unsigned Result = BaseT::getNumberOfRegisters(ClassID);
33
34 // For SIMD, use at least 16 registers, as a rough guess.
35 bool Vector = (ClassID == 1);
36 if (Vector)
37 Result = std::max(a: Result, b: 16u);
38
39 return Result;
40}
41
42TypeSize WebAssemblyTTIImpl::getRegisterBitWidth(
43 TargetTransformInfo::RegisterKind K) const {
44 switch (K) {
45 case TargetTransformInfo::RGK_Scalar:
46 return TypeSize::getFixed(ExactSize: 64);
47 case TargetTransformInfo::RGK_FixedWidthVector:
48 return TypeSize::getFixed(ExactSize: getST()->hasSIMD128() ? 128 : 64);
49 case TargetTransformInfo::RGK_ScalableVector:
50 return TypeSize::getScalable(MinimumSize: 0);
51 }
52
53 llvm_unreachable("Unsupported register kind");
54}
55
56InstructionCost WebAssemblyTTIImpl::getArithmeticInstrCost(
57 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
58 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
59 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
60
61 if (ST->hasSIMD128()) {
62 static const CostTblEntry ArithCostTbl[]{
63 // extmul + (maybe awkward) shuffle
64 {.ISD: ISD::MUL, .Type: MVT::v8i8, .Cost: 4},
65 // 2x extmul + (okay) shuffle
66 {.ISD: ISD::MUL, .Type: MVT::v16i8, .Cost: 4},
67 // extmul
68 {.ISD: ISD::MUL, .Type: MVT::v4i16, .Cost: 1},
69 // extmul
70 {.ISD: ISD::MUL, .Type: MVT::v2i32, .Cost: 1},
71 };
72 EVT DstVT = TLI->getValueType(DL, Ty);
73 if (DstVT.isSimple()) {
74 int ISD = TLI->InstructionOpcodeToISD(Opcode);
75 if (const auto *Entry =
76 CostTableLookup(Table: ArithCostTbl, ISD, Ty: DstVT.getSimpleVT()))
77 return Entry->Cost;
78 }
79 }
80
81 InstructionCost Cost =
82 BasicTTIImplBase<WebAssemblyTTIImpl>::getArithmeticInstrCost(
83 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
84
85 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
86 switch (Opcode) {
87 case Instruction::LShr:
88 case Instruction::AShr:
89 case Instruction::Shl:
90 // SIMD128's shifts currently only accept a scalar shift count. For each
91 // element, we'll need to extract, op, insert. The following is a rough
92 // approximation.
93 if (!Op2Info.isUniform())
94 Cost =
95 VTy->getNumElements() *
96 (TargetTransformInfo::TCC_Basic +
97 getArithmeticInstrCost(Opcode, Ty: VTy->getElementType(), CostKind) +
98 TargetTransformInfo::TCC_Basic);
99 break;
100 }
101 }
102 return Cost;
103}
104
105InstructionCost WebAssemblyTTIImpl::getCastInstrCost(
106 unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH,
107 TTI::TargetCostKind CostKind, const Instruction *I) const {
108 int ISD = TLI->InstructionOpcodeToISD(Opcode);
109 auto SrcTy = TLI->getValueType(DL, Ty: Src);
110 auto DstTy = TLI->getValueType(DL, Ty: Dst);
111
112 if (!SrcTy.isSimple() || !DstTy.isSimple()) {
113 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
114 }
115
116 if (!ST->hasSIMD128()) {
117 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
118 }
119
120 auto DstVT = DstTy.getSimpleVT();
121 auto SrcVT = SrcTy.getSimpleVT();
122
123 if (I && I->hasOneUser()) {
124 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
125 int UserISD = TLI->InstructionOpcodeToISD(Opcode: SingleUser->getOpcode());
126
127 // extmul_low support
128 if (UserISD == ISD::MUL &&
129 (ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND)) {
130 // Free low extensions.
131 if ((SrcVT == MVT::v8i8 && DstVT == MVT::v8i16) ||
132 (SrcVT == MVT::v4i16 && DstVT == MVT::v4i32) ||
133 (SrcVT == MVT::v2i32 && DstVT == MVT::v2i64)) {
134 return 0;
135 }
136 // Will require an additional extlow operation for the intermediate
137 // i16/i32 value.
138 if ((SrcVT == MVT::v4i8 && DstVT == MVT::v4i32) ||
139 (SrcVT == MVT::v2i16 && DstVT == MVT::v2i64)) {
140 return 1;
141 }
142 }
143 }
144
145 static constexpr TypeConversionCostTblEntry ConversionTbl[] = {
146 // extend_low
147 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1},
148 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1},
149 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1},
150 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1},
151 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1},
152 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1},
153 // 2 x extend_low
154 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2},
155 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2},
156 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2},
157 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2},
158 // extend_low, extend_high
159 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
160 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
161 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
162 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
163 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
164 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
165 // 2x extend_low, extend_high
166 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: 4},
167 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i32, .Cost: 4},
168 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: 4},
169 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i16, .Cost: 4},
170 // 6x extend_low, extend_high
171 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
172 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
173 // shuffle
174 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i32, .Cost: 2},
175 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i32, .Cost: 4},
176 // narrow, and
177 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 2},
178 // 2x narrow, and
179 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i32, .Cost: 3},
180 // narrow, and
181 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i16, .Cost: 2},
182 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i16, .Cost: 2},
183 // narrow, 2x and
184 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: 3},
185 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: 3},
186 // 2x narrow, 2x and
187 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i64, .Cost: 4},
188 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 4},
189 // 3x narrow, 2x and
190 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i64, .Cost: 5},
191 // 2x narrow, 4x and
192 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: 6},
193 // 3x narrow, 4x and
194 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i64, .Cost: 7},
195 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 7},
196 // 4x narrow, 4x and
197 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i64, .Cost: 8},
198 // 6x narrow, 8x and
199 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i64, .Cost: 14},
200 // 7x narrow, 8x and
201 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i64, .Cost: 15},
202 // convert_i32x4
203 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
204 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
205 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
206 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
207 // extend_low, convert
208 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2},
209 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2},
210 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
211 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
212 // extend_low x 2, convert
213 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
214 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
215 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3},
216 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3},
217 // several shuffles
218 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
219 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
220 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 10},
221 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 10},
222 // trunc_sat, const, and, 2x narrow
223 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 5},
224 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 5},
225 // trunc_sat, const, and, 2x narrow
226 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 5},
227 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 5},
228 // trunc_sat, const, and, narrow
229 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 4},
230 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 4},
231 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 4},
232 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 4},
233 // 2x trunc_sat, const, 2x and, 2x narrow
234 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i8, .Src: MVT::v8f32, .Cost: 7},
235 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i8, .Src: MVT::v8f32, .Cost: 7},
236 // 2x trunc_sat, const, 2x and, narrow
237 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 6},
238 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 6},
239 };
240
241 if (const auto *Entry =
242 ConvertCostTableLookup(Table: ConversionTbl, ISD, Dst: DstVT, Src: SrcVT)) {
243 return Entry->Cost;
244 }
245
246 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
247}
248
249WebAssemblyTTIImpl::TTI::MemCmpExpansionOptions
250WebAssemblyTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
251 TTI::MemCmpExpansionOptions Options;
252
253 Options.AllowOverlappingLoads = true;
254
255 if (ST->hasSIMD128())
256 Options.LoadSizes.push_back(Elt: 16);
257
258 Options.LoadSizes.append(IL: {8, 4, 2, 1});
259 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
260 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
261
262 return Options;
263}
264
265InstructionCost WebAssemblyTTIImpl::getMemoryOpCost(
266 unsigned Opcode, Type *Ty, Align Alignment, unsigned AddressSpace,
267 TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo,
268 const Instruction *I) const {
269 // FIXME: Load latency isn't handled here
270 if (!ST->hasSIMD128() || !isa<FixedVectorType>(Val: Ty) ||
271 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)) {
272 return BaseT::getMemoryOpCost(Opcode, Src: Ty, Alignment, AddressSpace,
273 CostKind);
274 }
275
276 EVT VT = TLI->getValueType(DL, Ty, AllowUnknown: true);
277 // Type legalization can't handle structs
278 if (VT == MVT::Other)
279 return BaseT::getMemoryOpCost(Opcode, Src: Ty, Alignment, AddressSpace,
280 CostKind);
281
282 auto LT = getTypeLegalizationCost(Ty);
283 if (!LT.first.isValid())
284 return InstructionCost::getInvalid();
285
286 int ISD = TLI->InstructionOpcodeToISD(Opcode);
287 unsigned width = VT.getSizeInBits();
288 if (ISD == ISD::LOAD) {
289 // 128-bit loads are a single instruction. 32-bit and 64-bit vector loads
290 // can be lowered to load32_zero and load64_zero respectively. Assume SIMD
291 // loads are twice as expensive as scalar.
292 switch (width) {
293 default:
294 break;
295 case 32:
296 case 64:
297 case 128:
298 return 2;
299 }
300 } else if (ISD == ISD::STORE) {
301 // For stores, we can use store lane operations.
302 switch (width) {
303 default:
304 break;
305 case 8:
306 case 16:
307 case 32:
308 case 64:
309 case 128:
310 return 2;
311 }
312 }
313
314 return BaseT::getMemoryOpCost(Opcode, Src: Ty, Alignment, AddressSpace, CostKind);
315}
316
317InstructionCost WebAssemblyTTIImpl::getShuffleCost(
318 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
319 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
320 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
321 TTI::VectorInstrContext VIC) const {
322 // Canonicalize the ShuffleKind in case optimizations didn't.
323 // Otherwise, we might end up with the wrong ShuffleKind to match against.
324
325 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
326
327 // Wasm SIMD128 has native splat instructions for all lane types.
328 if (ST->hasSIMD128() && Kind == TTI::SK_Broadcast &&
329 isa<FixedVectorType>(Val: SrcTy))
330 return 1;
331
332 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp,
333 Args, CtxI);
334}
335
336InstructionCost WebAssemblyTTIImpl::getInterleavedMemoryOpCost(
337 unsigned Opcode, Type *Ty, unsigned Factor, ArrayRef<unsigned> Indices,
338 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
339 bool UseMaskForCond, bool UseMaskForGaps) const {
340 assert(Factor >= 2 && "Invalid interleave factor");
341
342 auto *VecTy = cast<VectorType>(Val: Ty);
343 if (!ST->hasSIMD128() || !isa<FixedVectorType>(Val: VecTy)) {
344 return InstructionCost::getInvalid();
345 }
346
347 if (UseMaskForCond || UseMaskForGaps)
348 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy: Ty, Factor, Indices,
349 Alignment, AddressSpace, CostKind,
350 UseMaskForCond, UseMaskForGaps);
351
352 constexpr unsigned MaxInterleaveFactor = 4;
353 if (Factor <= MaxInterleaveFactor) {
354 unsigned MinElts = VecTy->getElementCount().getKnownMinValue();
355 // Ensure the number of vector elements is greater than 1.
356 if (MinElts < 2 || MinElts % Factor != 0)
357 return InstructionCost::getInvalid();
358
359 unsigned ElSize = DL.getTypeSizeInBits(Ty: VecTy->getElementType());
360 // Ensure the element type is legal.
361 if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
362 return InstructionCost::getInvalid();
363
364 if (Factor != 2 && Factor != 4)
365 return InstructionCost::getInvalid();
366
367 auto *SubVecTy =
368 VectorType::get(ElementType: VecTy->getElementType(),
369 EC: VecTy->getElementCount().divideCoefficientBy(RHS: Factor));
370 InstructionCost MemCost =
371 getMemoryOpCost(Opcode, Ty: SubVecTy, Alignment, AddressSpace, CostKind);
372
373 unsigned VecSize = DL.getTypeSizeInBits(Ty: SubVecTy);
374 unsigned MaxVecSize = 128;
375 unsigned NumAccesses =
376 std::max<unsigned>(a: 1, b: (MinElts * ElSize + MaxVecSize - 1) / VecSize);
377
378 // A stride of two is commonly supported via dedicated instructions, so it
379 // should be relatively cheap for all element sizes. A stride of four is
380 // more expensive as it will likely require more shuffles. Using two
381 // simd128 inputs is considered more expensive and we mainly account for
382 // shuffling two inputs (32 bytes), but we do model 4 x v4i32 to enable
383 // arithmetic kernels with smaller (i8/i16) inputs.
384 static const CostTblEntry ShuffleCostTbl[] = {
385 // One reg.
386 {.ISD: 2, .Type: MVT::v2i8, .Cost: 1}, // interleave 2 x 2i8 into 4i8
387 {.ISD: 2, .Type: MVT::v4i8, .Cost: 1}, // interleave 2 x 4i8 into 8i8
388 {.ISD: 2, .Type: MVT::v8i8, .Cost: 1}, // interleave 2 x 8i8 into 16i8
389 {.ISD: 2, .Type: MVT::v2i16, .Cost: 1}, // interleave 2 x 2i16 into 4i16
390 {.ISD: 2, .Type: MVT::v4i16, .Cost: 1}, // interleave 2 x 4i16 into 8i16
391 {.ISD: 2, .Type: MVT::v2i32, .Cost: 1}, // interleave 2 x 2i32 into 4i32
392 {.ISD: 2, .Type: MVT::v2f32, .Cost: 1}, // interleave 2 x 2f32 into 4f32
393
394 // Two regs.
395 {.ISD: 2, .Type: MVT::v16i8, .Cost: 2}, // interleave 2 x 16i8 into 32i8
396 {.ISD: 2, .Type: MVT::v8i16, .Cost: 2}, // interleave 2 x 8i16 into 16i16
397 {.ISD: 2, .Type: MVT::v4i32, .Cost: 2}, // interleave 2 x 4i32 into 8i32
398 {.ISD: 2, .Type: MVT::v4f32, .Cost: 2}, // interleave 2 x 4f32 into 8f32
399
400 // One reg.
401 {.ISD: 4, .Type: MVT::v2i8, .Cost: 4}, // interleave 4 x 2i8 into 8i8
402 {.ISD: 4, .Type: MVT::v4i8, .Cost: 4}, // interleave 4 x 4i8 into 16i8
403 {.ISD: 4, .Type: MVT::v2i16, .Cost: 4}, // interleave 4 x 2i16 into 8i16
404
405 // Two regs.
406 {.ISD: 4, .Type: MVT::v8i8, .Cost: 16}, // interleave 4 x 8i8 into 32i8
407 {.ISD: 4, .Type: MVT::v4i16, .Cost: 8}, // interleave 4 x 4i16 into 16i16
408 {.ISD: 4, .Type: MVT::v2i32, .Cost: 4}, // interleave 4 x 2i32 into 8i32
409 {.ISD: 4, .Type: MVT::v2f32, .Cost: 4}, // interleave 4 x 2f32 into 8f32
410
411 // Four regs.
412 {.ISD: 4, .Type: MVT::v4i32, .Cost: 16}, // interleave 4 x 4i32 into 16i32
413 };
414
415 EVT ETy = TLI->getValueType(DL, Ty: SubVecTy);
416 if (const auto *Entry =
417 CostTableLookup(Table: ShuffleCostTbl, ISD: Factor, Ty: ETy.getSimpleVT()))
418 return Entry->Cost + (NumAccesses * MemCost);
419 }
420
421 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
422 Alignment, AddressSpace, CostKind,
423 UseMaskForCond, UseMaskForGaps);
424}
425
426InstructionCost WebAssemblyTTIImpl::getVectorInstrCost(
427 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
428 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
429 InstructionCost Cost = BasicTTIImplBase::getVectorInstrCost(
430 Opcode, Val, CostKind, Index, Op0, Op1, VIC);
431
432 // SIMD128's insert/extract currently only take constant indices.
433 if (Index == -1u)
434 return Cost + 25 * TargetTransformInfo::TCC_Expensive;
435
436 return Cost;
437}
438
439InstructionCost WebAssemblyTTIImpl::getPartialReductionCost(
440 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
441 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
442 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
443 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
444 InstructionCost Invalid = InstructionCost::getInvalid();
445 if (!VF.isFixed() || !ST->hasSIMD128())
446 return Invalid;
447
448 if (CostKind != TTI::TCK_RecipThroughput)
449 return Invalid;
450
451 if (Opcode != Instruction::Add)
452 return Invalid;
453
454 EVT AccumEVT = EVT::getEVT(Ty: AccumType);
455 // TODO: Add i64 accumulator.
456 if (AccumEVT != MVT::i32)
457 return Invalid;
458
459 // Possible options:
460 // - i16x8.extadd_pairwise_i8x16_sx
461 // - i32x4.extadd_pairwise_i16x8_sx
462 // - i32x4.dot_i16x8_s
463 // Only try to support dot, for now.
464
465 EVT InputEVT = EVT::getEVT(Ty: InputTypeA);
466 if (!((InputEVT == MVT::i16 && VF.getFixedValue() == 8) ||
467 (InputEVT == MVT::i8 && VF.getFixedValue() == 16))) {
468 return Invalid;
469 }
470
471 if (OpAExtend == TTI::PR_None)
472 return Invalid;
473
474 InstructionCost Cost(TTI::TCC_Basic);
475 if (!BinOp)
476 return Cost;
477
478 if (OpAExtend != OpBExtend)
479 return Invalid;
480
481 if (*BinOp != Instruction::Mul)
482 return Invalid;
483
484 if (InputTypeA != InputTypeB)
485 return Invalid;
486
487 // Signed inputs can lower to dot
488 if (InputEVT == MVT::i16 && VF.getFixedValue() == 8)
489 return OpAExtend == TTI::PR_SignExtend ? Cost : Cost * 2;
490
491 // Double the size of the lowered sequence.
492 if (InputEVT == MVT::i8 && VF.getFixedValue() == 16)
493 return OpAExtend == TTI::PR_SignExtend ? Cost * 2 : Cost * 4;
494
495 return Invalid;
496}
497
498TTI::ReductionShuffle WebAssemblyTTIImpl::getPreferredExpandedReductionShuffle(
499 const IntrinsicInst *II) const {
500
501 switch (II->getIntrinsicID()) {
502 default:
503 break;
504 case Intrinsic::vector_reduce_fadd:
505 return TTI::ReductionShuffle::Pairwise;
506 }
507 return TTI::ReductionShuffle::SplitHalf;
508}
509
510void WebAssemblyTTIImpl::getUnrollingPreferences(
511 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
512 OptimizationRemarkEmitter *ORE) const {
513 // Scan the loop: don't unroll loops with calls. This is a standard approach
514 // for most (all?) targets.
515 for (BasicBlock *BB : L->blocks())
516 for (Instruction &I : *BB)
517 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I))
518 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction())
519 if (isLoweredToCall(F))
520 return;
521
522 // The chosen threshold is within the range of 'LoopMicroOpBufferSize' of
523 // the various microarchitectures that use the BasicTTI implementation and
524 // has been selected through heuristics across multiple cores and runtimes.
525 UP.Partial = UP.Runtime = UP.UpperBound = true;
526 UP.PartialThreshold = 30;
527
528 // Avoid unrolling when optimizing for size.
529 UP.OptSizeThreshold = 0;
530 UP.PartialOptSizeThreshold = 0;
531
532 // Set number of instructions optimized when "back edge"
533 // becomes "fall through" to default value of 2.
534 UP.BEInsns = 2;
535}
536
537bool WebAssemblyTTIImpl::supportsTailCalls() const {
538 return getST()->hasTailCall();
539}
540
541bool WebAssemblyTTIImpl::isProfitableToSinkOperands(
542 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
543 using namespace llvm::PatternMatch;
544
545 if (!I->getType()->isVectorTy() || !I->isShift())
546 return false;
547
548 Value *V = I->getOperand(i: 1);
549 // We dont need to sink constant splat.
550 if (isa<Constant>(Val: V))
551 return false;
552
553 if (match(V, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
554 v2: m_Value(), mask: m_ZeroMask()))) {
555 // Sink insert
556 Ops.push_back(Elt: &cast<Instruction>(Val: V)->getOperandUse(i: 0));
557 // Sink shuffle
558 Ops.push_back(Elt: &I->getOperandUse(i: 1));
559 return true;
560 }
561
562 return false;
563}
564
565/// Attempt to convert [relaxed_]swizzle to shufflevector if the mask is
566/// constant.
567static Value *simplifyWasmSwizzle(const IntrinsicInst &II,
568 InstCombiner::BuilderTy &Builder,
569 bool IsRelaxed) {
570 auto *V = dyn_cast<Constant>(Val: II.getArgOperand(i: 1));
571 if (!V)
572 return nullptr;
573
574 auto *VecTy = cast<FixedVectorType>(Val: II.getType());
575 unsigned NumElts = VecTy->getNumElements();
576 assert(NumElts == 16);
577
578 // Construct a shuffle mask from constant integers or UNDEFs.
579 int Indexes[16];
580 bool AnyOutOfBounds = false;
581
582 for (unsigned I = 0; I < NumElts; ++I) {
583 Constant *COp = V->getAggregateElement(Elt: I);
584 if (!COp || (!isa<UndefValue>(Val: COp) && !isa<ConstantInt>(Val: COp)))
585 return nullptr;
586
587 if (isa<UndefValue>(Val: COp)) {
588 Indexes[I] = -1;
589 continue;
590 }
591
592 if (IsRelaxed && cast<ConstantInt>(Val: COp)->getSExtValue() >= NumElts) {
593 // The relaxed_swizzle operation always returns 0 if the lane index is
594 // less than 0 when interpreted as a signed value. For lane indices above
595 // 15, however, it can choose between returning 0 or the lane at `Index %
596 // 16`. However, the choice must be made consistently. As the WebAssembly
597 // spec states:
598 //
599 // "The result of relaxed operators are implementation-dependent, because
600 // the set of possible results may depend on properties of the host
601 // environment, such as its hardware. Technically, their behaviour is
602 // controlled by a set of global parameters to the semantics that an
603 // implementation can instantiate in different ways. These choices are
604 // fixed, that is, parameters are constant during the execution of any
605 // given program."
606 //
607 // The WebAssembly runtime may choose differently from us, so we can't
608 // optimize a relaxed swizzle with lane indices above 15.
609 return nullptr;
610 }
611
612 uint64_t Index = cast<ConstantInt>(Val: COp)->getZExtValue();
613 if (Index >= NumElts) {
614 AnyOutOfBounds = true;
615 // If there are out-of-bounds indices, the swizzle instruction returns
616 // zeroes in those lanes. We'll provide an all-zeroes vector as the
617 // second argument to shufflevector and read the first element from it.
618 Indexes[I] = NumElts;
619 continue;
620 }
621
622 Indexes[I] = Index;
623 }
624
625 auto *V1 = II.getArgOperand(i: 0);
626 auto *V2 =
627 AnyOutOfBounds ? Constant::getNullValue(Ty: VecTy) : PoisonValue::get(T: VecTy);
628
629 return Builder.CreateShuffleVector(V1, V2, Mask: ArrayRef(Indexes, NumElts));
630}
631
632std::optional<Instruction *>
633WebAssemblyTTIImpl::instCombineIntrinsic(InstCombiner &IC,
634 IntrinsicInst &II) const {
635 Intrinsic::ID IID = II.getIntrinsicID();
636 switch (IID) {
637 case Intrinsic::wasm_swizzle:
638 case Intrinsic::wasm_relaxed_swizzle:
639 if (Value *V = simplifyWasmSwizzle(
640 II, Builder&: IC.Builder, IsRelaxed: IID == Intrinsic::wasm_relaxed_swizzle)) {
641 return IC.replaceInstUsesWith(I&: II, V);
642 }
643 break;
644 }
645
646 return std::nullopt;
647}
648