1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "WebAssemblyMachineFunctionInfo.h"
18#include "WebAssemblySubtarget.h"
19#include "WebAssemblyTargetMachine.h"
20#include "WebAssemblyUtilities.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineFrameInfo.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/CodeGen/MachineModuleInfo.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/SDPatternMatch.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/DiagnosticPrinter.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/IntrinsicInst.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsWebAssembly.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/MathExtras.h"
39#include "llvm/Target/TargetOptions.h"
40using namespace llvm;
41
42#define DEBUG_TYPE "wasm-lower"
43
44WebAssemblyTargetLowering::WebAssemblyTargetLowering(
45 const TargetMachine &TM, const WebAssemblySubtarget &STI)
46 : TargetLowering(TM, STI), Subtarget(&STI) {
47 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
48
49 // Set the load count for memcmp expand optimization
50 MaxLoadsPerMemcmp = 8;
51 MaxLoadsPerMemcmpOptSize = 4;
52
53 // Booleans always contain 0 or 1.
54 setBooleanContents(ZeroOrOneBooleanContent);
55 // Except in SIMD vectors
56 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
57 // We don't know the microarchitecture here, so just reduce register pressure.
58 setSchedulingPreference(Sched::RegPressure);
59 // Tell ISel that we have a stack pointer.
60 setStackPointerRegisterToSaveRestore(
61 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
62 // Set up the register classes.
63 addRegisterClass(VT: MVT::i32, RC: &WebAssembly::I32RegClass);
64 addRegisterClass(VT: MVT::i64, RC: &WebAssembly::I64RegClass);
65 addRegisterClass(VT: MVT::f32, RC: &WebAssembly::F32RegClass);
66 addRegisterClass(VT: MVT::f64, RC: &WebAssembly::F64RegClass);
67 if (Subtarget->hasSIMD128()) {
68 addRegisterClass(VT: MVT::v16i8, RC: &WebAssembly::V128RegClass);
69 addRegisterClass(VT: MVT::v8i16, RC: &WebAssembly::V128RegClass);
70 addRegisterClass(VT: MVT::v4i32, RC: &WebAssembly::V128RegClass);
71 addRegisterClass(VT: MVT::v4f32, RC: &WebAssembly::V128RegClass);
72 addRegisterClass(VT: MVT::v2i64, RC: &WebAssembly::V128RegClass);
73 addRegisterClass(VT: MVT::v2f64, RC: &WebAssembly::V128RegClass);
74 }
75 if (Subtarget->hasFP16()) {
76 addRegisterClass(VT: MVT::v8f16, RC: &WebAssembly::V128RegClass);
77 }
78 if (Subtarget->hasReferenceTypes()) {
79 addRegisterClass(VT: MVT::externref, RC: &WebAssembly::EXTERNREFRegClass);
80 addRegisterClass(VT: MVT::funcref, RC: &WebAssembly::FUNCREFRegClass);
81 if (Subtarget->hasExceptionHandling()) {
82 addRegisterClass(VT: MVT::exnref, RC: &WebAssembly::EXNREFRegClass);
83 }
84 }
85 // Compute derived properties from the register classes.
86 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
87
88 // Transform loads and stores to pointers in address space 1 to loads and
89 // stores to WebAssembly global variables, outside linear memory.
90 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
91 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
92 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
93 }
94 if (Subtarget->hasSIMD128()) {
95 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
96 MVT::v2f64}) {
97 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
98 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
99 }
100 }
101 if (Subtarget->hasFP16()) {
102 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Custom);
103 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Custom);
104 }
105 if (Subtarget->hasReferenceTypes()) {
106 // We need custom load and store lowering for both externref, funcref and
107 // Other. The MVT::Other here represents tables of reference types.
108 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
109 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
110 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
111 }
112 }
113
114 setOperationAction(Op: ISD::GlobalAddress, VT: MVTPtr, Action: Custom);
115 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVTPtr, Action: Custom);
116 setOperationAction(Op: ISD::ExternalSymbol, VT: MVTPtr, Action: Custom);
117 setOperationAction(Op: ISD::JumpTable, VT: MVTPtr, Action: Custom);
118 setOperationAction(Op: ISD::BlockAddress, VT: MVTPtr, Action: Custom);
119 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Custom);
120 setOperationAction(Op: ISD::CLEAR_CACHE, VT: MVT::Other, Action: Custom);
121
122 // Take the default expansion for va_arg, va_copy, and va_end. There is no
123 // default action for va_start, so we do that custom.
124 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
125 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
126 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
127 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
128
129 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {
130 if (!Subtarget->hasFP16() && T == MVT::v8f16) {
131 continue;
132 }
133 // Don't expand the floating-point types to constant pools.
134 setOperationAction(Op: ISD::ConstantFP, VT: T, Action: Legal);
135 // Expand floating-point comparisons.
136 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
137 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
138 setCondCodeAction(CCs: CC, VT: T, Action: Expand);
139 // Expand floating-point library function operators.
140 for (auto Op : {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FMA})
141 setOperationAction(Op, VT: T, Action: Expand);
142 // Expand vector FREM, but use a libcall rather than an expansion for scalar
143 if (MVT(T).isVector())
144 setOperationAction(Op: ISD::FREM, VT: T, Action: Expand);
145 else
146 setOperationAction(Op: ISD::FREM, VT: T, Action: LibCall);
147 // Note supported floating-point library function operators that otherwise
148 // default to expand.
149 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
150 ISD::FRINT, ISD::FROUNDEVEN})
151 setOperationAction(Op, VT: T, Action: Legal);
152 // Support minimum and maximum, which otherwise default to expand.
153 setOperationAction(Op: ISD::FMINIMUM, VT: T, Action: Legal);
154 setOperationAction(Op: ISD::FMAXIMUM, VT: T, Action: Legal);
155 // When experimental v8f16 support is enabled these instructions don't need
156 // to be expanded.
157 if (T != MVT::v8f16) {
158 setOperationAction(Op: ISD::FP16_TO_FP, VT: T, Action: Expand);
159 setOperationAction(Op: ISD::FP_TO_FP16, VT: T, Action: Expand);
160 }
161 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: T, MemVT: MVT::f16, Action: Expand);
162 setTruncStoreAction(ValVT: T, MemVT: MVT::f16, Action: Expand);
163 }
164
165 // Expand unavailable integer operations.
166 for (auto Op :
167 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
168 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
169 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
170 for (auto T : {MVT::i32, MVT::i64})
171 setOperationAction(Op, VT: T, Action: Expand);
172 if (Subtarget->hasSIMD128())
173 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
174 setOperationAction(Op, VT: T, Action: Expand);
175 }
176
177 if (Subtarget->hasWideArithmetic()) {
178 setOperationAction(Op: ISD::ADD, VT: MVT::i128, Action: Custom);
179 setOperationAction(Op: ISD::SUB, VT: MVT::i128, Action: Custom);
180 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Custom);
181 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Custom);
182 setOperationAction(Op: ISD::UADDO, VT: MVT::i64, Action: Custom);
183 }
184
185 if (Subtarget->hasNontrappingFPToInt())
186 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
187 for (auto T : {MVT::i32, MVT::i64})
188 setOperationAction(Op, VT: T, Action: Custom);
189
190 if (Subtarget->hasRelaxedSIMD()) {
191 setOperationAction(
192 Ops: {ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM},
193 VTs: {MVT::v4f32, MVT::v2f64}, Action: Custom);
194 }
195 // SIMD-specific configuration
196 if (Subtarget->hasSIMD128()) {
197
198 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
199
200 // Combine wide-vector muls, with extend inputs, to extmul_half.
201 setTargetDAGCombine(ISD::MUL);
202 setTargetDAGCombine(ISD::SHL);
203
204 // Combine vector mask reductions into alltrue/anytrue
205 setTargetDAGCombine(ISD::SETCC);
206
207 // Convert vector to integer bitcasts to bitmask
208 setTargetDAGCombine(ISD::BITCAST);
209
210 // Hoist bitcasts out of shuffles
211 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
212
213 // Combine extends of extract_subvectors into widening ops
214 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND});
215
216 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
217 // conversions ops
218 setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND,
219 ISD::EXTRACT_SUBVECTOR});
220
221 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
222 // into conversion ops
223 setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
224 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_ROUND,
225 ISD::CONCAT_VECTORS});
226
227 setTargetDAGCombine(ISD::TRUNCATE);
228
229 // Support saturating add/sub for i8x16 and i16x8
230 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
231 for (auto T : {MVT::v16i8, MVT::v8i16})
232 setOperationAction(Op, VT: T, Action: Legal);
233
234 // Support integer abs
235 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
236 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
237
238 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
239 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
240 MVT::v2f64})
241 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
242
243 if (Subtarget->hasFP16())
244 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
245
246 // We have custom shuffle lowering to expose the shuffle mask
247 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
248 MVT::v2f64})
249 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: T, Action: Custom);
250
251 if (Subtarget->hasFP16())
252 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8f16, Action: Custom);
253
254 // Support splatting
255 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
256 MVT::v2f64})
257 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
258
259 setOperationAction(Ops: ISD::AVGCEILU, VTs: {MVT::v8i16, MVT::v16i8}, Action: Legal);
260
261 // Custom lowering since wasm shifts must have a scalar shift amount
262 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
263 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
264 setOperationAction(Op, VT: T, Action: Custom);
265
266 // Custom lower lane accesses to expand out variable indices
267 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
268 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
269 MVT::v2f64})
270 setOperationAction(Op, VT: T, Action: Custom);
271
272 // There is no i8x16.mul instruction
273 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Expand);
274
275 // Expand integer operations supported for scalars but not SIMD
276 for (auto Op :
277 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
278 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
279 setOperationAction(Op, VT: T, Action: Expand);
280
281 // But we do have integer min and max operations
282 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
283 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
284 setOperationAction(Op, VT: T, Action: Legal);
285
286 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
287 setOperationAction(Op: ISD::CTPOP, VT: MVT::v16i8, Action: Legal);
288 setOperationAction(Op: ISD::CTLZ, VT: MVT::v16i8, Action: Expand);
289 setOperationAction(Op: ISD::CTTZ, VT: MVT::v16i8, Action: Expand);
290
291 // Custom lower bit counting operations for other types to scalarize them.
292 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
293 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
294 setOperationAction(Op, VT: T, Action: Custom);
295
296 // Expand float operations supported for scalars but not SIMD
297 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
298 ISD::FEXP, ISD::FEXP2, ISD::FEXP10})
299 for (auto T : {MVT::v4f32, MVT::v2f64})
300 setOperationAction(Op, VT: T, Action: Expand);
301
302 // Unsigned comparison operations are unavailable for i64x2 vectors.
303 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
304 setCondCodeAction(CCs: CC, VT: MVT::v2i64, Action: Custom);
305
306 // 64x2 conversions are not in the spec
307 for (auto Op :
308 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
309 for (auto T : {MVT::v2i64, MVT::v2f64})
310 setOperationAction(Op, VT: T, Action: Expand);
311
312 // But saturating fp_to_int converstions are
313 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) {
314 setOperationAction(Op, VT: MVT::v4i32, Action: Custom);
315 if (Subtarget->hasFP16()) {
316 setOperationAction(Op, VT: MVT::v8i16, Action: Custom);
317 }
318 }
319
320 // Support vector extending
321 for (auto T : MVT::integer_fixedlen_vector_valuetypes()) {
322 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
323 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
324 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
325 }
326
327 if (Subtarget->hasFP16()) {
328 setOperationAction(Op: ISD::FMA, VT: MVT::v8f16, Action: Legal);
329 }
330
331 if (Subtarget->hasRelaxedSIMD()) {
332 setOperationAction(Op: ISD::FMULADD, VT: MVT::v4f32, Action: Legal);
333 setOperationAction(Op: ISD::FMULADD, VT: MVT::v2f64, Action: Legal);
334 }
335
336 // Partial MLA reductions.
337 for (auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {
338 setPartialReduceMLAAction(Opc: Op, AccVT: MVT::v4i32, InputVT: MVT::v16i8, Action: Legal);
339 setPartialReduceMLAAction(Opc: Op, AccVT: MVT::v4i32, InputVT: MVT::v8i16, Action: Legal);
340 }
341 }
342
343 // As a special case, these operators use the type to mean the type to
344 // sign-extend from.
345 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
346 if (!Subtarget->hasSignExt()) {
347 // Sign extends are legal only when extending a vector extract
348 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
349 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
350 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action);
351 }
352 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
353 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Expand);
354
355 // Dynamic stack allocation: use the default expansion.
356 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
357 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
358 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVTPtr, Action: Expand);
359
360 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i32, Action: Custom);
361 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i64, Action: Custom);
362 setOperationAction(Op: ISD::CopyToReg, VT: MVT::Other, Action: Custom);
363
364 // Expand these forms; we pattern-match the forms that we can handle in isel.
365 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
366 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
367 setOperationAction(Op, VT: T, Action: Expand);
368
369 if (Subtarget->hasReferenceTypes())
370 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
371 for (auto T : {MVT::externref, MVT::funcref})
372 setOperationAction(Op, VT: T, Action: Expand);
373
374 // There is no vector conditional select instruction
375 for (auto T :
376 {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, MVT::v2f64})
377 setOperationAction(Op: ISD::SELECT_CC, VT: T, Action: Expand);
378
379 // We have custom switch handling.
380 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Custom);
381
382 // WebAssembly doesn't have:
383 // - Floating-point extending loads.
384 // - Floating-point truncating stores.
385 // - i1 extending loads.
386 // - truncating SIMD stores and most extending loads
387 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
388 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
389 for (auto T : MVT::integer_valuetypes())
390 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
391 setLoadExtAction(ExtType: Ext, ValVT: T, MemVT: MVT::i1, Action: Promote);
392 if (Subtarget->hasSIMD128()) {
393 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
394 MVT::v2f64}) {
395 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
396 if (MVT(T) != MemT) {
397 setTruncStoreAction(ValVT: T, MemVT: MemT, Action: Expand);
398 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
399 setLoadExtAction(ExtType: Ext, ValVT: T, MemVT: MemT, Action: Expand);
400 }
401 }
402 }
403 // But some vector extending loads are legal
404 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
405 setLoadExtAction(ExtType: Ext, ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Legal);
406 setLoadExtAction(ExtType: Ext, ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Legal);
407 setLoadExtAction(ExtType: Ext, ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Legal);
408 }
409 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Legal);
410 }
411
412 // Don't do anything clever with build_pairs
413 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
414
415 // Trap lowers to wasm unreachable
416 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
417 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
418
419 // Exception handling intrinsics
420 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
421 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
422 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
423
424 setMaxAtomicSizeInBitsSupported(64);
425
426 // Always convert switches to br_tables unless there is only one case, which
427 // is equivalent to a simple branch. This reduces code size for wasm, and we
428 // defer possible jump table optimizations to the VM.
429 setMinimumJumpTableEntries(2);
430}
431
432MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL,
433 uint32_t AS) const {
434 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
435 return MVT::externref;
436 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
437 return MVT::funcref;
438 return TargetLowering::getPointerTy(DL, AS);
439}
440
441MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
442 uint32_t AS) const {
443 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
444 return MVT::externref;
445 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
446 return MVT::funcref;
447 return TargetLowering::getPointerMemTy(DL, AS);
448}
449
450TargetLowering::AtomicExpansionKind
451WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(
452 const AtomicRMWInst *AI) const {
453 // We have wasm instructions for these
454 switch (AI->getOperation()) {
455 case AtomicRMWInst::Add:
456 case AtomicRMWInst::Sub:
457 case AtomicRMWInst::And:
458 case AtomicRMWInst::Or:
459 case AtomicRMWInst::Xor:
460 case AtomicRMWInst::Xchg:
461 return AtomicExpansionKind::None;
462 default:
463 break;
464 }
465 return AtomicExpansionKind::CmpXChg;
466}
467
468bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
469 // Implementation copied from X86TargetLowering.
470 unsigned Opc = VecOp.getOpcode();
471
472 // Assume target opcodes can't be scalarized.
473 // TODO - do we have any exceptions?
474 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
475 return false;
476
477 // If the vector op is not supported, try to convert to scalar.
478 EVT VecVT = VecOp.getValueType();
479 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
480 return true;
481
482 // If the vector op is supported, but the scalar op is not, the transform may
483 // not be worthwhile.
484 EVT ScalarVT = VecVT.getScalarType();
485 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
486}
487
488FastISel *WebAssemblyTargetLowering::createFastISel(
489 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
490 const LibcallLoweringInfo *LibcallLowering) const {
491 return WebAssembly::createFastISel(funcInfo&: FuncInfo, libInfo: LibInfo, libcallLowering: LibcallLowering);
492}
493
494MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
495 EVT VT) const {
496 unsigned BitWidth = NextPowerOf2(A: VT.getSizeInBits() - 1);
497 if (BitWidth > 1 && BitWidth < 8)
498 BitWidth = 8;
499
500 if (BitWidth > 64) {
501 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
502 // the count to be an i32.
503 BitWidth = 32;
504 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&
505 "32-bit shift counts ought to be enough for anyone");
506 }
507
508 MVT Result = MVT::getIntegerVT(BitWidth);
509 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&
510 "Unable to represent scalar shift amount type");
511 return Result;
512}
513
514// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
515// undefined result on invalid/overflow, to the WebAssembly opcode, which
516// traps on invalid/overflow.
517static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
518 MachineBasicBlock *BB,
519 const TargetInstrInfo &TII,
520 bool IsUnsigned, bool Int64,
521 bool Float64, unsigned LoweredOpcode) {
522 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
523
524 Register OutReg = MI.getOperand(i: 0).getReg();
525 Register InReg = MI.getOperand(i: 1).getReg();
526
527 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
528 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
529 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
530 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
531 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
532 unsigned Eqz = WebAssembly::EQZ_I32;
533 unsigned And = WebAssembly::AND_I32;
534 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
535 int64_t Substitute = IsUnsigned ? 0 : Limit;
536 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
537 auto &Context = BB->getParent()->getFunction().getContext();
538 Type *Ty = Float64 ? Type::getDoubleTy(C&: Context) : Type::getFloatTy(C&: Context);
539
540 const BasicBlock *LLVMBB = BB->getBasicBlock();
541 MachineFunction *F = BB->getParent();
542 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
543 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
544 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
545
546 MachineFunction::iterator It = ++BB->getIterator();
547 F->insert(MBBI: It, MBB: FalseMBB);
548 F->insert(MBBI: It, MBB: TrueMBB);
549 F->insert(MBBI: It, MBB: DoneMBB);
550
551 // Transfer the remainder of BB and its successor edges to DoneMBB.
552 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
553 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
554
555 BB->addSuccessor(Succ: TrueMBB);
556 BB->addSuccessor(Succ: FalseMBB);
557 TrueMBB->addSuccessor(Succ: DoneMBB);
558 FalseMBB->addSuccessor(Succ: DoneMBB);
559
560 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
561 Tmp0 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
562 Tmp1 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
563 CmpReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
564 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
565 FalseReg = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: OutReg));
566 TrueReg = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: OutReg));
567
568 MI.eraseFromParent();
569 // For signed numbers, we can do a single comparison to determine whether
570 // fabs(x) is within range.
571 if (IsUnsigned) {
572 Tmp0 = InReg;
573 } else {
574 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Abs), DestReg: Tmp0).addReg(RegNo: InReg);
575 }
576 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: FConst), DestReg: Tmp1)
577 .addFPImm(Val: cast<ConstantFP>(Val: ConstantFP::get(Ty, V: CmpVal)));
578 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: LT), DestReg: CmpReg).addReg(RegNo: Tmp0).addReg(RegNo: Tmp1);
579
580 // For unsigned numbers, we have to do a separate comparison with zero.
581 if (IsUnsigned) {
582 Tmp1 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
583 Register SecondCmpReg =
584 MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
585 Register AndReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
586 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: FConst), DestReg: Tmp1)
587 .addFPImm(Val: cast<ConstantFP>(Val: ConstantFP::get(Ty, V: 0.0)));
588 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: GE), DestReg: SecondCmpReg).addReg(RegNo: Tmp0).addReg(RegNo: Tmp1);
589 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: And), DestReg: AndReg).addReg(RegNo: CmpReg).addReg(RegNo: SecondCmpReg);
590 CmpReg = AndReg;
591 }
592
593 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).addReg(RegNo: CmpReg);
594
595 // Create the CFG diamond to select between doing the conversion or using
596 // the substitute value.
597 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: TrueMBB).addReg(RegNo: EqzReg);
598 BuildMI(BB: FalseMBB, MIMD: DL, MCID: TII.get(Opcode: LoweredOpcode), DestReg: FalseReg).addReg(RegNo: InReg);
599 BuildMI(BB: FalseMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
600 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: IConst), DestReg: TrueReg).addImm(Val: Substitute);
601 BuildMI(BB&: *DoneMBB, I: DoneMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: OutReg)
602 .addReg(RegNo: FalseReg)
603 .addMBB(MBB: FalseMBB)
604 .addReg(RegNo: TrueReg)
605 .addMBB(MBB: TrueMBB);
606
607 return DoneMBB;
608}
609
610// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`
611// instuction to handle the zero-length case.
612static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL,
613 MachineBasicBlock *BB,
614 const TargetInstrInfo &TII, bool Int64) {
615 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
616
617 MachineOperand DstMem = MI.getOperand(i: 0);
618 MachineOperand SrcMem = MI.getOperand(i: 1);
619 MachineOperand Dst = MI.getOperand(i: 2);
620 MachineOperand Src = MI.getOperand(i: 3);
621 MachineOperand Len = MI.getOperand(i: 4);
622
623 // If the length is a constant, we don't actually need the check.
624 if (MachineInstr *Def = MRI.getVRegDef(Reg: Len.getReg())) {
625 if (Def->getOpcode() == WebAssembly::CONST_I32 ||
626 Def->getOpcode() == WebAssembly::CONST_I64) {
627 if (Def->getOperand(i: 1).getImm() == 0) {
628 // A zero-length memcpy is a no-op.
629 MI.eraseFromParent();
630 return BB;
631 }
632 // A non-zero-length memcpy doesn't need a zero check.
633 unsigned MemoryCopy =
634 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
635 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: MemoryCopy))
636 .add(MO: DstMem)
637 .add(MO: SrcMem)
638 .add(MO: Dst)
639 .add(MO: Src)
640 .add(MO: Len);
641 MI.eraseFromParent();
642 return BB;
643 }
644 }
645
646 // We're going to add an extra use to `Len` to test if it's zero; that
647 // use shouldn't be a kill, even if the original use is.
648 MachineOperand NoKillLen = Len;
649 NoKillLen.setIsKill(false);
650
651 // Decide on which `MachineInstr` opcode we're going to use.
652 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
653 unsigned MemoryCopy =
654 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
655
656 // Create two new basic blocks; one for the new `memory.fill` that we can
657 // branch over, and one for the rest of the instructions after the original
658 // `memory.fill`.
659 const BasicBlock *LLVMBB = BB->getBasicBlock();
660 MachineFunction *F = BB->getParent();
661 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
662 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
663
664 MachineFunction::iterator It = ++BB->getIterator();
665 F->insert(MBBI: It, MBB: TrueMBB);
666 F->insert(MBBI: It, MBB: DoneMBB);
667
668 // Transfer the remainder of BB and its successor edges to DoneMBB.
669 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
670 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
671
672 // Connect the CFG edges.
673 BB->addSuccessor(Succ: TrueMBB);
674 BB->addSuccessor(Succ: DoneMBB);
675 TrueMBB->addSuccessor(Succ: DoneMBB);
676
677 // Create a virtual register for the `Eqz` result.
678 unsigned EqzReg;
679 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
680
681 // Erase the original `memory.copy`.
682 MI.eraseFromParent();
683
684 // Test if `Len` is zero.
685 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).add(MO: NoKillLen);
686
687 // Insert a new `memory.copy`.
688 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: MemoryCopy))
689 .add(MO: DstMem)
690 .add(MO: SrcMem)
691 .add(MO: Dst)
692 .add(MO: Src)
693 .add(MO: Len);
694
695 // Create the CFG triangle.
696 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: DoneMBB).addReg(RegNo: EqzReg);
697 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
698
699 return DoneMBB;
700}
701
702// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`
703// instuction to handle the zero-length case.
704static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL,
705 MachineBasicBlock *BB,
706 const TargetInstrInfo &TII, bool Int64) {
707 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
708
709 MachineOperand Mem = MI.getOperand(i: 0);
710 MachineOperand Dst = MI.getOperand(i: 1);
711 MachineOperand Val = MI.getOperand(i: 2);
712 MachineOperand Len = MI.getOperand(i: 3);
713
714 // If the length is a constant, we don't actually need the check.
715 if (MachineInstr *Def = MRI.getVRegDef(Reg: Len.getReg())) {
716 if (Def->getOpcode() == WebAssembly::CONST_I32 ||
717 Def->getOpcode() == WebAssembly::CONST_I64) {
718 if (Def->getOperand(i: 1).getImm() == 0) {
719 // A zero-length memset is a no-op.
720 MI.eraseFromParent();
721 return BB;
722 }
723 // A non-zero-length memset doesn't need a zero check.
724 unsigned MemoryFill =
725 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
726 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: MemoryFill))
727 .add(MO: Mem)
728 .add(MO: Dst)
729 .add(MO: Val)
730 .add(MO: Len);
731 MI.eraseFromParent();
732 return BB;
733 }
734 }
735
736 // We're going to add an extra use to `Len` to test if it's zero; that
737 // use shouldn't be a kill, even if the original use is.
738 MachineOperand NoKillLen = Len;
739 NoKillLen.setIsKill(false);
740
741 // Decide on which `MachineInstr` opcode we're going to use.
742 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
743 unsigned MemoryFill =
744 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
745
746 // Create two new basic blocks; one for the new `memory.fill` that we can
747 // branch over, and one for the rest of the instructions after the original
748 // `memory.fill`.
749 const BasicBlock *LLVMBB = BB->getBasicBlock();
750 MachineFunction *F = BB->getParent();
751 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
752 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
753
754 MachineFunction::iterator It = ++BB->getIterator();
755 F->insert(MBBI: It, MBB: TrueMBB);
756 F->insert(MBBI: It, MBB: DoneMBB);
757
758 // Transfer the remainder of BB and its successor edges to DoneMBB.
759 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
760 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
761
762 // Connect the CFG edges.
763 BB->addSuccessor(Succ: TrueMBB);
764 BB->addSuccessor(Succ: DoneMBB);
765 TrueMBB->addSuccessor(Succ: DoneMBB);
766
767 // Create a virtual register for the `Eqz` result.
768 unsigned EqzReg;
769 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
770
771 // Erase the original `memory.fill`.
772 MI.eraseFromParent();
773
774 // Test if `Len` is zero.
775 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).add(MO: NoKillLen);
776
777 // Insert a new `memory.copy`.
778 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: MemoryFill)).add(MO: Mem).add(MO: Dst).add(MO: Val).add(MO: Len);
779
780 // Create the CFG triangle.
781 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: DoneMBB).addReg(RegNo: EqzReg);
782 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
783
784 return DoneMBB;
785}
786
787static MachineBasicBlock *
788LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
789 const WebAssemblySubtarget *Subtarget,
790 const TargetInstrInfo &TII) {
791 MachineInstr &CallParams = *CallResults.getPrevNode();
792 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS);
793 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
794 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
795
796 bool IsIndirect =
797 CallParams.getOperand(i: 0).isReg() || CallParams.getOperand(i: 0).isFI();
798 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
799
800 bool IsFuncrefCall = false;
801 if (IsIndirect && CallParams.getOperand(i: 0).isReg()) {
802 Register Reg = CallParams.getOperand(i: 0).getReg();
803 const MachineFunction *MF = BB->getParent();
804 const MachineRegisterInfo &MRI = MF->getRegInfo();
805 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
806 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
807 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
808 }
809
810 unsigned CallOp;
811 if (IsIndirect && IsRetCall) {
812 CallOp = WebAssembly::RET_CALL_INDIRECT;
813 } else if (IsIndirect) {
814 CallOp = WebAssembly::CALL_INDIRECT;
815 } else if (IsRetCall) {
816 CallOp = WebAssembly::RET_CALL;
817 } else {
818 CallOp = WebAssembly::CALL;
819 }
820
821 MachineFunction &MF = *BB->getParent();
822 const MCInstrDesc &MCID = TII.get(Opcode: CallOp);
823 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
824
825 // Move the function pointer to the end of the arguments for indirect calls
826 if (IsIndirect) {
827 auto FnPtr = CallParams.getOperand(i: 0);
828 CallParams.removeOperand(OpNo: 0);
829
830 // For funcrefs, call_indirect is done through __funcref_call_table and the
831 // funcref is always installed in slot 0 of the table, therefore instead of
832 // having the function pointer added at the end of the params list, a zero
833 // (the index in
834 // __funcref_call_table is added).
835 if (IsFuncrefCall) {
836 Register RegZero =
837 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
838 MachineInstrBuilder MIBC0 =
839 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::CONST_I32), DestReg: RegZero).addImm(Val: 0);
840
841 BB->insert(I: CallResults.getIterator(), M: MIBC0);
842 MachineInstrBuilder(MF, CallParams).addReg(RegNo: RegZero);
843 } else
844 CallParams.addOperand(Op: FnPtr);
845 }
846
847 for (auto Def : CallResults.defs())
848 MIB.add(MO: Def);
849
850 if (IsIndirect) {
851 // Placeholder for the type index.
852 // This gets replaced with the correct value in WebAssemblyMCInstLower.cpp
853 MIB.addImm(Val: 0);
854 // The table into which this call_indirect indexes.
855 MCSymbolWasm *Table = IsFuncrefCall
856 ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
857 Ctx&: MF.getContext(), Subtarget)
858 : WebAssembly::getOrCreateFunctionTableSymbol(
859 Ctx&: MF.getContext(), Subtarget);
860 if (Subtarget->hasCallIndirectOverlong()) {
861 MIB.addSym(Sym: Table);
862 } else {
863 // For the MVP there is at most one table whose number is 0, but we can't
864 // write a table symbol or issue relocations. Instead we just ensure the
865 // table is live and write a zero.
866 Table->setNoStrip();
867 MIB.addImm(Val: 0);
868 }
869 }
870
871 for (auto Use : CallParams.uses())
872 MIB.add(MO: Use);
873
874 BB->insert(I: CallResults.getIterator(), M: MIB);
875 CallParams.eraseFromParent();
876 CallResults.eraseFromParent();
877
878 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
879 // table slot with ref.null upon call_indirect return.
880 //
881 // This generates the following code, which comes right after a call_indirect
882 // of a funcref:
883 //
884 // i32.const 0
885 // ref.null func
886 // table.set __funcref_call_table
887 if (IsIndirect && IsFuncrefCall) {
888 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
889 Ctx&: MF.getContext(), Subtarget);
890 Register RegZero =
891 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
892 MachineInstr *Const0 =
893 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::CONST_I32), DestReg: RegZero).addImm(Val: 0);
894 BB->insertAfter(I: MIB.getInstr()->getIterator(), MI: Const0);
895
896 Register RegFuncref =
897 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::FUNCREFRegClass);
898 MachineInstr *RefNull =
899 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::REF_NULL_FUNCREF), DestReg: RegFuncref);
900 BB->insertAfter(I: Const0->getIterator(), MI: RefNull);
901
902 MachineInstr *TableSet =
903 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::TABLE_SET_FUNCREF))
904 .addSym(Sym: Table)
905 .addReg(RegNo: RegZero)
906 .addReg(RegNo: RegFuncref);
907 BB->insertAfter(I: RefNull->getIterator(), MI: TableSet);
908 }
909
910 return BB;
911}
912
913MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
914 MachineInstr &MI, MachineBasicBlock *BB) const {
915 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
916 DebugLoc DL = MI.getDebugLoc();
917
918 switch (MI.getOpcode()) {
919 default:
920 llvm_unreachable("Unexpected instr type to insert");
921 case WebAssembly::FP_TO_SINT_I32_F32:
922 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: false, Float64: false,
923 LoweredOpcode: WebAssembly::I32_TRUNC_S_F32);
924 case WebAssembly::FP_TO_UINT_I32_F32:
925 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: false, Float64: false,
926 LoweredOpcode: WebAssembly::I32_TRUNC_U_F32);
927 case WebAssembly::FP_TO_SINT_I64_F32:
928 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: true, Float64: false,
929 LoweredOpcode: WebAssembly::I64_TRUNC_S_F32);
930 case WebAssembly::FP_TO_UINT_I64_F32:
931 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: true, Float64: false,
932 LoweredOpcode: WebAssembly::I64_TRUNC_U_F32);
933 case WebAssembly::FP_TO_SINT_I32_F64:
934 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: false, Float64: true,
935 LoweredOpcode: WebAssembly::I32_TRUNC_S_F64);
936 case WebAssembly::FP_TO_UINT_I32_F64:
937 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: false, Float64: true,
938 LoweredOpcode: WebAssembly::I32_TRUNC_U_F64);
939 case WebAssembly::FP_TO_SINT_I64_F64:
940 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: true, Float64: true,
941 LoweredOpcode: WebAssembly::I64_TRUNC_S_F64);
942 case WebAssembly::FP_TO_UINT_I64_F64:
943 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: true, Float64: true,
944 LoweredOpcode: WebAssembly::I64_TRUNC_U_F64);
945 case WebAssembly::MEMCPY_A32:
946 return LowerMemcpy(MI, DL, BB, TII, Int64: false);
947 case WebAssembly::MEMCPY_A64:
948 return LowerMemcpy(MI, DL, BB, TII, Int64: true);
949 case WebAssembly::MEMSET_A32:
950 return LowerMemset(MI, DL, BB, TII, Int64: false);
951 case WebAssembly::MEMSET_A64:
952 return LowerMemset(MI, DL, BB, TII, Int64: true);
953 case WebAssembly::CALL_RESULTS:
954 case WebAssembly::RET_CALL_RESULTS:
955 return LowerCallResults(CallResults&: MI, DL, BB, Subtarget, TII);
956 }
957}
958
959std::pair<unsigned, const TargetRegisterClass *>
960WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
961 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
962 // First, see if this is a constraint that directly corresponds to a
963 // WebAssembly register class.
964 if (Constraint.size() == 1) {
965 switch (Constraint[0]) {
966 case 'r':
967 assert(VT != MVT::iPTR && "Pointer MVT not expected here");
968 if (Subtarget->hasSIMD128() && VT.isVector()) {
969 if (VT.getSizeInBits() == 128)
970 return std::make_pair(x: 0U, y: &WebAssembly::V128RegClass);
971 }
972 if (VT.isInteger() && !VT.isVector()) {
973 if (VT.getSizeInBits() <= 32)
974 return std::make_pair(x: 0U, y: &WebAssembly::I32RegClass);
975 if (VT.getSizeInBits() <= 64)
976 return std::make_pair(x: 0U, y: &WebAssembly::I64RegClass);
977 }
978 if (VT.isFloatingPoint() && !VT.isVector()) {
979 switch (VT.getSizeInBits()) {
980 case 32:
981 return std::make_pair(x: 0U, y: &WebAssembly::F32RegClass);
982 case 64:
983 return std::make_pair(x: 0U, y: &WebAssembly::F64RegClass);
984 default:
985 break;
986 }
987 }
988 break;
989 default:
990 break;
991 }
992 }
993
994 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
995}
996
997bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
998 // Assume ctz is a relatively cheap operation.
999 return true;
1000}
1001
1002bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
1003 // Assume clz is a relatively cheap operation.
1004 return true;
1005}
1006
1007bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1008 const AddrMode &AM,
1009 Type *Ty, unsigned AS,
1010 Instruction *I) const {
1011 // WebAssembly offsets are added as unsigned without wrapping. The
1012 // isLegalAddressingMode gives us no way to determine if wrapping could be
1013 // happening, so we approximate this by accepting only non-negative offsets.
1014 if (AM.BaseOffs < 0)
1015 return false;
1016
1017 // WebAssembly has no scale register operands.
1018 if (AM.Scale != 0)
1019 return false;
1020
1021 // Everything else is legal.
1022 return true;
1023}
1024
1025bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
1026 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
1027 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
1028 // WebAssembly supports unaligned accesses, though it should be declared
1029 // with the p2align attribute on loads and stores which do so, and there
1030 // may be a performance impact. We tell LLVM they're "fast" because
1031 // for the kinds of things that LLVM uses this for (merging adjacent stores
1032 // of constants, etc.), WebAssembly implementations will either want the
1033 // unaligned access or they'll split anyway.
1034 if (Fast)
1035 *Fast = 1;
1036 return true;
1037}
1038
1039bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
1040 AttributeList Attr) const {
1041 // The current thinking is that wasm engines will perform this optimization,
1042 // so we can save on code size.
1043 return true;
1044}
1045
1046bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
1047 EVT ExtT = ExtVal.getValueType();
1048 SDValue N0 = ExtVal->getOperand(Num: 0);
1049 if (N0.getOpcode() == ISD::FREEZE)
1050 N0 = N0.getOperand(i: 0);
1051 auto *Load = dyn_cast<LoadSDNode>(Val&: N0);
1052 if (!Load)
1053 return false;
1054 EVT MemT = Load->getValueType(ResNo: 0);
1055 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
1056 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
1057 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
1058}
1059
1060bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
1061 const GlobalAddressSDNode *GA) const {
1062 // Wasm doesn't support function addresses with offsets
1063 const GlobalValue *GV = GA->getGlobal();
1064 return isa<Function>(Val: GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
1065}
1066
1067EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
1068 LLVMContext &C,
1069 EVT VT) const {
1070 if (VT.isVector())
1071 return VT.changeVectorElementTypeToInteger();
1072
1073 // So far, all branch instructions in Wasm take an I32 condition.
1074 // The default TargetLowering::getSetCCResultType returns the pointer size,
1075 // which would be useful to reduce instruction counts when testing
1076 // against 64-bit pointers/values if at some point Wasm supports that.
1077 return EVT::getIntegerVT(Context&: C, BitWidth: 32);
1078}
1079
1080void WebAssemblyTargetLowering::getTgtMemIntrinsic(
1081 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
1082 MachineFunction &MF, unsigned Intrinsic) const {
1083 IntrinsicInfo Info;
1084 switch (Intrinsic) {
1085 case Intrinsic::wasm_memory_atomic_notify:
1086 Info.opc = ISD::INTRINSIC_W_CHAIN;
1087 Info.memVT = MVT::i32;
1088 Info.ptrVal = I.getArgOperand(i: 0);
1089 Info.offset = 0;
1090 Info.align = Align(4);
1091 // atomic.notify instruction does not really load the memory specified with
1092 // this argument, but MachineMemOperand should either be load or store, so
1093 // we set this to a load.
1094 // FIXME Volatile isn't really correct, but currently all LLVM atomic
1095 // instructions are treated as volatiles in the backend, so we should be
1096 // consistent. The same applies for wasm_atomic_wait intrinsics too.
1097 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1098 Infos.push_back(Elt: Info);
1099 return;
1100 case Intrinsic::wasm_memory_atomic_wait32:
1101 Info.opc = ISD::INTRINSIC_W_CHAIN;
1102 Info.memVT = MVT::i32;
1103 Info.ptrVal = I.getArgOperand(i: 0);
1104 Info.offset = 0;
1105 Info.align = Align(4);
1106 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1107 Infos.push_back(Elt: Info);
1108 return;
1109 case Intrinsic::wasm_memory_atomic_wait64:
1110 Info.opc = ISD::INTRINSIC_W_CHAIN;
1111 Info.memVT = MVT::i64;
1112 Info.ptrVal = I.getArgOperand(i: 0);
1113 Info.offset = 0;
1114 Info.align = Align(8);
1115 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1116 Infos.push_back(Elt: Info);
1117 return;
1118 case Intrinsic::wasm_loadf16_f32:
1119 Info.opc = ISD::INTRINSIC_W_CHAIN;
1120 Info.memVT = MVT::f16;
1121 Info.ptrVal = I.getArgOperand(i: 0);
1122 Info.offset = 0;
1123 Info.align = Align(2);
1124 Info.flags = MachineMemOperand::MOLoad;
1125 Infos.push_back(Elt: Info);
1126 return;
1127 case Intrinsic::wasm_storef16_f32:
1128 Info.opc = ISD::INTRINSIC_VOID;
1129 Info.memVT = MVT::f16;
1130 Info.ptrVal = I.getArgOperand(i: 1);
1131 Info.offset = 0;
1132 Info.align = Align(2);
1133 Info.flags = MachineMemOperand::MOStore;
1134 Infos.push_back(Elt: Info);
1135 return;
1136 default:
1137 return;
1138 }
1139}
1140
1141void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
1142 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
1143 const SelectionDAG &DAG, unsigned Depth) const {
1144 switch (Op.getOpcode()) {
1145 default:
1146 break;
1147 case ISD::INTRINSIC_WO_CHAIN: {
1148 unsigned IntNo = Op.getConstantOperandVal(i: 0);
1149 switch (IntNo) {
1150 default:
1151 break;
1152 case Intrinsic::wasm_bitmask: {
1153 unsigned BitWidth = Known.getBitWidth();
1154 EVT VT = Op.getOperand(i: 1).getSimpleValueType();
1155 unsigned PossibleBits = VT.getVectorNumElements();
1156 APInt ZeroMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - PossibleBits);
1157 Known.Zero |= ZeroMask;
1158 break;
1159 }
1160 }
1161 break;
1162 }
1163 case WebAssemblyISD::EXTEND_LOW_U:
1164 case WebAssemblyISD::EXTEND_HIGH_U: {
1165 // We know the high half, of each destination vector element, will be zero.
1166 SDValue SrcOp = Op.getOperand(i: 0);
1167 EVT VT = SrcOp.getSimpleValueType();
1168 unsigned BitWidth = Known.getBitWidth();
1169 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1170 assert(BitWidth >= 8 && "Unexpected width!");
1171 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 8);
1172 Known.Zero |= Mask;
1173 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1174 assert(BitWidth >= 16 && "Unexpected width!");
1175 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
1176 Known.Zero |= Mask;
1177 } else if (VT == MVT::v2i32 || VT == MVT::v4i32) {
1178 assert(BitWidth >= 32 && "Unexpected width!");
1179 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 32);
1180 Known.Zero |= Mask;
1181 }
1182 break;
1183 }
1184 // For 128-bit addition if the upper bits are all zero then it's known that
1185 // the upper bits of the result will have all bits guaranteed zero except the
1186 // first.
1187 case WebAssemblyISD::I64_ADD128:
1188 if (Op.getResNo() == 1) {
1189 SDValue LHS_HI = Op.getOperand(i: 1);
1190 SDValue RHS_HI = Op.getOperand(i: 3);
1191 if (isNullConstant(V: LHS_HI) && isNullConstant(V: RHS_HI))
1192 Known.Zero.setBitsFrom(1);
1193 }
1194 break;
1195 }
1196}
1197
1198TargetLoweringBase::LegalizeTypeAction
1199WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
1200 if (VT.isFixedLengthVector()) {
1201 MVT EltVT = VT.getVectorElementType();
1202 // We have legal vector types with these lane types, so widening the
1203 // vector would let us use some of the lanes directly without having to
1204 // extend or truncate values.
1205 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
1206 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
1207 return TypeWidenVector;
1208 }
1209
1210 return TargetLoweringBase::getPreferredVectorAction(VT);
1211}
1212
1213bool WebAssemblyTargetLowering::isFMAFasterThanFMulAndFAdd(
1214 const MachineFunction &MF, EVT VT) const {
1215 if (!Subtarget->hasFP16() || !VT.isVector())
1216 return false;
1217
1218 EVT ScalarVT = VT.getScalarType();
1219 if (!ScalarVT.isSimple())
1220 return false;
1221
1222 return ScalarVT.getSimpleVT().SimpleTy == MVT::f16;
1223}
1224
1225bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
1226 SDValue Op, const TargetLoweringOpt &TLO) const {
1227 // ISel process runs DAGCombiner after legalization; this step is called
1228 // SelectionDAG optimization phase. This post-legalization combining process
1229 // runs DAGCombiner on each node, and if there was a change to be made,
1230 // re-runs legalization again on it and its user nodes to make sure
1231 // everythiing is in a legalized state.
1232 //
1233 // The legalization calls lowering routines, and we do our custom lowering for
1234 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
1235 // into zeros. But there is a set of routines in DAGCombiner that turns unused
1236 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
1237 // turns unused vector elements into undefs. But this routine does not work
1238 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
1239 // combination can result in a infinite loop, in which undefs are converted to
1240 // zeros in legalization and back to undefs in combining.
1241 //
1242 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
1243 // running for build_vectors.
1244 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
1245 return false;
1246 return true;
1247}
1248
1249//===----------------------------------------------------------------------===//
1250// WebAssembly Lowering private implementation.
1251//===----------------------------------------------------------------------===//
1252
1253//===----------------------------------------------------------------------===//
1254// Lowering Code
1255//===----------------------------------------------------------------------===//
1256
1257static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
1258 MachineFunction &MF = DAG.getMachineFunction();
1259 DAG.getContext()->diagnose(
1260 DI: DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
1261}
1262
1263// Test whether the given calling convention is supported.
1264static bool callingConvSupported(CallingConv::ID CallConv) {
1265 // We currently support the language-independent target-independent
1266 // conventions. We don't yet have a way to annotate calls with properties like
1267 // "cold", and we don't have any call-clobbered registers, so these are mostly
1268 // all handled the same.
1269 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
1270 CallConv == CallingConv::Cold ||
1271 CallConv == CallingConv::PreserveMost ||
1272 CallConv == CallingConv::PreserveAll ||
1273 CallConv == CallingConv::CXX_FAST_TLS ||
1274 CallConv == CallingConv::WASM_EmscriptenInvoke ||
1275 CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail;
1276}
1277
1278SDValue
1279WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
1280 SmallVectorImpl<SDValue> &InVals) const {
1281 SelectionDAG &DAG = CLI.DAG;
1282 SDLoc DL = CLI.DL;
1283 SDValue Chain = CLI.Chain;
1284 SDValue Callee = CLI.Callee;
1285 MachineFunction &MF = DAG.getMachineFunction();
1286 auto Layout = MF.getDataLayout();
1287
1288 CallingConv::ID CallConv = CLI.CallConv;
1289 if (!callingConvSupported(CallConv))
1290 fail(DL, DAG,
1291 Msg: "WebAssembly doesn't support language-specific or target-specific "
1292 "calling conventions yet");
1293 if (CLI.IsPatchPoint)
1294 fail(DL, DAG, Msg: "WebAssembly doesn't support patch point yet");
1295
1296 if (CLI.IsTailCall) {
1297 auto NoTail = [&](const char *Msg) {
1298 if (CLI.CB && CLI.CB->isMustTailCall())
1299 fail(DL, DAG, Msg);
1300 CLI.IsTailCall = false;
1301 };
1302
1303 if (!Subtarget->hasTailCall())
1304 NoTail("WebAssembly 'tail-call' feature not enabled");
1305
1306 // Varargs calls cannot be tail calls because the buffer is on the stack
1307 if (CLI.IsVarArg)
1308 NoTail("WebAssembly does not support varargs tail calls");
1309
1310 // Do not tail call unless caller and callee return types match
1311 const Function &F = MF.getFunction();
1312 const TargetMachine &TM = getTargetMachine();
1313 Type *RetTy = F.getReturnType();
1314 SmallVector<MVT, 4> CallerRetTys;
1315 SmallVector<MVT, 4> CalleeRetTys;
1316 computeLegalValueVTs(F, TM, Ty: RetTy, ValueVTs&: CallerRetTys);
1317 computeLegalValueVTs(F, TM, Ty: CLI.RetTy, ValueVTs&: CalleeRetTys);
1318 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1319 std::equal(first1: CallerRetTys.begin(), last1: CallerRetTys.end(),
1320 first2: CalleeRetTys.begin());
1321 if (!TypesMatch)
1322 NoTail("WebAssembly tail call requires caller and callee return types to "
1323 "match");
1324
1325 // If pointers to local stack values are passed, we cannot tail call
1326 if (CLI.CB) {
1327 for (auto &Arg : CLI.CB->args()) {
1328 Value *Val = Arg.get();
1329 // Trace the value back through pointer operations
1330 while (true) {
1331 Value *Src = Val->stripPointerCastsAndAliases();
1332 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: Src))
1333 Src = GEP->getPointerOperand();
1334 if (Val == Src)
1335 break;
1336 Val = Src;
1337 }
1338 if (isa<AllocaInst>(Val)) {
1339 NoTail(
1340 "WebAssembly does not support tail calling with stack arguments");
1341 break;
1342 }
1343 }
1344 }
1345 }
1346
1347 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1348 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1349 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1350
1351 // The generic code may have added an sret argument. If we're lowering an
1352 // invoke function, the ABI requires that the function pointer be the first
1353 // argument, so we may have to swap the arguments.
1354 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1355 Outs[0].Flags.isSRet()) {
1356 std::swap(a&: Outs[0], b&: Outs[1]);
1357 std::swap(a&: OutVals[0], b&: OutVals[1]);
1358 }
1359
1360 bool HasSwiftSelfArg = false;
1361 bool HasSwiftErrorArg = false;
1362 bool HasSwiftAsyncArg = false;
1363 unsigned NumFixedArgs = 0;
1364 for (unsigned I = 0; I < Outs.size(); ++I) {
1365 const ISD::OutputArg &Out = Outs[I];
1366 SDValue &OutVal = OutVals[I];
1367 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1368 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1369 HasSwiftAsyncArg |= Out.Flags.isSwiftAsync();
1370 if (Out.Flags.isNest())
1371 fail(DL, DAG, Msg: "WebAssembly hasn't implemented nest arguments");
1372 if (Out.Flags.isInAlloca())
1373 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca arguments");
1374 if (Out.Flags.isInConsecutiveRegs())
1375 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs arguments");
1376 if (Out.Flags.isInConsecutiveRegsLast())
1377 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last arguments");
1378 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1379 auto &MFI = MF.getFrameInfo();
1380 int FI = MFI.CreateStackObject(Size: Out.Flags.getByValSize(),
1381 Alignment: Out.Flags.getNonZeroByValAlign(),
1382 /*isSS=*/isSpillSlot: false);
1383 SDValue SizeNode =
1384 DAG.getConstant(Val: Out.Flags.getByValSize(), DL, VT: MVT::i32);
1385 SDValue FINode = DAG.getFrameIndex(FI, VT: getPointerTy(DL: Layout));
1386 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FINode, Src: OutVal, Size: SizeNode,
1387 Alignment: Out.Flags.getNonZeroByValAlign(),
1388 /*isVolatile*/ isVol: false, /*AlwaysInline=*/false,
1389 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
1390 SrcPtrInfo: MachinePointerInfo());
1391 OutVal = FINode;
1392 }
1393 // Count the number of fixed args *after* legalization.
1394 NumFixedArgs += !Out.Flags.isVarArg();
1395 }
1396
1397 bool IsVarArg = CLI.IsVarArg;
1398 auto PtrVT = getPointerTy(DL: Layout);
1399
1400 // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
1401 // (for swifttailcc) swiftasync arguments if there aren't. These additional
1402 // arguments are also added for callee signature. They are necessary to match
1403 // callee and caller signature for indirect call.
1404 if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
1405 Type *PtrTy = PointerType::getUnqual(C&: *DAG.getContext());
1406 if (!HasSwiftSelfArg) {
1407 NumFixedArgs++;
1408 ISD::ArgFlagsTy Flags;
1409 Flags.setSwiftSelf();
1410 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1411 CLI.Outs.push_back(Elt: Arg);
1412 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1413 CLI.OutVals.push_back(Elt: ArgVal);
1414 }
1415 if (!HasSwiftErrorArg) {
1416 NumFixedArgs++;
1417 ISD::ArgFlagsTy Flags;
1418 Flags.setSwiftError();
1419 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1420 CLI.Outs.push_back(Elt: Arg);
1421 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1422 CLI.OutVals.push_back(Elt: ArgVal);
1423 }
1424 if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
1425 NumFixedArgs++;
1426 ISD::ArgFlagsTy Flags;
1427 Flags.setSwiftAsync();
1428 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1429 CLI.Outs.push_back(Elt: Arg);
1430 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1431 CLI.OutVals.push_back(Elt: ArgVal);
1432 }
1433 }
1434
1435 // Analyze operands of the call, assigning locations to each operand.
1436 SmallVector<CCValAssign, 16> ArgLocs;
1437 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1438
1439 if (IsVarArg) {
1440 // Outgoing non-fixed arguments are placed in a buffer. First
1441 // compute their offsets and the total amount of buffer space needed.
1442 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1443 const ISD::OutputArg &Out = Outs[I];
1444 SDValue &Arg = OutVals[I];
1445 EVT VT = Arg.getValueType();
1446 assert(VT != MVT::iPTR && "Legalized args should be concrete");
1447 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
1448 Align Alignment =
1449 std::max(a: Out.Flags.getNonZeroOrigAlign(), b: Layout.getABITypeAlign(Ty));
1450 unsigned Offset =
1451 CCInfo.AllocateStack(Size: Layout.getTypeAllocSize(Ty), Alignment);
1452 CCInfo.addLoc(V: CCValAssign::getMem(ValNo: ArgLocs.size(), ValVT: VT.getSimpleVT(),
1453 Offset, LocVT: VT.getSimpleVT(),
1454 HTP: CCValAssign::Full));
1455 }
1456 }
1457
1458 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1459
1460 SDValue FINode;
1461 if (IsVarArg && NumBytes) {
1462 // For non-fixed arguments, next emit stores to store the argument values
1463 // to the stack buffer at the offsets computed above.
1464 MaybeAlign StackAlign = Layout.getStackAlignment();
1465 assert(StackAlign && "data layout string is missing stack alignment");
1466 int FI = MF.getFrameInfo().CreateStackObject(Size: NumBytes, Alignment: *StackAlign,
1467 /*isSS=*/isSpillSlot: false);
1468 unsigned ValNo = 0;
1469 SmallVector<SDValue, 8> Chains;
1470 for (SDValue Arg : drop_begin(RangeOrContainer&: OutVals, N: NumFixedArgs)) {
1471 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1472 "ArgLocs should remain in order and only hold varargs args");
1473 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1474 FINode = DAG.getFrameIndex(FI, VT: getPointerTy(DL: Layout));
1475 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FINode,
1476 N2: DAG.getConstant(Val: Offset, DL, VT: PtrVT));
1477 Chains.push_back(
1478 Elt: DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: Add,
1479 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset)));
1480 }
1481 if (!Chains.empty())
1482 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
1483 } else if (IsVarArg) {
1484 FINode = DAG.getIntPtrConstant(Val: 0, DL);
1485 }
1486
1487 if (Callee->getOpcode() == ISD::GlobalAddress) {
1488 // If the callee is a GlobalAddress node (quite common, every direct call
1489 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1490 // doesn't at MO_GOT which is not needed for direct calls.
1491 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Callee);
1492 Callee = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL,
1493 VT: getPointerTy(DL: DAG.getDataLayout()),
1494 offset: GA->getOffset());
1495 Callee = DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL,
1496 VT: getPointerTy(DL: DAG.getDataLayout()), Operand: Callee);
1497 }
1498
1499 // Compute the operands for the CALLn node.
1500 SmallVector<SDValue, 16> Ops;
1501 Ops.push_back(Elt: Chain);
1502 Ops.push_back(Elt: Callee);
1503
1504 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1505 // isn't reliable.
1506 Ops.append(in_start: OutVals.begin(),
1507 in_end: IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1508 // Add a pointer to the vararg buffer.
1509 if (IsVarArg)
1510 Ops.push_back(Elt: FINode);
1511
1512 SmallVector<EVT, 8> InTys;
1513 for (const auto &In : Ins) {
1514 assert(!In.Flags.isByVal() && "byval is not valid for return values");
1515 assert(!In.Flags.isNest() && "nest is not valid for return values");
1516 if (In.Flags.isInAlloca())
1517 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca return values");
1518 if (In.Flags.isInConsecutiveRegs())
1519 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs return values");
1520 if (In.Flags.isInConsecutiveRegsLast())
1521 fail(DL, DAG,
1522 Msg: "WebAssembly hasn't implemented cons regs last return values");
1523 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1524 // registers.
1525 InTys.push_back(Elt: In.VT);
1526 }
1527
1528 // Lastly, if this is a call to a funcref we need to add an instruction
1529 // table.set to the chain and transform the call.
1530 if (CLI.CB && WebAssembly::isWebAssemblyFuncrefType(
1531 Ty: CLI.CB->getCalledOperand()->getType())) {
1532 // In the absence of function references proposal where a funcref call is
1533 // lowered to call_ref, using reference types we generate a table.set to set
1534 // the funcref to a special table used solely for this purpose, followed by
1535 // a call_indirect. Here we just generate the table set, and return the
1536 // SDValue of the table.set so that LowerCall can finalize the lowering by
1537 // generating the call_indirect.
1538 SDValue Chain = Ops[0];
1539
1540 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
1541 Ctx&: MF.getContext(), Subtarget);
1542 SDValue Sym = DAG.getMCSymbol(Sym: Table, VT: PtrVT);
1543 SDValue TableSlot = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
1544 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1545 SDValue TableSet = DAG.getMemIntrinsicNode(
1546 Opcode: WebAssemblyISD::TABLE_SET, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: TableSetOps,
1547 MemVT: MVT::funcref,
1548 // Machine Mem Operand args
1549 PtrInfo: MachinePointerInfo(
1550 WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF),
1551 Alignment: CLI.CB->getCalledOperand()->getPointerAlignment(DL: DAG.getDataLayout()),
1552 Flags: MachineMemOperand::MOStore);
1553
1554 Ops[0] = TableSet; // The new chain is the TableSet itself
1555 }
1556
1557 if (CLI.IsTailCall) {
1558 // ret_calls do not return values to the current frame
1559 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1560 return DAG.getNode(Opcode: WebAssemblyISD::RET_CALL, DL, VTList: NodeTys, Ops);
1561 }
1562
1563 InTys.push_back(Elt: MVT::Other);
1564 SDVTList InTyList = DAG.getVTList(VTs: InTys);
1565 SDValue Res = DAG.getNode(Opcode: WebAssemblyISD::CALL, DL, VTList: InTyList, Ops);
1566
1567 for (size_t I = 0; I < Ins.size(); ++I)
1568 InVals.push_back(Elt: Res.getValue(R: I));
1569
1570 // Return the chain
1571 return Res.getValue(R: Ins.size());
1572}
1573
1574bool WebAssemblyTargetLowering::CanLowerReturn(
1575 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1576 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext & /*Context*/,
1577 const Type *RetTy) const {
1578 // WebAssembly can only handle returning tuples with multivalue enabled
1579 return WebAssembly::canLowerReturn(ResultSize: Outs.size(), Subtarget);
1580}
1581
1582SDValue WebAssemblyTargetLowering::LowerReturn(
1583 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1584 const SmallVectorImpl<ISD::OutputArg> &Outs,
1585 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1586 SelectionDAG &DAG) const {
1587 assert(WebAssembly::canLowerReturn(Outs.size(), Subtarget) &&
1588 "MVP WebAssembly can only return up to one value");
1589 if (!callingConvSupported(CallConv))
1590 fail(DL, DAG, Msg: "WebAssembly doesn't support non-C calling conventions");
1591
1592 SmallVector<SDValue, 4> RetOps(1, Chain);
1593 RetOps.append(in_start: OutVals.begin(), in_end: OutVals.end());
1594 Chain = DAG.getNode(Opcode: WebAssemblyISD::RETURN, DL, VT: MVT::Other, Ops: RetOps);
1595
1596 // Record the number and types of the return values.
1597 for (const ISD::OutputArg &Out : Outs) {
1598 assert(!Out.Flags.isByVal() && "byval is not valid for return values");
1599 assert(!Out.Flags.isNest() && "nest is not valid for return values");
1600 assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");
1601 if (Out.Flags.isInAlloca())
1602 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca results");
1603 if (Out.Flags.isInConsecutiveRegs())
1604 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs results");
1605 if (Out.Flags.isInConsecutiveRegsLast())
1606 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last results");
1607 }
1608
1609 return Chain;
1610}
1611
1612SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1613 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1614 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1615 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1616 if (!callingConvSupported(CallConv))
1617 fail(DL, DAG, Msg: "WebAssembly doesn't support non-C calling conventions");
1618
1619 MachineFunction &MF = DAG.getMachineFunction();
1620 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1621
1622 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1623 // of the incoming values before they're represented by virtual registers.
1624 MF.getRegInfo().addLiveIn(Reg: WebAssembly::ARGUMENTS);
1625
1626 bool HasSwiftErrorArg = false;
1627 bool HasSwiftSelfArg = false;
1628 bool HasSwiftAsyncArg = false;
1629 for (const ISD::InputArg &In : Ins) {
1630 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1631 HasSwiftErrorArg |= In.Flags.isSwiftError();
1632 HasSwiftAsyncArg |= In.Flags.isSwiftAsync();
1633 if (In.Flags.isInAlloca())
1634 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca arguments");
1635 if (In.Flags.isNest())
1636 fail(DL, DAG, Msg: "WebAssembly hasn't implemented nest arguments");
1637 if (In.Flags.isInConsecutiveRegs())
1638 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs arguments");
1639 if (In.Flags.isInConsecutiveRegsLast())
1640 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last arguments");
1641 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1642 // registers.
1643 InVals.push_back(Elt: In.Used ? DAG.getNode(Opcode: WebAssemblyISD::ARGUMENT, DL, VT: In.VT,
1644 Operand: DAG.getTargetConstant(Val: InVals.size(),
1645 DL, VT: MVT::i32))
1646 : DAG.getUNDEF(VT: In.VT));
1647
1648 // Record the number and types of arguments.
1649 MFI->addParam(VT: In.VT);
1650 }
1651
1652 // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
1653 // (for swifttailcc) swiftasync arguments if there aren't. These additional
1654 // arguments are also added for callee signature. They are necessary to match
1655 // callee and caller signature for indirect call.
1656 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
1657 if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
1658 if (!HasSwiftSelfArg) {
1659 MFI->addParam(VT: PtrVT);
1660 }
1661 if (!HasSwiftErrorArg) {
1662 MFI->addParam(VT: PtrVT);
1663 }
1664 if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
1665 MFI->addParam(VT: PtrVT);
1666 }
1667 }
1668 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1669 // the buffer is passed as an argument.
1670 if (IsVarArg) {
1671 MVT PtrVT = getPointerTy(DL: MF.getDataLayout());
1672 Register VarargVreg =
1673 MF.getRegInfo().createVirtualRegister(RegClass: getRegClassFor(VT: PtrVT));
1674 MFI->setVarargBufferVreg(VarargVreg);
1675 Chain = DAG.getCopyToReg(
1676 Chain, dl: DL, Reg: VarargVreg,
1677 N: DAG.getNode(Opcode: WebAssemblyISD::ARGUMENT, DL, VT: PtrVT,
1678 Operand: DAG.getTargetConstant(Val: Ins.size(), DL, VT: MVT::i32)));
1679 MFI->addParam(VT: PtrVT);
1680 }
1681
1682 // Record the number and types of arguments and results.
1683 SmallVector<MVT, 4> Params;
1684 SmallVector<MVT, 4> Results;
1685 computeSignatureVTs(Ty: MF.getFunction().getFunctionType(), TargetFunc: &MF.getFunction(),
1686 ContextFunc: MF.getFunction(), TM: DAG.getTarget(), Params, Results);
1687 for (MVT VT : Results)
1688 MFI->addResult(VT);
1689 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1690 // the param logic here with ComputeSignatureVTs
1691 assert(MFI->getParams().size() == Params.size() &&
1692 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1693 Params.begin()));
1694
1695 return Chain;
1696}
1697
1698void WebAssemblyTargetLowering::ReplaceNodeResults(
1699 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1700 switch (N->getOpcode()) {
1701 case ISD::SIGN_EXTEND_INREG:
1702 // Do not add any results, signifying that N should not be custom lowered
1703 // after all. This happens because simd128 turns on custom lowering for
1704 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1705 // illegal type.
1706 break;
1707 case ISD::ANY_EXTEND_VECTOR_INREG:
1708 case ISD::SIGN_EXTEND_VECTOR_INREG:
1709 case ISD::ZERO_EXTEND_VECTOR_INREG:
1710 // Do not add any results, signifying that N should not be custom lowered.
1711 // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
1712 break;
1713 case ISD::ADD:
1714 case ISD::SUB:
1715 Results.push_back(Elt: Replace128Op(N, DAG));
1716 break;
1717 default:
1718 llvm_unreachable(
1719 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1720 }
1721}
1722
1723//===----------------------------------------------------------------------===//
1724// Custom lowering hooks.
1725//===----------------------------------------------------------------------===//
1726
1727SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1728 SelectionDAG &DAG) const {
1729 SDLoc DL(Op);
1730 switch (Op.getOpcode()) {
1731 default:
1732 llvm_unreachable("unimplemented operation lowering");
1733 return SDValue();
1734 case ISD::FrameIndex:
1735 return LowerFrameIndex(Op, DAG);
1736 case ISD::GlobalAddress:
1737 return LowerGlobalAddress(Op, DAG);
1738 case ISD::GlobalTLSAddress:
1739 return LowerGlobalTLSAddress(Op, DAG);
1740 case ISD::ExternalSymbol:
1741 return LowerExternalSymbol(Op, DAG);
1742 case ISD::JumpTable:
1743 return LowerJumpTable(Op, DAG);
1744 case ISD::BR_JT:
1745 return LowerBR_JT(Op, DAG);
1746 case ISD::VASTART:
1747 return LowerVASTART(Op, DAG);
1748 case ISD::BlockAddress:
1749 case ISD::BRIND:
1750 fail(DL, DAG, Msg: "WebAssembly hasn't implemented computed gotos");
1751 return SDValue();
1752 case ISD::RETURNADDR:
1753 return LowerRETURNADDR(Op, DAG);
1754 case ISD::FRAMEADDR:
1755 return LowerFRAMEADDR(Op, DAG);
1756 case ISD::CopyToReg:
1757 return LowerCopyToReg(Op, DAG);
1758 case ISD::EXTRACT_VECTOR_ELT:
1759 case ISD::INSERT_VECTOR_ELT:
1760 return LowerAccessVectorElement(Op, DAG);
1761 case ISD::INTRINSIC_VOID:
1762 case ISD::INTRINSIC_WO_CHAIN:
1763 case ISD::INTRINSIC_W_CHAIN:
1764 return LowerIntrinsic(Op, DAG);
1765 case ISD::SIGN_EXTEND_INREG:
1766 return LowerSIGN_EXTEND_INREG(Op, DAG);
1767 case ISD::ZERO_EXTEND_VECTOR_INREG:
1768 case ISD::SIGN_EXTEND_VECTOR_INREG:
1769 case ISD::ANY_EXTEND_VECTOR_INREG:
1770 return LowerEXTEND_VECTOR_INREG(Op, DAG);
1771 case ISD::BUILD_VECTOR:
1772 return LowerBUILD_VECTOR(Op, DAG);
1773 case ISD::VECTOR_SHUFFLE:
1774 return LowerVECTOR_SHUFFLE(Op, DAG);
1775 case ISD::SETCC:
1776 return LowerSETCC(Op, DAG);
1777 case ISD::SHL:
1778 case ISD::SRA:
1779 case ISD::SRL:
1780 return LowerShift(Op, DAG);
1781 case ISD::FP_TO_SINT_SAT:
1782 case ISD::FP_TO_UINT_SAT:
1783 return LowerFP_TO_INT_SAT(Op, DAG);
1784 case ISD::FMINNUM:
1785 case ISD::FMINIMUMNUM:
1786 return LowerFMIN(Op, DAG);
1787 case ISD::FMAXNUM:
1788 case ISD::FMAXIMUMNUM:
1789 return LowerFMAX(Op, DAG);
1790 case ISD::LOAD:
1791 return LowerLoad(Op, DAG);
1792 case ISD::STORE:
1793 return LowerStore(Op, DAG);
1794 case ISD::CTPOP:
1795 case ISD::CTLZ:
1796 case ISD::CTTZ:
1797 return DAG.UnrollVectorOp(N: Op.getNode());
1798 case ISD::CLEAR_CACHE:
1799 report_fatal_error(reason: "llvm.clear_cache is not supported on wasm");
1800 case ISD::SMUL_LOHI:
1801 case ISD::UMUL_LOHI:
1802 return LowerMUL_LOHI(Op, DAG);
1803 case ISD::UADDO:
1804 return LowerUADDO(Op, DAG);
1805 }
1806}
1807
1808static bool IsWebAssemblyGlobal(SDValue Op) {
1809 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op))
1810 return WebAssembly::isWasmVarAddressSpace(AS: GA->getAddressSpace());
1811
1812 return false;
1813}
1814
1815static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
1816 SelectionDAG &DAG) {
1817 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op);
1818 if (!FI)
1819 return std::nullopt;
1820
1821 auto &MF = DAG.getMachineFunction();
1822 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FrameIndex: FI->getIndex());
1823}
1824
1825SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1826 SelectionDAG &DAG) const {
1827 SDLoc DL(Op);
1828 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
1829 const SDValue &Value = SN->getValue();
1830 const SDValue &Base = SN->getBasePtr();
1831 const SDValue &Offset = SN->getOffset();
1832
1833 if (IsWebAssemblyGlobal(Op: Base)) {
1834 if (!Offset->isUndef())
1835 report_fatal_error(reason: "unexpected offset when storing to webassembly global",
1836 gen_crash_diag: false);
1837
1838 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
1839 SDValue Ops[] = {SN->getChain(), Value, Base};
1840 return DAG.getMemIntrinsicNode(Opcode: WebAssemblyISD::GLOBAL_SET, dl: DL, VTList: Tys, Ops,
1841 MemVT: SN->getMemoryVT(), MMO: SN->getMemOperand());
1842 }
1843
1844 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Op: Base, DAG)) {
1845 if (!Offset->isUndef())
1846 report_fatal_error(reason: "unexpected offset when storing to webassembly local",
1847 gen_crash_diag: false);
1848
1849 SDValue Idx = DAG.getTargetConstant(Val: *Local, DL: Base, VT: MVT::i32);
1850 SDVTList Tys = DAG.getVTList(VT: MVT::Other); // The chain.
1851 SDValue Ops[] = {SN->getChain(), Idx, Value};
1852 return DAG.getNode(Opcode: WebAssemblyISD::LOCAL_SET, DL, VTList: Tys, Ops);
1853 }
1854
1855 if (WebAssembly::isWasmVarAddressSpace(AS: SN->getAddressSpace()))
1856 report_fatal_error(
1857 reason: "Encountered an unlowerable store to the wasm_var address space",
1858 gen_crash_diag: false);
1859
1860 return Op;
1861}
1862
1863SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1864 SelectionDAG &DAG) const {
1865 SDLoc DL(Op);
1866 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
1867 const SDValue &Base = LN->getBasePtr();
1868 const SDValue &Offset = LN->getOffset();
1869
1870 if (IsWebAssemblyGlobal(Op: Base)) {
1871 if (!Offset->isUndef())
1872 report_fatal_error(
1873 reason: "unexpected offset when loading from webassembly global", gen_crash_diag: false);
1874
1875 SDVTList Tys = DAG.getVTList(VT1: LN->getValueType(ResNo: 0), VT2: MVT::Other);
1876 SDValue Ops[] = {LN->getChain(), Base};
1877 return DAG.getMemIntrinsicNode(Opcode: WebAssemblyISD::GLOBAL_GET, dl: DL, VTList: Tys, Ops,
1878 MemVT: LN->getMemoryVT(), MMO: LN->getMemOperand());
1879 }
1880
1881 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Op: Base, DAG)) {
1882 if (!Offset->isUndef())
1883 report_fatal_error(
1884 reason: "unexpected offset when loading from webassembly local", gen_crash_diag: false);
1885
1886 SDValue Idx = DAG.getTargetConstant(Val: *Local, DL: Base, VT: MVT::i32);
1887 EVT LocalVT = LN->getValueType(ResNo: 0);
1888 return DAG.getNode(Opcode: WebAssemblyISD::LOCAL_GET, DL, ResultTys: {LocalVT, MVT::Other},
1889 Ops: {LN->getChain(), Idx});
1890 }
1891
1892 if (WebAssembly::isWasmVarAddressSpace(AS: LN->getAddressSpace()))
1893 report_fatal_error(
1894 reason: "Encountered an unlowerable load from the wasm_var address space",
1895 gen_crash_diag: false);
1896
1897 return Op;
1898}
1899
1900SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
1901 SelectionDAG &DAG) const {
1902 assert(Subtarget->hasWideArithmetic());
1903 assert(Op.getValueType() == MVT::i64);
1904 SDLoc DL(Op);
1905 unsigned Opcode;
1906 switch (Op.getOpcode()) {
1907 case ISD::UMUL_LOHI:
1908 Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
1909 break;
1910 case ISD::SMUL_LOHI:
1911 Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
1912 break;
1913 default:
1914 llvm_unreachable("unexpected opcode");
1915 }
1916 SDValue LHS = Op.getOperand(i: 0);
1917 SDValue RHS = Op.getOperand(i: 1);
1918 SDValue Lo =
1919 DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64), N1: LHS, N2: RHS);
1920 SDValue Hi(Lo.getNode(), 1);
1921 SDValue Ops[] = {Lo, Hi};
1922 return DAG.getMergeValues(Ops, dl: DL);
1923}
1924
1925// Lowers `UADDO` intrinsics to an `i64.add128` instruction when it's enabled.
1926//
1927// This enables generating a single wasm instruction for this operation where
1928// the upper half of both operands are constant zeros. The upper half of the
1929// result is then whether the overflow happened.
1930SDValue WebAssemblyTargetLowering::LowerUADDO(SDValue Op,
1931 SelectionDAG &DAG) const {
1932 assert(Subtarget->hasWideArithmetic());
1933 assert(Op.getValueType() == MVT::i64);
1934 assert(Op.getOpcode() == ISD::UADDO);
1935 SDLoc DL(Op);
1936 SDValue LHS = Op.getOperand(i: 0);
1937 SDValue RHS = Op.getOperand(i: 1);
1938 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
1939 SDValue Result =
1940 DAG.getNode(Opcode: WebAssemblyISD::I64_ADD128, DL,
1941 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64), N1: LHS, N2: Zero, N3: RHS, N4: Zero);
1942 SDValue CarryI64(Result.getNode(), 1);
1943 SDValue CarryI32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: CarryI64);
1944 SDValue Ops[] = {Result, CarryI32};
1945 return DAG.getMergeValues(Ops, dl: DL);
1946}
1947
1948SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
1949 SelectionDAG &DAG) const {
1950 assert(Subtarget->hasWideArithmetic());
1951 assert(N->getValueType(0) == MVT::i128);
1952 SDLoc DL(N);
1953 unsigned Opcode;
1954 switch (N->getOpcode()) {
1955 case ISD::ADD:
1956 Opcode = WebAssemblyISD::I64_ADD128;
1957 break;
1958 case ISD::SUB:
1959 Opcode = WebAssemblyISD::I64_SUB128;
1960 break;
1961 default:
1962 llvm_unreachable("unexpected opcode");
1963 }
1964 SDValue LHS = N->getOperand(Num: 0);
1965 SDValue RHS = N->getOperand(Num: 1);
1966
1967 SDValue C0 = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
1968 SDValue C1 = DAG.getConstant(Val: 1, DL, VT: MVT::i64);
1969 SDValue LHS_0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: LHS, N2: C0);
1970 SDValue LHS_1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: LHS, N2: C1);
1971 SDValue RHS_0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: RHS, N2: C0);
1972 SDValue RHS_1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: RHS, N2: C1);
1973 SDValue Result_LO = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64),
1974 N1: LHS_0, N2: LHS_1, N3: RHS_0, N4: RHS_1);
1975 SDValue Result_HI(Result_LO.getNode(), 1);
1976 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VTList: N->getVTList(), N1: Result_LO, N2: Result_HI);
1977}
1978
1979SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1980 SelectionDAG &DAG) const {
1981 SDValue Src = Op.getOperand(i: 2);
1982 if (isa<FrameIndexSDNode>(Val: Src.getNode())) {
1983 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1984 // the FI to some LEA-like instruction, but since we don't have that, we
1985 // need to insert some kind of instruction that can take an FI operand and
1986 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1987 // local.copy between Op and its FI operand.
1988 SDValue Chain = Op.getOperand(i: 0);
1989 SDLoc DL(Op);
1990 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i: 1))->getReg();
1991 EVT VT = Src.getValueType();
1992 SDValue Copy(DAG.getMachineNode(Opcode: VT == MVT::i32 ? WebAssembly::COPY_I32
1993 : WebAssembly::COPY_I64,
1994 dl: DL, VT, Op1: Src),
1995 0);
1996 return Op.getNode()->getNumValues() == 1
1997 ? DAG.getCopyToReg(Chain, dl: DL, Reg, N: Copy)
1998 : DAG.getCopyToReg(Chain, dl: DL, Reg, N: Copy,
1999 Glue: Op.getNumOperands() == 4 ? Op.getOperand(i: 3)
2000 : SDValue());
2001 }
2002 return SDValue();
2003}
2004
2005SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
2006 SelectionDAG &DAG) const {
2007 int FI = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
2008 return DAG.getTargetFrameIndex(FI, VT: Op.getValueType());
2009}
2010
2011SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
2012 SelectionDAG &DAG) const {
2013 SDLoc DL(Op);
2014
2015 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
2016 fail(DL, DAG,
2017 Msg: "Non-Emscripten WebAssembly hasn't implemented "
2018 "__builtin_return_address");
2019 return SDValue();
2020 }
2021
2022 unsigned Depth = Op.getConstantOperandVal(i: 0);
2023 MakeLibCallOptions CallOptions;
2024 return makeLibCall(DAG, LC: RTLIB::RETURN_ADDRESS, RetVT: Op.getValueType(),
2025 Ops: {DAG.getConstant(Val: Depth, DL, VT: MVT::i32)}, CallOptions, dl: DL)
2026 .first;
2027}
2028
2029SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
2030 SelectionDAG &DAG) const {
2031 // Non-zero depths are not supported by WebAssembly currently. Use the
2032 // legalizer's default expansion, which is to return 0 (what this function is
2033 // documented to do).
2034 if (Op.getConstantOperandVal(i: 0) > 0)
2035 return SDValue();
2036
2037 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
2038 EVT VT = Op.getValueType();
2039 Register FP =
2040 Subtarget->getRegisterInfo()->getFrameRegister(MF: DAG.getMachineFunction());
2041 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg: FP, VT);
2042}
2043
2044SDValue
2045WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2046 SelectionDAG &DAG) const {
2047 SDLoc DL(Op);
2048 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
2049
2050 MachineFunction &MF = DAG.getMachineFunction();
2051 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
2052 report_fatal_error(reason: "cannot use thread-local storage without bulk memory",
2053 gen_crash_diag: false);
2054
2055 const GlobalValue *GV = GA->getGlobal();
2056
2057 // Currently only Emscripten supports dynamic linking with threads. Therefore,
2058 // on other targets, if we have thread-local storage, only the local-exec
2059 // model is possible.
2060 auto model = Subtarget->getTargetTriple().isOSEmscripten()
2061 ? GV->getThreadLocalMode()
2062 : GlobalValue::LocalExecTLSModel;
2063
2064 // Unsupported TLS modes
2065 assert(model != GlobalValue::NotThreadLocal);
2066 assert(model != GlobalValue::InitialExecTLSModel);
2067
2068 if (model == GlobalValue::LocalExecTLSModel ||
2069 model == GlobalValue::LocalDynamicTLSModel ||
2070 (model == GlobalValue::GeneralDynamicTLSModel &&
2071 getTargetMachine().shouldAssumeDSOLocal(GV))) {
2072 // For DSO-local TLS variables we use offset from __tls_base
2073
2074 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2075 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
2076 : WebAssembly::GLOBAL_GET_I32;
2077 const char *BaseName = MF.createExternalSymbolName(Name: "__tls_base");
2078
2079 SDValue BaseAddr(
2080 DAG.getMachineNode(Opcode: GlobalGet, dl: DL, VT: PtrVT,
2081 Op1: DAG.getTargetExternalSymbol(Sym: BaseName, VT: PtrVT)),
2082 0);
2083
2084 SDValue TLSOffset = DAG.getTargetGlobalAddress(
2085 GV, DL, VT: PtrVT, offset: GA->getOffset(), TargetFlags: WebAssemblyII::MO_TLS_BASE_REL);
2086 SDValue SymOffset =
2087 DAG.getNode(Opcode: WebAssemblyISD::WrapperREL, DL, VT: PtrVT, Operand: TLSOffset);
2088
2089 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BaseAddr, N2: SymOffset);
2090 }
2091
2092 assert(model == GlobalValue::GeneralDynamicTLSModel);
2093
2094 EVT VT = Op.getValueType();
2095 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2096 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT,
2097 offset: GA->getOffset(),
2098 TargetFlags: WebAssemblyII::MO_GOT_TLS));
2099}
2100
2101SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
2102 SelectionDAG &DAG) const {
2103 SDLoc DL(Op);
2104 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
2105 EVT VT = Op.getValueType();
2106 assert(GA->getTargetFlags() == 0 &&
2107 "Unexpected target flags on generic GlobalAddressSDNode");
2108 if (!WebAssembly::isValidAddressSpace(AS: GA->getAddressSpace()))
2109 fail(DL, DAG, Msg: "Invalid address space for WebAssembly target");
2110
2111 unsigned OperandFlags = 0;
2112 const GlobalValue *GV = GA->getGlobal();
2113 // Since WebAssembly tables cannot yet be shared accross modules, we don't
2114 // need special treatment for tables in PIC mode.
2115 if (isPositionIndependent() &&
2116 !WebAssembly::isWebAssemblyTableType(Ty: GV->getValueType())) {
2117 if (getTargetMachine().shouldAssumeDSOLocal(GV)) {
2118 MachineFunction &MF = DAG.getMachineFunction();
2119 MVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2120 const char *BaseName;
2121 if (GV->getValueType()->isFunctionTy()) {
2122 BaseName = MF.createExternalSymbolName(Name: "__table_base");
2123 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
2124 } else {
2125 BaseName = MF.createExternalSymbolName(Name: "__memory_base");
2126 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
2127 }
2128 SDValue BaseAddr =
2129 DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT,
2130 Operand: DAG.getTargetExternalSymbol(Sym: BaseName, VT: PtrVT));
2131
2132 SDValue SymAddr = DAG.getNode(
2133 Opcode: WebAssemblyISD::WrapperREL, DL, VT,
2134 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT, offset: GA->getOffset(),
2135 TargetFlags: OperandFlags));
2136
2137 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BaseAddr, N2: SymAddr);
2138 }
2139 OperandFlags = WebAssemblyII::MO_GOT;
2140 }
2141
2142 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2143 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT,
2144 offset: GA->getOffset(), TargetFlags: OperandFlags));
2145}
2146
2147SDValue
2148WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
2149 SelectionDAG &DAG) const {
2150 SDLoc DL(Op);
2151 const auto *ES = cast<ExternalSymbolSDNode>(Val&: Op);
2152 EVT VT = Op.getValueType();
2153 assert(ES->getTargetFlags() == 0 &&
2154 "Unexpected target flags on generic ExternalSymbolSDNode");
2155 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2156 Operand: DAG.getTargetExternalSymbol(Sym: ES->getSymbol(), VT));
2157}
2158
2159SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
2160 SelectionDAG &DAG) const {
2161 // There's no need for a Wrapper node because we always incorporate a jump
2162 // table operand into a BR_TABLE instruction, rather than ever
2163 // materializing it in a register.
2164 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
2165 return DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Op.getValueType(),
2166 TargetFlags: JT->getTargetFlags());
2167}
2168
2169SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
2170 SelectionDAG &DAG) const {
2171 SDLoc DL(Op);
2172 SDValue Chain = Op.getOperand(i: 0);
2173 const auto *JT = cast<JumpTableSDNode>(Val: Op.getOperand(i: 1));
2174 SDValue Index = Op.getOperand(i: 2);
2175 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
2176
2177 SmallVector<SDValue, 8> Ops;
2178 Ops.push_back(Elt: Chain);
2179 Ops.push_back(Elt: Index);
2180
2181 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
2182 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
2183
2184 // Add an operand for each case.
2185 for (auto *MBB : MBBs)
2186 Ops.push_back(Elt: DAG.getBasicBlock(MBB));
2187
2188 // Add the first MBB as a dummy default target for now. This will be replaced
2189 // with the proper default target (and the preceding range check eliminated)
2190 // if possible by WebAssemblyFixBrTableDefaults.
2191 Ops.push_back(Elt: DAG.getBasicBlock(MBB: *MBBs.begin()));
2192 return DAG.getNode(Opcode: WebAssemblyISD::BR_TABLE, DL, VT: MVT::Other, Ops);
2193}
2194
2195SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
2196 SelectionDAG &DAG) const {
2197 SDLoc DL(Op);
2198 EVT PtrVT = getPointerTy(DL: DAG.getMachineFunction().getDataLayout());
2199
2200 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
2201 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
2202
2203 SDValue ArgN = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
2204 Reg: MFI->getVarargBufferVreg(), VT: PtrVT);
2205 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: ArgN, Ptr: Op.getOperand(i: 1),
2206 PtrInfo: MachinePointerInfo(SV));
2207}
2208
2209SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
2210 SelectionDAG &DAG) const {
2211 MachineFunction &MF = DAG.getMachineFunction();
2212 unsigned IntNo;
2213 switch (Op.getOpcode()) {
2214 case ISD::INTRINSIC_VOID:
2215 case ISD::INTRINSIC_W_CHAIN:
2216 IntNo = Op.getConstantOperandVal(i: 1);
2217 break;
2218 case ISD::INTRINSIC_WO_CHAIN:
2219 IntNo = Op.getConstantOperandVal(i: 0);
2220 break;
2221 default:
2222 llvm_unreachable("Invalid intrinsic");
2223 }
2224 SDLoc DL(Op);
2225
2226 switch (IntNo) {
2227 default:
2228 return SDValue(); // Don't custom lower most intrinsics.
2229
2230 case Intrinsic::wasm_lsda: {
2231 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
2232 const char *SymName = MF.createExternalSymbolName(
2233 Name: "GCC_except_table" + std::to_string(val: MF.getFunctionNumber()));
2234 if (isPositionIndependent()) {
2235 SDValue Node = DAG.getTargetExternalSymbol(
2236 Sym: SymName, VT: PtrVT, TargetFlags: WebAssemblyII::MO_MEMORY_BASE_REL);
2237 const char *BaseName = MF.createExternalSymbolName(Name: "__memory_base");
2238 SDValue BaseAddr =
2239 DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT,
2240 Operand: DAG.getTargetExternalSymbol(Sym: BaseName, VT: PtrVT));
2241 SDValue SymAddr =
2242 DAG.getNode(Opcode: WebAssemblyISD::WrapperREL, DL, VT: PtrVT, Operand: Node);
2243 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BaseAddr, N2: SymAddr);
2244 }
2245 SDValue Node = DAG.getTargetExternalSymbol(Sym: SymName, VT: PtrVT);
2246 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT, Operand: Node);
2247 }
2248
2249 case Intrinsic::wasm_shuffle: {
2250 // Drop in-chain and replace undefs, but otherwise pass through unchanged
2251 SDValue Ops[18];
2252 size_t OpIdx = 0;
2253 Ops[OpIdx++] = Op.getOperand(i: 1);
2254 Ops[OpIdx++] = Op.getOperand(i: 2);
2255 while (OpIdx < 18) {
2256 const SDValue &MaskIdx = Op.getOperand(i: OpIdx + 1);
2257 if (MaskIdx.isUndef() || MaskIdx.getNode()->getAsZExtVal() >= 32) {
2258 bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;
2259 Ops[OpIdx++] = DAG.getConstant(Val: 0, DL, VT: MVT::i32, isTarget);
2260 } else {
2261 Ops[OpIdx++] = MaskIdx;
2262 }
2263 }
2264 return DAG.getNode(Opcode: WebAssemblyISD::SHUFFLE, DL, VT: Op.getValueType(), Ops);
2265 }
2266
2267 case Intrinsic::thread_pointer: {
2268 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2269 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
2270 : WebAssembly::GLOBAL_GET_I32;
2271 const char *TlsBase = MF.createExternalSymbolName(Name: "__tls_base");
2272 return SDValue(
2273 DAG.getMachineNode(Opcode: GlobalGet, dl: DL, VT: PtrVT,
2274 Op1: DAG.getTargetExternalSymbol(Sym: TlsBase, VT: PtrVT)),
2275 0);
2276 }
2277 }
2278}
2279
2280SDValue
2281WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2282 SelectionDAG &DAG) const {
2283 SDLoc DL(Op);
2284 // If sign extension operations are disabled, allow sext_inreg only if operand
2285 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
2286 // extension operations, but allowing sext_inreg in this context lets us have
2287 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
2288 // everywhere would be simpler in this file, but would necessitate large and
2289 // brittle patterns to undo the expansion and select extract_lane_s
2290 // instructions.
2291 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
2292 if (Op.getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2293 return SDValue();
2294
2295 const SDValue &Extract = Op.getOperand(i: 0);
2296 MVT VecT = Extract.getOperand(i: 0).getSimpleValueType();
2297 if (VecT.getVectorElementType().getSizeInBits() > 32)
2298 return SDValue();
2299 MVT ExtractedLaneT =
2300 cast<VTSDNode>(Val: Op.getOperand(i: 1).getNode())->getVT().getSimpleVT();
2301 MVT ExtractedVecT =
2302 MVT::getVectorVT(VT: ExtractedLaneT, NumElements: 128 / ExtractedLaneT.getSizeInBits());
2303 if (ExtractedVecT == VecT)
2304 return Op;
2305
2306 // Bitcast vector to appropriate type to ensure ISel pattern coverage
2307 const SDNode *Index = Extract.getOperand(i: 1).getNode();
2308 if (!isa<ConstantSDNode>(Val: Index))
2309 return SDValue();
2310 unsigned IndexVal = Index->getAsZExtVal();
2311 unsigned Scale =
2312 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
2313 assert(Scale > 1);
2314 SDValue NewIndex =
2315 DAG.getConstant(Val: IndexVal * Scale, DL, VT: Index->getValueType(ResNo: 0));
2316 SDValue NewExtract = DAG.getNode(
2317 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: Extract.getValueType(),
2318 N1: DAG.getBitcast(VT: ExtractedVecT, V: Extract.getOperand(i: 0)), N2: NewIndex);
2319 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: Op.getValueType(), N1: NewExtract,
2320 N2: Op.getOperand(i: 1));
2321}
2322
2323static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT,
2324 SelectionDAG &DAG) {
2325 SDValue Source = peekThroughBitcasts(V: Op);
2326 if (Source.getOpcode() != ISD::VECTOR_SHUFFLE)
2327 return SDValue();
2328
2329 assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||
2330 UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&
2331 "expected extend_low");
2332 auto *Shuffle = cast<ShuffleVectorSDNode>(Val: Source.getNode());
2333
2334 ArrayRef<int> Mask = Shuffle->getMask();
2335 // Look for a shuffle which moves from the high half to the low half.
2336 size_t FirstIdx = Mask.size() / 2;
2337 for (size_t i = 0; i < Mask.size() / 2; ++i) {
2338 if (Mask[i] != static_cast<int>(FirstIdx + i)) {
2339 return SDValue();
2340 }
2341 }
2342
2343 SDLoc DL(Op);
2344 unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S
2345 ? WebAssemblyISD::EXTEND_HIGH_S
2346 : WebAssemblyISD::EXTEND_HIGH_U;
2347 SDValue ShuffleSrc = Shuffle->getOperand(Num: 0);
2348 if (Op.getOpcode() == ISD::BITCAST)
2349 ShuffleSrc = DAG.getBitcast(VT: Op.getValueType(), V: ShuffleSrc);
2350
2351 return DAG.getNode(Opcode: Opc, DL, VT, Operand: ShuffleSrc);
2352}
2353
2354SDValue
2355WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,
2356 SelectionDAG &DAG) const {
2357 SDLoc DL(Op);
2358 EVT VT = Op.getValueType();
2359 SDValue Src = Op.getOperand(i: 0);
2360 EVT SrcVT = Src.getValueType();
2361
2362 if (SrcVT.getVectorElementType() == MVT::i1 ||
2363 SrcVT.getVectorElementType() == MVT::i64)
2364 return SDValue();
2365
2366 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
2367 "Unexpected extension factor.");
2368 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
2369
2370 if (Scale != 2 && Scale != 4 && Scale != 8)
2371 return SDValue();
2372
2373 unsigned Ext;
2374 switch (Op.getOpcode()) {
2375 default:
2376 llvm_unreachable("unexpected opcode");
2377 case ISD::ANY_EXTEND_VECTOR_INREG:
2378 case ISD::ZERO_EXTEND_VECTOR_INREG:
2379 Ext = WebAssemblyISD::EXTEND_LOW_U;
2380 break;
2381 case ISD::SIGN_EXTEND_VECTOR_INREG:
2382 Ext = WebAssemblyISD::EXTEND_LOW_S;
2383 break;
2384 }
2385
2386 if (Scale == 2) {
2387 // See if we can use EXTEND_HIGH.
2388 if (auto ExtendHigh = GetExtendHigh(Op: Op.getOperand(i: 0), UserOpc: Ext, VT, DAG))
2389 return ExtendHigh;
2390 }
2391
2392 SDValue Ret = Src;
2393 while (Scale != 1) {
2394 Ret = DAG.getNode(Opcode: Ext, DL,
2395 VT: Ret.getValueType()
2396 .widenIntegerVectorElementType(Context&: *DAG.getContext())
2397 .getHalfNumVectorElementsVT(Context&: *DAG.getContext()),
2398 Operand: Ret);
2399 Scale /= 2;
2400 }
2401 assert(Ret.getValueType() == VT);
2402 return Ret;
2403}
2404
2405static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
2406 SDLoc DL(Op);
2407 if (Op.getValueType() != MVT::v2f64)
2408 return SDValue();
2409
2410 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
2411 unsigned &Index) -> bool {
2412 switch (Op.getOpcode()) {
2413 case ISD::SINT_TO_FP:
2414 Opcode = WebAssemblyISD::CONVERT_LOW_S;
2415 break;
2416 case ISD::UINT_TO_FP:
2417 Opcode = WebAssemblyISD::CONVERT_LOW_U;
2418 break;
2419 case ISD::FP_EXTEND:
2420 Opcode = WebAssemblyISD::PROMOTE_LOW;
2421 break;
2422 default:
2423 return false;
2424 }
2425
2426 auto ExtractVector = Op.getOperand(i: 0);
2427 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2428 return false;
2429
2430 if (!isa<ConstantSDNode>(Val: ExtractVector.getOperand(i: 1).getNode()))
2431 return false;
2432
2433 SrcVec = ExtractVector.getOperand(i: 0);
2434 Index = ExtractVector.getConstantOperandVal(i: 1);
2435 return true;
2436 };
2437
2438 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
2439 SDValue LHSSrcVec, RHSSrcVec;
2440 if (!GetConvertedLane(Op.getOperand(i: 0), LHSOpcode, LHSSrcVec, LHSIndex) ||
2441 !GetConvertedLane(Op.getOperand(i: 1), RHSOpcode, RHSSrcVec, RHSIndex))
2442 return SDValue();
2443
2444 if (LHSOpcode != RHSOpcode)
2445 return SDValue();
2446
2447 MVT ExpectedSrcVT;
2448 switch (LHSOpcode) {
2449 case WebAssemblyISD::CONVERT_LOW_S:
2450 case WebAssemblyISD::CONVERT_LOW_U:
2451 ExpectedSrcVT = MVT::v4i32;
2452 break;
2453 case WebAssemblyISD::PROMOTE_LOW:
2454 ExpectedSrcVT = MVT::v4f32;
2455 break;
2456 }
2457 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
2458 return SDValue();
2459
2460 auto Src = LHSSrcVec;
2461 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
2462 // Shuffle the source vector so that the converted lanes are the low lanes.
2463 Src = DAG.getVectorShuffle(
2464 VT: ExpectedSrcVT, dl: DL, N1: LHSSrcVec, N2: RHSSrcVec,
2465 Mask: {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
2466 }
2467 return DAG.getNode(Opcode: LHSOpcode, DL, VT: MVT::v2f64, Operand: Src);
2468}
2469
2470SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
2471 SelectionDAG &DAG) const {
2472 MVT VT = Op.getSimpleValueType();
2473 if (VT == MVT::v8f16) {
2474 // BUILD_VECTOR can't handle FP16 operands since Wasm doesn't have a scaler
2475 // FP16 type, so cast them to I16s.
2476 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
2477 SmallVector<SDValue, 8> NewOps;
2478 for (unsigned I = 0, E = Op.getNumOperands(); I < E; ++I)
2479 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: I)));
2480 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SDLoc(), VT: IVT, Ops: NewOps);
2481 return DAG.getBitcast(VT, V: Res);
2482 }
2483
2484 if (auto ConvertLow = LowerConvertLow(Op, DAG))
2485 return ConvertLow;
2486
2487 SDLoc DL(Op);
2488 const EVT VecT = Op.getValueType();
2489 const EVT LaneT = Op.getOperand(i: 0).getValueType();
2490 const size_t Lanes = Op.getNumOperands();
2491 bool CanSwizzle = VecT == MVT::v16i8;
2492
2493 // BUILD_VECTORs are lowered to the instruction that initializes the highest
2494 // possible number of lanes at once followed by a sequence of replace_lane
2495 // instructions to individually initialize any remaining lanes.
2496
2497 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
2498 // swizzled lanes should be given greater weight.
2499
2500 // TODO: Investigate looping rather than always extracting/replacing specific
2501 // lanes to fill gaps.
2502
2503 auto IsConstant = [](const SDValue &V) {
2504 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
2505 };
2506
2507 // Returns the source vector and index vector pair if they exist. Checks for:
2508 // (extract_vector_elt
2509 // $src,
2510 // (sign_extend_inreg (extract_vector_elt $indices, $i))
2511 // )
2512 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
2513 auto Bail = std::make_pair(x: SDValue(), y: SDValue());
2514 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2515 return Bail;
2516 const SDValue &SwizzleSrc = Lane->getOperand(Num: 0);
2517 const SDValue &IndexExt = Lane->getOperand(Num: 1);
2518 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
2519 return Bail;
2520 const SDValue &Index = IndexExt->getOperand(Num: 0);
2521 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2522 return Bail;
2523 const SDValue &SwizzleIndices = Index->getOperand(Num: 0);
2524 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
2525 SwizzleIndices.getValueType() != MVT::v16i8 ||
2526 Index->getOperand(Num: 1)->getOpcode() != ISD::Constant ||
2527 Index->getConstantOperandVal(Num: 1) != I)
2528 return Bail;
2529 return std::make_pair(x: SwizzleSrc, y: SwizzleIndices);
2530 };
2531
2532 // If the lane is extracted from another vector at a constant index, return
2533 // that vector. The source vector must not have more lanes than the dest
2534 // because the shufflevector indices are in terms of the destination lanes and
2535 // would not be able to address the smaller individual source lanes.
2536 auto GetShuffleSrc = [&](const SDValue &Lane) {
2537 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2538 return SDValue();
2539 if (!isa<ConstantSDNode>(Val: Lane->getOperand(Num: 1).getNode()))
2540 return SDValue();
2541 if (Lane->getOperand(Num: 0).getValueType().getVectorNumElements() >
2542 VecT.getVectorNumElements())
2543 return SDValue();
2544 return Lane->getOperand(Num: 0);
2545 };
2546
2547 using ValueEntry = std::pair<SDValue, size_t>;
2548 SmallVector<ValueEntry, 16> SplatValueCounts;
2549
2550 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2551 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2552
2553 using ShuffleEntry = std::pair<SDValue, size_t>;
2554 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2555
2556 auto AddCount = [](auto &Counts, const auto &Val) {
2557 auto CountIt =
2558 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2559 if (CountIt == Counts.end()) {
2560 Counts.emplace_back(Val, 1);
2561 } else {
2562 CountIt->second++;
2563 }
2564 };
2565
2566 auto GetMostCommon = [](auto &Counts) {
2567 auto CommonIt = llvm::max_element(Counts, llvm::less_second());
2568 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
2569 return *CommonIt;
2570 };
2571
2572 size_t NumConstantLanes = 0;
2573
2574 // Count eligible lanes for each type of vector creation op
2575 for (size_t I = 0; I < Lanes; ++I) {
2576 const SDValue &Lane = Op->getOperand(Num: I);
2577 if (Lane.isUndef())
2578 continue;
2579
2580 AddCount(SplatValueCounts, Lane);
2581
2582 if (IsConstant(Lane))
2583 NumConstantLanes++;
2584 if (auto ShuffleSrc = GetShuffleSrc(Lane))
2585 AddCount(ShuffleCounts, ShuffleSrc);
2586 if (CanSwizzle) {
2587 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2588 if (SwizzleSrcs.first)
2589 AddCount(SwizzleCounts, SwizzleSrcs);
2590 }
2591 }
2592
2593 SDValue SplatValue;
2594 size_t NumSplatLanes;
2595 std::tie(args&: SplatValue, args&: NumSplatLanes) = GetMostCommon(SplatValueCounts);
2596
2597 SDValue SwizzleSrc;
2598 SDValue SwizzleIndices;
2599 size_t NumSwizzleLanes = 0;
2600 if (SwizzleCounts.size())
2601 std::forward_as_tuple(args: std::tie(args&: SwizzleSrc, args&: SwizzleIndices),
2602 args&: NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2603
2604 // Shuffles can draw from up to two vectors, so find the two most common
2605 // sources.
2606 SDValue ShuffleSrc1, ShuffleSrc2;
2607 size_t NumShuffleLanes = 0;
2608 if (ShuffleCounts.size()) {
2609 std::tie(args&: ShuffleSrc1, args&: NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2610 llvm::erase_if(C&: ShuffleCounts,
2611 P: [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2612 }
2613 if (ShuffleCounts.size()) {
2614 size_t AdditionalShuffleLanes;
2615 std::tie(args&: ShuffleSrc2, args&: AdditionalShuffleLanes) =
2616 GetMostCommon(ShuffleCounts);
2617 NumShuffleLanes += AdditionalShuffleLanes;
2618 }
2619
2620 // Predicate returning true if the lane is properly initialized by the
2621 // original instruction
2622 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2623 SDValue Result;
2624 // Prefer swizzles over shuffles over vector consts over splats
2625 if (NumSwizzleLanes >= NumShuffleLanes &&
2626 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2627 Result = DAG.getNode(Opcode: WebAssemblyISD::SWIZZLE, DL, VT: VecT, N1: SwizzleSrc,
2628 N2: SwizzleIndices);
2629 auto Swizzled = std::make_pair(x&: SwizzleSrc, y&: SwizzleIndices);
2630 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2631 return Swizzled == GetSwizzleSrcs(I, Lane);
2632 };
2633 } else if (NumShuffleLanes >= NumConstantLanes &&
2634 NumShuffleLanes >= NumSplatLanes) {
2635 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2636 size_t DestLaneCount = VecT.getVectorNumElements();
2637 size_t Scale1 = 1;
2638 size_t Scale2 = 1;
2639 SDValue Src1 = ShuffleSrc1;
2640 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VT: VecT);
2641 if (Src1.getValueType() != VecT) {
2642 size_t LaneSize =
2643 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2644 assert(LaneSize > DestLaneSize);
2645 Scale1 = LaneSize / DestLaneSize;
2646 Src1 = DAG.getBitcast(VT: VecT, V: Src1);
2647 }
2648 if (Src2.getValueType() != VecT) {
2649 size_t LaneSize =
2650 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2651 assert(LaneSize > DestLaneSize);
2652 Scale2 = LaneSize / DestLaneSize;
2653 Src2 = DAG.getBitcast(VT: VecT, V: Src2);
2654 }
2655
2656 int Mask[16];
2657 assert(DestLaneCount <= 16);
2658 for (size_t I = 0; I < DestLaneCount; ++I) {
2659 const SDValue &Lane = Op->getOperand(Num: I);
2660 SDValue Src = GetShuffleSrc(Lane);
2661 if (Src == ShuffleSrc1) {
2662 Mask[I] = Lane->getConstantOperandVal(Num: 1) * Scale1;
2663 } else if (Src && Src == ShuffleSrc2) {
2664 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(Num: 1) * Scale2;
2665 } else {
2666 Mask[I] = -1;
2667 }
2668 }
2669 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2670 Result = DAG.getVectorShuffle(VT: VecT, dl: DL, N1: Src1, N2: Src2, Mask: MaskRef);
2671 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2672 auto Src = GetShuffleSrc(Lane);
2673 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2674 };
2675 } else if (NumConstantLanes >= NumSplatLanes) {
2676 SmallVector<SDValue, 16> ConstLanes;
2677 for (const SDValue &Lane : Op->op_values()) {
2678 if (IsConstant(Lane)) {
2679 // Values may need to be fixed so that they will sign extend to be
2680 // within the expected range during ISel. Check whether the value is in
2681 // bounds based on the lane bit width and if it is out of bounds, lop
2682 // off the extra bits.
2683 uint64_t LaneBits = 128 / Lanes;
2684 if (auto *Const = dyn_cast<ConstantSDNode>(Val: Lane.getNode())) {
2685 ConstLanes.push_back(Elt: DAG.getConstant(
2686 Val: Const->getAPIntValue().trunc(width: LaneBits).getZExtValue(),
2687 DL: SDLoc(Lane), VT: LaneT));
2688 } else {
2689 ConstLanes.push_back(Elt: Lane);
2690 }
2691 } else if (LaneT.isFloatingPoint()) {
2692 ConstLanes.push_back(Elt: DAG.getConstantFP(Val: 0, DL, VT: LaneT));
2693 } else {
2694 ConstLanes.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: LaneT));
2695 }
2696 }
2697 Result = DAG.getBuildVector(VT: VecT, DL, Ops: ConstLanes);
2698 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2699 return IsConstant(Lane);
2700 };
2701 } else {
2702 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits();
2703 if (NumSplatLanes == 1 && Op->getOperand(Num: 0) == SplatValue &&
2704 (DestLaneSize == 32 || DestLaneSize == 64)) {
2705 // Could be selected to load_zero.
2706 Result = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecT, Operand: SplatValue);
2707 } else {
2708 // Use a splat (which might be selected as a load splat)
2709 Result = DAG.getSplatBuildVector(VT: VecT, DL, Op: SplatValue);
2710 }
2711 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2712 return Lane == SplatValue;
2713 };
2714 }
2715
2716 assert(Result);
2717 assert(IsLaneConstructed);
2718
2719 // Add replace_lane instructions for any unhandled values
2720 for (size_t I = 0; I < Lanes; ++I) {
2721 const SDValue &Lane = Op->getOperand(Num: I);
2722 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2723 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: VecT, N1: Result, N2: Lane,
2724 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
2725 }
2726
2727 return Result;
2728}
2729
2730SDValue
2731WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2732 SelectionDAG &DAG) const {
2733 SDLoc DL(Op);
2734 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Op.getNode())->getMask();
2735 MVT VecType = Op.getOperand(i: 0).getSimpleValueType();
2736 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
2737 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2738
2739 // Space for two vector args and sixteen mask indices
2740 SDValue Ops[18];
2741 size_t OpIdx = 0;
2742 Ops[OpIdx++] = Op.getOperand(i: 0);
2743 Ops[OpIdx++] = Op.getOperand(i: 1);
2744
2745 // Expand mask indices to byte indices and materialize them as operands
2746 for (int M : Mask) {
2747 for (size_t J = 0; J < LaneBytes; ++J) {
2748 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2749 // whole lane of vector input, to allow further reduction at VM. E.g.
2750 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2751 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2752 Ops[OpIdx++] = DAG.getConstant(Val: ByteIndex, DL, VT: MVT::i32);
2753 }
2754 }
2755
2756 return DAG.getNode(Opcode: WebAssemblyISD::SHUFFLE, DL, VT: Op.getValueType(), Ops);
2757}
2758
2759SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2760 SelectionDAG &DAG) const {
2761 SDLoc DL(Op);
2762 // The legalizer does not know how to expand the unsupported comparison modes
2763 // of i64x2 vectors, so we manually unroll them here.
2764 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2765 SmallVector<SDValue, 2> LHS, RHS;
2766 DAG.ExtractVectorElements(Op: Op->getOperand(Num: 0), Args&: LHS);
2767 DAG.ExtractVectorElements(Op: Op->getOperand(Num: 1), Args&: RHS);
2768 const SDValue &CC = Op->getOperand(Num: 2);
2769 auto MakeLane = [&](unsigned I) {
2770 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: MVT::i64, N1: LHS[I], N2: RHS[I],
2771 N3: DAG.getConstant(Val: uint64_t(-1), DL, VT: MVT::i64),
2772 N4: DAG.getConstant(Val: uint64_t(0), DL, VT: MVT::i64), N5: CC);
2773 };
2774 return DAG.getBuildVector(VT: Op->getValueType(ResNo: 0), DL,
2775 Ops: {MakeLane(0), MakeLane(1)});
2776}
2777
2778SDValue
2779WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2780 SelectionDAG &DAG) const {
2781 // Allow constant lane indices, expand variable lane indices
2782 SDNode *IdxNode = Op.getOperand(i: Op.getNumOperands() - 1).getNode();
2783 if (isa<ConstantSDNode>(Val: IdxNode)) {
2784 // Ensure the index type is i32 to match the tablegen patterns
2785 uint64_t Idx = IdxNode->getAsZExtVal();
2786 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
2787 Ops[Op.getNumOperands() - 1] =
2788 DAG.getConstant(Val: Idx, DL: SDLoc(IdxNode), VT: MVT::i32);
2789 return DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT: Op.getValueType(), Ops);
2790 }
2791 // Perform default expansion
2792 return SDValue();
2793}
2794
2795static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
2796 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2797 // 32-bit and 64-bit unrolled shifts will have proper semantics
2798 if (LaneT.bitsGE(VT: MVT::i32))
2799 return DAG.UnrollVectorOp(N: Op.getNode());
2800 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2801 SDLoc DL(Op);
2802 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2803 SDValue Mask = DAG.getConstant(Val: LaneT.getSizeInBits() - 1, DL, VT: MVT::i32);
2804 unsigned ShiftOpcode = Op.getOpcode();
2805 SmallVector<SDValue, 16> ShiftedElements;
2806 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args&: ShiftedElements, Start: 0, Count: 0, EltVT: MVT::i32);
2807 SmallVector<SDValue, 16> ShiftElements;
2808 DAG.ExtractVectorElements(Op: Op.getOperand(i: 1), Args&: ShiftElements, Start: 0, Count: 0, EltVT: MVT::i32);
2809 SmallVector<SDValue, 16> UnrolledOps;
2810 for (size_t i = 0; i < NumLanes; ++i) {
2811 SDValue MaskedShiftValue =
2812 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: ShiftElements[i], N2: Mask);
2813 SDValue ShiftedValue = ShiftedElements[i];
2814 if (ShiftOpcode == ISD::SRA)
2815 ShiftedValue = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32,
2816 N1: ShiftedValue, N2: DAG.getValueType(LaneT));
2817 UnrolledOps.push_back(
2818 Elt: DAG.getNode(Opcode: ShiftOpcode, DL, VT: MVT::i32, N1: ShiftedValue, N2: MaskedShiftValue));
2819 }
2820 return DAG.getBuildVector(VT: Op.getValueType(), DL, Ops: UnrolledOps);
2821}
2822
2823SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2824 SelectionDAG &DAG) const {
2825 SDLoc DL(Op);
2826 // Only manually lower vector shifts
2827 assert(Op.getSimpleValueType().isVector());
2828
2829 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
2830 auto ShiftVal = Op.getOperand(i: 1);
2831
2832 // Try to skip bitmask operation since it is implied inside shift instruction
2833 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
2834 if (MaskOp.getOpcode() != ISD::AND)
2835 return MaskOp;
2836 SDValue LHS = MaskOp.getOperand(i: 0);
2837 SDValue RHS = MaskOp.getOperand(i: 1);
2838 if (MaskOp.getValueType().isVector()) {
2839 APInt MaskVal;
2840 if (!ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: MaskVal))
2841 std::swap(a&: LHS, b&: RHS);
2842
2843 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: MaskVal) &&
2844 MaskVal == MaskBits)
2845 MaskOp = LHS;
2846 } else {
2847 if (!isa<ConstantSDNode>(Val: RHS.getNode()))
2848 std::swap(a&: LHS, b&: RHS);
2849
2850 auto ConstantRHS = dyn_cast<ConstantSDNode>(Val: RHS.getNode());
2851 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2852 MaskOp = LHS;
2853 }
2854
2855 return MaskOp;
2856 };
2857
2858 // Skip vector and operation
2859 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2860 ShiftVal = DAG.getSplatValue(V: ShiftVal);
2861 if (!ShiftVal)
2862 return unrollVectorShift(Op, DAG);
2863
2864 // Skip scalar and operation
2865 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2866 // Use anyext because none of the high bits can affect the shift
2867 ShiftVal = DAG.getAnyExtOrTrunc(Op: ShiftVal, DL, VT: MVT::i32);
2868
2869 unsigned Opcode;
2870 switch (Op.getOpcode()) {
2871 case ISD::SHL:
2872 Opcode = WebAssemblyISD::VEC_SHL;
2873 break;
2874 case ISD::SRA:
2875 Opcode = WebAssemblyISD::VEC_SHR_S;
2876 break;
2877 case ISD::SRL:
2878 Opcode = WebAssemblyISD::VEC_SHR_U;
2879 break;
2880 default:
2881 llvm_unreachable("unexpected opcode");
2882 }
2883
2884 return DAG.getNode(Opcode, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: ShiftVal);
2885}
2886
2887SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2888 SelectionDAG &DAG) const {
2889 EVT ResT = Op.getValueType();
2890 EVT SatVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2891
2892 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2893 (SatVT == MVT::i32 || SatVT == MVT::i64))
2894 return Op;
2895
2896 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2897 return Op;
2898
2899 if (ResT == MVT::v8i16 && SatVT == MVT::i16)
2900 return Op;
2901
2902 return SDValue();
2903}
2904
2905static bool HasNoSignedZerosOrNaNs(SDValue Op, SelectionDAG &DAG) {
2906 return (Op->getFlags().hasNoNaNs() ||
2907 (DAG.isKnownNeverNaN(Op: Op->getOperand(Num: 0)) &&
2908 DAG.isKnownNeverNaN(Op: Op->getOperand(Num: 1)))) &&
2909 (Op->getFlags().hasNoSignedZeros() ||
2910 DAG.isKnownNeverZeroFloat(Op: Op->getOperand(Num: 0)) ||
2911 DAG.isKnownNeverZeroFloat(Op: Op->getOperand(Num: 1)));
2912}
2913
2914SDValue WebAssemblyTargetLowering::LowerFMIN(SDValue Op,
2915 SelectionDAG &DAG) const {
2916 if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
2917 return DAG.getNode(Opcode: WebAssemblyISD::RELAXED_FMIN, DL: SDLoc(Op),
2918 VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2919 }
2920 return SDValue();
2921}
2922
2923SDValue WebAssemblyTargetLowering::LowerFMAX(SDValue Op,
2924 SelectionDAG &DAG) const {
2925 if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
2926 return DAG.getNode(Opcode: WebAssemblyISD::RELAXED_FMAX, DL: SDLoc(Op),
2927 VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2928 }
2929 return SDValue();
2930}
2931
2932//===----------------------------------------------------------------------===//
2933// Custom DAG combine hooks
2934//===----------------------------------------------------------------------===//
2935static SDValue
2936performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2937 auto &DAG = DCI.DAG;
2938 auto Shuffle = cast<ShuffleVectorSDNode>(Val: N);
2939
2940 // Hoist vector bitcasts that don't change the number of lanes out of unary
2941 // shuffles, where they are less likely to get in the way of other combines.
2942 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2943 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2944 SDValue Bitcast = N->getOperand(Num: 0);
2945 if (Bitcast.getOpcode() != ISD::BITCAST)
2946 return SDValue();
2947 if (!N->getOperand(Num: 1).isUndef())
2948 return SDValue();
2949 SDValue CastOp = Bitcast.getOperand(i: 0);
2950 EVT SrcType = CastOp.getValueType();
2951 EVT DstType = Bitcast.getValueType();
2952 if (!SrcType.is128BitVector() ||
2953 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2954 return SDValue();
2955 SDValue NewShuffle = DAG.getVectorShuffle(
2956 VT: SrcType, dl: SDLoc(N), N1: CastOp, N2: DAG.getUNDEF(VT: SrcType), Mask: Shuffle->getMask());
2957 return DAG.getBitcast(VT: DstType, V: NewShuffle);
2958}
2959
2960/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
2961/// split up into scalar instructions during legalization, and the vector
2962/// extending instructions are selected in performVectorExtendCombine below.
2963static SDValue
2964performVectorExtendToFPCombine(SDNode *N,
2965 TargetLowering::DAGCombinerInfo &DCI) {
2966 auto &DAG = DCI.DAG;
2967 assert(N->getOpcode() == ISD::UINT_TO_FP ||
2968 N->getOpcode() == ISD::SINT_TO_FP);
2969
2970 EVT InVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
2971 EVT ResVT = N->getValueType(ResNo: 0);
2972 MVT ExtVT;
2973 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
2974 ExtVT = MVT::v4i32;
2975 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
2976 ExtVT = MVT::v2i32;
2977 else
2978 return SDValue();
2979
2980 unsigned Op =
2981 N->getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
2982 SDValue Conv = DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ExtVT, Operand: N->getOperand(Num: 0));
2983 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, Operand: Conv);
2984}
2985
2986static SDValue
2987performVectorNonNegToFPCombine(SDNode *N,
2988 TargetLowering::DAGCombinerInfo &DCI) {
2989 auto &DAG = DCI.DAG;
2990
2991 SDNodeFlags Flags = N->getFlags();
2992 SDValue Op0 = N->getOperand(Num: 0);
2993 EVT VT = N->getValueType(ResNo: 0);
2994
2995 // Optimize uitofp to sitofp when the sign bit is known to be zero.
2996 // Depending on the target (runtime) backend, this might be performance
2997 // neutral (e.g. AArch64) or a significant improvement (e.g. x86_64).
2998 if (VT.isVector() && (Flags.hasNonNeg() || DAG.SignBitIsZero(Op: Op0))) {
2999 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(N), VT, Operand: Op0);
3000 }
3001
3002 return SDValue();
3003}
3004
3005static SDValue
3006performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
3007 auto &DAG = DCI.DAG;
3008 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
3009 N->getOpcode() == ISD::ZERO_EXTEND);
3010
3011 EVT ResVT = N->getValueType(ResNo: 0);
3012 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
3013 SDLoc DL(N);
3014
3015 if (ResVT == MVT::v16i32 && N->getOperand(Num: 0)->getValueType(ResNo: 0) == MVT::v16i8) {
3016 // Use a tree of extend low/high to split and extend the input in two
3017 // layers to avoid doing several shuffles and even more extends.
3018 unsigned LowOp =
3019 IsSext ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3020 unsigned HighOp =
3021 IsSext ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3022 SDValue Input = N->getOperand(Num: 0);
3023 SDValue LowHalf = DAG.getNode(Opcode: LowOp, DL, VT: MVT::v8i16, Operand: Input);
3024 SDValue HighHalf = DAG.getNode(Opcode: HighOp, DL, VT: MVT::v8i16, Operand: Input);
3025 SDValue Subvectors[] = {
3026 DAG.getNode(Opcode: LowOp, DL, VT: MVT::v4i32, Operand: LowHalf),
3027 DAG.getNode(Opcode: HighOp, DL, VT: MVT::v4i32, Operand: LowHalf),
3028 DAG.getNode(Opcode: LowOp, DL, VT: MVT::v4i32, Operand: HighHalf),
3029 DAG.getNode(Opcode: HighOp, DL, VT: MVT::v4i32, Operand: HighHalf),
3030 };
3031 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, Ops: Subvectors);
3032 }
3033
3034 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
3035 // possible before the extract_subvector can be expanded.
3036 auto Extract = N->getOperand(Num: 0);
3037 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
3038 return SDValue();
3039 auto Source = Extract.getOperand(i: 0);
3040 auto *IndexNode = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
3041 if (IndexNode == nullptr)
3042 return SDValue();
3043 auto Index = IndexNode->getZExtValue();
3044
3045 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
3046 // extracted subvector is the low or high half of its source.
3047 if (ResVT == MVT::v8i16) {
3048 if (Extract.getValueType() != MVT::v8i8 ||
3049 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
3050 return SDValue();
3051 } else if (ResVT == MVT::v4i32) {
3052 if (Extract.getValueType() != MVT::v4i16 ||
3053 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
3054 return SDValue();
3055 } else if (ResVT == MVT::v2i64) {
3056 if (Extract.getValueType() != MVT::v2i32 ||
3057 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
3058 return SDValue();
3059 } else {
3060 return SDValue();
3061 }
3062
3063 bool IsLow = Index == 0;
3064
3065 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
3066 : WebAssemblyISD::EXTEND_HIGH_S)
3067 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
3068 : WebAssemblyISD::EXTEND_HIGH_U);
3069
3070 return DAG.getNode(Opcode: Op, DL, VT: ResVT, Operand: Source);
3071}
3072
3073static SDValue
3074performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
3075 auto &DAG = DCI.DAG;
3076
3077 auto GetWasmConversionOp = [](unsigned Op) {
3078 switch (Op) {
3079 case ISD::FP_TO_SINT_SAT:
3080 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
3081 case ISD::FP_TO_UINT_SAT:
3082 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
3083 case ISD::FP_ROUND:
3084 return WebAssemblyISD::DEMOTE_ZERO;
3085 }
3086 llvm_unreachable("unexpected op");
3087 };
3088
3089 auto IsZeroSplat = [](SDValue SplatVal) {
3090 auto *Splat = dyn_cast<BuildVectorSDNode>(Val: SplatVal.getNode());
3091 APInt SplatValue, SplatUndef;
3092 unsigned SplatBitSize;
3093 bool HasAnyUndefs;
3094 // Endianness doesn't matter in this context because we are looking for
3095 // an all-zero value.
3096 return Splat &&
3097 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
3098 HasAnyUndefs) &&
3099 SplatValue == 0;
3100 };
3101
3102 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
3103 // Combine this:
3104 //
3105 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
3106 //
3107 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
3108 //
3109 // Or this:
3110 //
3111 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
3112 //
3113 // into (f32x4.demote_zero_f64x2 $x).
3114 EVT ResVT;
3115 EVT ExpectedConversionType;
3116 auto Conversion = N->getOperand(Num: 0);
3117 auto ConversionOp = Conversion.getOpcode();
3118 switch (ConversionOp) {
3119 case ISD::FP_TO_SINT_SAT:
3120 case ISD::FP_TO_UINT_SAT:
3121 ResVT = MVT::v4i32;
3122 ExpectedConversionType = MVT::v2i32;
3123 break;
3124 case ISD::FP_ROUND:
3125 ResVT = MVT::v4f32;
3126 ExpectedConversionType = MVT::v2f32;
3127 break;
3128 default:
3129 return SDValue();
3130 }
3131
3132 if (N->getValueType(ResNo: 0) != ResVT)
3133 return SDValue();
3134
3135 if (Conversion.getValueType() != ExpectedConversionType)
3136 return SDValue();
3137
3138 auto Source = Conversion.getOperand(i: 0);
3139 if (Source.getValueType() != MVT::v2f64)
3140 return SDValue();
3141
3142 if (!IsZeroSplat(N->getOperand(Num: 1)) ||
3143 N->getOperand(Num: 1).getValueType() != ExpectedConversionType)
3144 return SDValue();
3145
3146 unsigned Op = GetWasmConversionOp(ConversionOp);
3147 return DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ResVT, Operand: Source);
3148 }
3149
3150 // Combine this:
3151 //
3152 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
3153 //
3154 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
3155 //
3156 // Or this:
3157 //
3158 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
3159 //
3160 // into (f32x4.demote_zero_f64x2 $x).
3161 EVT ResVT;
3162 auto ConversionOp = N->getOpcode();
3163 switch (ConversionOp) {
3164 case ISD::FP_TO_SINT_SAT:
3165 case ISD::FP_TO_UINT_SAT:
3166 ResVT = MVT::v4i32;
3167 break;
3168 case ISD::FP_ROUND:
3169 ResVT = MVT::v4f32;
3170 break;
3171 default:
3172 llvm_unreachable("unexpected op");
3173 }
3174
3175 if (N->getValueType(ResNo: 0) != ResVT)
3176 return SDValue();
3177
3178 auto Concat = N->getOperand(Num: 0);
3179 if (Concat.getValueType() != MVT::v4f64)
3180 return SDValue();
3181
3182 auto Source = Concat.getOperand(i: 0);
3183 if (Source.getValueType() != MVT::v2f64)
3184 return SDValue();
3185
3186 if (!IsZeroSplat(Concat.getOperand(i: 1)) ||
3187 Concat.getOperand(i: 1).getValueType() != MVT::v2f64)
3188 return SDValue();
3189
3190 unsigned Op = GetWasmConversionOp(ConversionOp);
3191 return DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ResVT, Operand: Source);
3192}
3193
3194// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
3195static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3196 const SDLoc &DL, unsigned VectorWidth) {
3197 EVT VT = Vec.getValueType();
3198 EVT ElVT = VT.getVectorElementType();
3199 unsigned Factor = VT.getSizeInBits() / VectorWidth;
3200 EVT ResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElVT,
3201 NumElements: VT.getVectorNumElements() / Factor);
3202
3203 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
3204 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
3205 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3206
3207 // This is the index of the first element of the VectorWidth-bit chunk
3208 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3209 IdxVal &= ~(ElemsPerChunk - 1);
3210
3211 // If the input is a buildvector just emit a smaller one.
3212 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3213 return DAG.getBuildVector(VT: ResultVT, DL,
3214 Ops: Vec->ops().slice(N: IdxVal, M: ElemsPerChunk));
3215
3216 SDValue VecIdx = DAG.getIntPtrConstant(Val: IdxVal, DL);
3217 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ResultVT, N1: Vec, N2: VecIdx);
3218}
3219
3220// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
3221// is the expected destination value type after recursion. In is the initial
3222// input. Note that the input should have enough leading zero bits to prevent
3223// NARROW_U from saturating results.
3224static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,
3225 SelectionDAG &DAG) {
3226 EVT SrcVT = In.getValueType();
3227
3228 // No truncation required, we might get here due to recursive calls.
3229 if (SrcVT == DstVT)
3230 return In;
3231
3232 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
3233 unsigned NumElems = SrcVT.getVectorNumElements();
3234 if (!isPowerOf2_32(Value: NumElems))
3235 return SDValue();
3236 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
3237 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
3238
3239 LLVMContext &Ctx = *DAG.getContext();
3240 EVT PackedSVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: SrcVT.getScalarSizeInBits() / 2);
3241
3242 // Narrow to the largest type possible:
3243 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
3244 EVT InVT = MVT::i16, OutVT = MVT::i8;
3245 if (SrcVT.getScalarSizeInBits() > 16) {
3246 InVT = MVT::i32;
3247 OutVT = MVT::i16;
3248 }
3249 unsigned SubSizeInBits = SrcSizeInBits / 2;
3250 InVT = EVT::getVectorVT(Context&: Ctx, VT: InVT, NumElements: SubSizeInBits / InVT.getSizeInBits());
3251 OutVT = EVT::getVectorVT(Context&: Ctx, VT: OutVT, NumElements: SubSizeInBits / OutVT.getSizeInBits());
3252
3253 // Split lower/upper subvectors.
3254 SDValue Lo = extractSubVector(Vec: In, IdxVal: 0, DAG, DL, VectorWidth: SubSizeInBits);
3255 SDValue Hi = extractSubVector(Vec: In, IdxVal: NumElems / 2, DAG, DL, VectorWidth: SubSizeInBits);
3256
3257 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
3258 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
3259 Lo = DAG.getBitcast(VT: InVT, V: Lo);
3260 Hi = DAG.getBitcast(VT: InVT, V: Hi);
3261 SDValue Res = DAG.getNode(Opcode: WebAssemblyISD::NARROW_U, DL, VT: OutVT, N1: Lo, N2: Hi);
3262 return DAG.getBitcast(VT: DstVT, V: Res);
3263 }
3264
3265 // Recursively narrow lower/upper subvectors, concat result and narrow again.
3266 EVT PackedVT = EVT::getVectorVT(Context&: Ctx, VT: PackedSVT, NumElements: NumElems / 2);
3267 Lo = truncateVectorWithNARROW(DstVT: PackedVT, In: Lo, DL, DAG);
3268 Hi = truncateVectorWithNARROW(DstVT: PackedVT, In: Hi, DL, DAG);
3269
3270 PackedVT = EVT::getVectorVT(Context&: Ctx, VT: PackedSVT, NumElements: NumElems);
3271 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedVT, N1: Lo, N2: Hi);
3272 return truncateVectorWithNARROW(DstVT, In: Res, DL, DAG);
3273}
3274
3275static SDValue performTruncateCombine(SDNode *N,
3276 TargetLowering::DAGCombinerInfo &DCI) {
3277 auto &DAG = DCI.DAG;
3278
3279 SDValue In = N->getOperand(Num: 0);
3280 EVT InVT = In.getValueType();
3281 if (!InVT.isSimple())
3282 return SDValue();
3283
3284 EVT OutVT = N->getValueType(ResNo: 0);
3285 if (!OutVT.isVector())
3286 return SDValue();
3287
3288 EVT OutSVT = OutVT.getVectorElementType();
3289 EVT InSVT = InVT.getVectorElementType();
3290 // Currently only cover truncate to v16i8 or v8i16.
3291 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
3292 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
3293 return SDValue();
3294
3295 SDLoc DL(N);
3296 APInt Mask = APInt::getLowBitsSet(numBits: InVT.getScalarSizeInBits(),
3297 loBitsSet: OutVT.getScalarSizeInBits());
3298 In = DAG.getNode(Opcode: ISD::AND, DL, VT: InVT, N1: In, N2: DAG.getConstant(Val: Mask, DL, VT: InVT));
3299 return truncateVectorWithNARROW(DstVT: OutVT, In, DL, DAG);
3300}
3301
3302static SDValue performBitcastCombine(SDNode *N,
3303 TargetLowering::DAGCombinerInfo &DCI) {
3304 using namespace llvm::SDPatternMatch;
3305 auto &DAG = DCI.DAG;
3306 SDLoc DL(N);
3307 SDValue Src = N->getOperand(Num: 0);
3308 EVT VT = N->getValueType(ResNo: 0);
3309 EVT SrcVT = Src.getValueType();
3310
3311 if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
3312 SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1))
3313 return SDValue();
3314
3315 unsigned NumElts = SrcVT.getVectorNumElements();
3316 EVT Width = MVT::getIntegerVT(BitWidth: 128 / NumElts);
3317
3318 // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
3319 // ==> bitmask
3320 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
3321 return DAG.getZExtOrTrunc(
3322 Op: DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3323 Ops: {DAG.getConstant(Val: Intrinsic::wasm_bitmask, DL, VT: MVT::i32),
3324 DAG.getSExtOrTrunc(Op: N->getOperand(Num: 0), DL,
3325 VT: SrcVT.changeVectorElementType(
3326 Context&: *DAG.getContext(), EltVT: Width))}),
3327 DL, VT);
3328 }
3329
3330 // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
3331 if (NumElts == 32 || NumElts == 64) {
3332 // Strategy: We will setcc them separately in v16i8 -> v16i1
3333 // Bitcast them to i16, extend them to either i32 or i64.
3334 // Add them together, shifting left 1 by 1.
3335 SDValue Concat, SetCCVector;
3336 ISD::CondCode SetCond;
3337
3338 if (!sd_match(N, P: m_BitCast(Op: m_c_SetCC(LHS: m_Value(N&: Concat), RHS: m_Value(N&: SetCCVector),
3339 CC: m_CondCode(CC&: SetCond)))))
3340 return SDValue();
3341 if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
3342 return SDValue();
3343
3344 uint64_t ElementWidth =
3345 SetCCVector.getValueType().getVectorElementType().getFixedSizeInBits();
3346
3347 SmallVector<SDValue> VectorsToShuffle;
3348 for (size_t I = 0; I < Concat->ops().size(); I++) {
3349 VectorsToShuffle.push_back(Elt: DAG.getBitcast(
3350 VT: MVT::i16,
3351 V: DAG.getSetCC(DL, VT: MVT::v16i1, LHS: Concat->ops()[I],
3352 RHS: extractSubVector(Vec: SetCCVector, IdxVal: I * (128 / ElementWidth),
3353 DAG, DL, VectorWidth: 128),
3354 Cond: SetCond)));
3355 }
3356
3357 MVT ReturnType = VectorsToShuffle.size() == 2 ? MVT::i32 : MVT::i64;
3358 SDValue ReturningInteger = DAG.getConstant(Val: 0, DL, VT: ReturnType);
3359
3360 for (SDValue V : VectorsToShuffle) {
3361 ReturningInteger = DAG.getNode(
3362 Opcode: ISD::SHL, DL, VT: ReturnType,
3363 Ops: {DAG.getShiftAmountConstant(Val: 16, VT: ReturnType, DL), ReturningInteger});
3364
3365 SDValue ExtendedV = DAG.getZExtOrTrunc(Op: V, DL, VT: ReturnType);
3366 ReturningInteger =
3367 DAG.getNode(Opcode: ISD::ADD, DL, VT: ReturnType, Ops: {ReturningInteger, ExtendedV});
3368 }
3369
3370 return ReturningInteger;
3371 }
3372
3373 return SDValue();
3374}
3375
3376static SDValue performBitmaskCombine(SDNode *N, SelectionDAG &DAG) {
3377 // bitmask (setcc <X>, 0, setlt) => bitmask X
3378 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
3379 using namespace llvm::SDPatternMatch;
3380
3381 if (N->getConstantOperandVal(Num: 0) != Intrinsic::wasm_bitmask)
3382 return SDValue();
3383
3384 SDValue LHS;
3385 if (!sd_match(N: N->getOperand(Num: 1), P: m_c_SetCC(LHS: m_Value(N&: LHS), RHS: m_Zero(),
3386 CC: m_SpecificCondCode(CC: ISD::SETLT))))
3387 return SDValue();
3388
3389 SDLoc DL(N);
3390 return DAG.getNode(
3391 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: N->getValueType(ResNo: 0),
3392 Ops: {DAG.getConstant(Val: Intrinsic::wasm_bitmask, DL, VT: MVT::i32), LHS});
3393}
3394
3395static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {
3396 // any_true (setcc <X>, 0, eq) => (not (all_true X))
3397 // all_true (setcc <X>, 0, eq) => (not (any_true X))
3398 // any_true (setcc <X>, 0, ne) => (any_true X)
3399 // all_true (setcc <X>, 0, ne) => (all_true X)
3400 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
3401 using namespace llvm::SDPatternMatch;
3402
3403 SDValue LHS;
3404 if (N->getNumOperands() < 2 ||
3405 !sd_match(N: N->getOperand(Num: 1),
3406 P: m_c_SetCC(LHS: m_Value(N&: LHS), RHS: m_Zero(), CC: m_CondCode())))
3407 return SDValue();
3408 EVT LT = LHS.getValueType();
3409 if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
3410 return SDValue();
3411
3412 auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
3413 ISD::CondCode SetType,
3414 Intrinsic::WASMIntrinsics InPost) {
3415 if (N->getConstantOperandVal(Num: 0) != InPre)
3416 return SDValue();
3417
3418 SDValue LHS;
3419 if (!sd_match(N: N->getOperand(Num: 1), P: m_c_SetCC(LHS: m_Value(N&: LHS), RHS: m_Zero(),
3420 CC: m_SpecificCondCode(CC: SetType))))
3421 return SDValue();
3422
3423 SDLoc DL(N);
3424 SDValue Ret = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3425 Ops: {DAG.getConstant(Val: InPost, DL, VT: MVT::i32), LHS});
3426 if (SetType == ISD::SETEQ)
3427 Ret = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Ret,
3428 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
3429 return DAG.getZExtOrTrunc(Op: Ret, DL, VT: N->getValueType(ResNo: 0));
3430 };
3431
3432 if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
3433 Intrinsic::wasm_alltrue))
3434 return AnyTrueEQ;
3435 if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
3436 Intrinsic::wasm_anytrue))
3437 return AllTrueEQ;
3438 if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
3439 Intrinsic::wasm_anytrue))
3440 return AnyTrueNE;
3441 if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
3442 Intrinsic::wasm_alltrue))
3443 return AllTrueNE;
3444
3445 return SDValue();
3446}
3447
3448template <int MatchRHS, ISD::CondCode MatchCond, bool RequiresNegate,
3449 Intrinsic::ID Intrin>
3450static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG) {
3451 SDValue LHS = N->getOperand(Num: 0);
3452 SDValue RHS = N->getOperand(Num: 1);
3453 SDValue Cond = N->getOperand(Num: 2);
3454 if (MatchCond != cast<CondCodeSDNode>(Val&: Cond)->get())
3455 return SDValue();
3456
3457 if (MatchRHS != cast<ConstantSDNode>(Val&: RHS)->getSExtValue())
3458 return SDValue();
3459
3460 SDLoc DL(N);
3461 SDValue Ret =
3462 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3463 Ops: {DAG.getConstant(Val: Intrin, DL, VT: MVT::i32),
3464 DAG.getSExtOrTrunc(Op: LHS->getOperand(Num: 0), DL, VT: VecVT)});
3465 if (RequiresNegate)
3466 Ret = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Ret,
3467 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
3468 return DAG.getZExtOrTrunc(Op: Ret, DL, VT: N->getValueType(ResNo: 0));
3469}
3470
3471/// Try to convert a i128 comparison to a v16i8 comparison before type
3472/// legalization splits it up into chunks
3473static SDValue
3474combineVectorSizedSetCCEquality(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
3475 const WebAssemblySubtarget *Subtarget) {
3476
3477 SDLoc DL(N);
3478 SDValue X = N->getOperand(Num: 0);
3479 SDValue Y = N->getOperand(Num: 1);
3480 EVT VT = N->getValueType(ResNo: 0);
3481 EVT OpVT = X.getValueType();
3482
3483 SelectionDAG &DAG = DCI.DAG;
3484 if (DCI.DAG.getMachineFunction().getFunction().hasFnAttribute(
3485 Kind: Attribute::NoImplicitFloat))
3486 return SDValue();
3487
3488 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3489 // We're looking for an oversized integer equality comparison with SIMD
3490 if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||
3491 !Subtarget->hasSIMD128() || !isIntEqualitySetCC(Code: CC))
3492 return SDValue();
3493
3494 // Don't perform this combine if constructing the vector will be expensive.
3495 auto IsVectorBitCastCheap = [](SDValue X) {
3496 X = peekThroughBitcasts(V: X);
3497 return isa<ConstantSDNode>(Val: X) || X.getOpcode() == ISD::LOAD;
3498 };
3499
3500 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
3501 return SDValue();
3502
3503 SDValue VecX = DAG.getBitcast(VT: MVT::v16i8, V: X);
3504 SDValue VecY = DAG.getBitcast(VT: MVT::v16i8, V: Y);
3505 SDValue Cmp = DAG.getSetCC(DL, VT: MVT::v16i8, LHS: VecX, RHS: VecY, Cond: CC);
3506
3507 SDValue Intr =
3508 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3509 Ops: {DAG.getConstant(Val: CC == ISD::SETEQ ? Intrinsic::wasm_alltrue
3510 : Intrinsic::wasm_anytrue,
3511 DL, VT: MVT::i32),
3512 Cmp});
3513
3514 return DAG.getSetCC(DL, VT, LHS: Intr, RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3515 Cond: ISD::SETNE);
3516}
3517
3518static SDValue performSETCCCombine(SDNode *N,
3519 TargetLowering::DAGCombinerInfo &DCI,
3520 const WebAssemblySubtarget *Subtarget) {
3521 if (!DCI.isBeforeLegalize())
3522 return SDValue();
3523
3524 EVT VT = N->getValueType(ResNo: 0);
3525 if (!VT.isScalarInteger())
3526 return SDValue();
3527
3528 if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))
3529 return V;
3530
3531 SDValue LHS = N->getOperand(Num: 0);
3532 if (LHS->getOpcode() != ISD::BITCAST)
3533 return SDValue();
3534
3535 EVT FromVT = LHS->getOperand(Num: 0).getValueType();
3536 if (!FromVT.isFixedLengthVector() || FromVT.getVectorElementType() != MVT::i1)
3537 return SDValue();
3538
3539 unsigned NumElts = FromVT.getVectorNumElements();
3540 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
3541 return SDValue();
3542
3543 if (!cast<ConstantSDNode>(Val: N->getOperand(Num: 1)))
3544 return SDValue();
3545
3546 auto &DAG = DCI.DAG;
3547 EVT VecVT = FromVT.changeVectorElementType(Context&: *DAG.getContext(),
3548 EltVT: MVT::getIntegerVT(BitWidth: 128 / NumElts));
3549 // setcc (iN (bitcast (vNi1 X))), 0, ne
3550 // ==> any_true (vNi1 X)
3551 if (auto Match = TryMatchTrue<0, ISD::SETNE, false, Intrinsic::wasm_anytrue>(
3552 N, VecVT, DAG)) {
3553 return Match;
3554 }
3555 // setcc (iN (bitcast (vNi1 X))), 0, eq
3556 // ==> xor (any_true (vNi1 X)), -1
3557 if (auto Match = TryMatchTrue<0, ISD::SETEQ, true, Intrinsic::wasm_anytrue>(
3558 N, VecVT, DAG)) {
3559 return Match;
3560 }
3561 // setcc (iN (bitcast (vNi1 X))), -1, eq
3562 // ==> all_true (vNi1 X)
3563 if (auto Match = TryMatchTrue<-1, ISD::SETEQ, false, Intrinsic::wasm_alltrue>(
3564 N, VecVT, DAG)) {
3565 return Match;
3566 }
3567 // setcc (iN (bitcast (vNi1 X))), -1, ne
3568 // ==> xor (all_true (vNi1 X)), -1
3569 if (auto Match = TryMatchTrue<-1, ISD::SETNE, true, Intrinsic::wasm_alltrue>(
3570 N, VecVT, DAG)) {
3571 return Match;
3572 }
3573 return SDValue();
3574}
3575
3576static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
3577 EVT VT = N->getValueType(ResNo: 0);
3578 if (VT != MVT::v8i32 && VT != MVT::v16i32)
3579 return SDValue();
3580
3581 // Mul with extending inputs.
3582 SDValue LHS = N->getOperand(Num: 0);
3583 SDValue RHS = N->getOperand(Num: 1);
3584 if (LHS.getOpcode() != RHS.getOpcode())
3585 return SDValue();
3586
3587 if (LHS.getOpcode() != ISD::SIGN_EXTEND &&
3588 LHS.getOpcode() != ISD::ZERO_EXTEND)
3589 return SDValue();
3590
3591 if (LHS->getOperand(Num: 0).getValueType() != RHS->getOperand(Num: 0).getValueType())
3592 return SDValue();
3593
3594 EVT FromVT = LHS->getOperand(Num: 0).getValueType();
3595 EVT EltTy = FromVT.getVectorElementType();
3596 if (EltTy != MVT::i8)
3597 return SDValue();
3598
3599 // For an input DAG that looks like this
3600 // %a = input_type
3601 // %b = input_type
3602 // %lhs = extend %a to output_type
3603 // %rhs = extend %b to output_type
3604 // %mul = mul %lhs, %rhs
3605
3606 // input_type | output_type | instructions
3607 // v16i8 | v16i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3608 // | | %high = i16x8.extmul_high_i8x16_, %a, %b
3609 // | | %low_low = i32x4.ext_low_i16x8_ %low
3610 // | | %low_high = i32x4.ext_high_i16x8_ %low
3611 // | | %high_low = i32x4.ext_low_i16x8_ %high
3612 // | | %high_high = i32x4.ext_high_i16x8_ %high
3613 // | | %res = concat_vector(...)
3614 // v8i8 | v8i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3615 // | | %low_low = i32x4.ext_low_i16x8_ %low
3616 // | | %low_high = i32x4.ext_high_i16x8_ %low
3617 // | | %res = concat_vector(%low_low, %low_high)
3618
3619 SDLoc DL(N);
3620 unsigned NumElts = VT.getVectorNumElements();
3621 SDValue ExtendInLHS = LHS->getOperand(Num: 0);
3622 SDValue ExtendInRHS = RHS->getOperand(Num: 0);
3623 bool IsSigned = LHS->getOpcode() == ISD::SIGN_EXTEND;
3624 unsigned ExtendLowOpc =
3625 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3626 unsigned ExtendHighOpc =
3627 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3628
3629 auto GetExtendLow = [&DAG, &DL, &ExtendLowOpc](EVT VT, SDValue Op) {
3630 return DAG.getNode(Opcode: ExtendLowOpc, DL, VT, Operand: Op);
3631 };
3632 auto GetExtendHigh = [&DAG, &DL, &ExtendHighOpc](EVT VT, SDValue Op) {
3633 return DAG.getNode(Opcode: ExtendHighOpc, DL, VT, Operand: Op);
3634 };
3635
3636 if (NumElts == 16) {
3637 SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);
3638 SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);
3639 SDValue MulLow = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: LowLHS, N2: LowRHS);
3640 SDValue HighLHS = GetExtendHigh(MVT::v8i16, ExtendInLHS);
3641 SDValue HighRHS = GetExtendHigh(MVT::v8i16, ExtendInRHS);
3642 SDValue MulHigh = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: HighLHS, N2: HighRHS);
3643 SDValue SubVectors[] = {
3644 GetExtendLow(MVT::v4i32, MulLow),
3645 GetExtendHigh(MVT::v4i32, MulLow),
3646 GetExtendLow(MVT::v4i32, MulHigh),
3647 GetExtendHigh(MVT::v4i32, MulHigh),
3648 };
3649 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: SubVectors);
3650 } else {
3651 assert(NumElts == 8);
3652 SDValue LowLHS = DAG.getNode(Opcode: LHS->getOpcode(), DL, VT: MVT::v8i16, Operand: ExtendInLHS);
3653 SDValue LowRHS = DAG.getNode(Opcode: RHS->getOpcode(), DL, VT: MVT::v8i16, Operand: ExtendInRHS);
3654 SDValue MulLow = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: LowLHS, N2: LowRHS);
3655 SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
3656 SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);
3657 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
3658 }
3659 return SDValue();
3660}
3661
3662static SDValue performMulCombine(SDNode *N,
3663 TargetLowering::DAGCombinerInfo &DCI) {
3664 assert(N->getOpcode() == ISD::MUL);
3665 EVT VT = N->getValueType(ResNo: 0);
3666 if (!VT.isVector())
3667 return SDValue();
3668
3669 if (auto Res = TryWideExtMulCombine(N, DAG&: DCI.DAG))
3670 return Res;
3671
3672 // We don't natively support v16i8 or v8i8 mul, but we do support v8i16. So,
3673 // extend them to v8i16.
3674 if (VT != MVT::v8i8 && VT != MVT::v16i8)
3675 return SDValue();
3676
3677 SDLoc DL(N);
3678 SelectionDAG &DAG = DCI.DAG;
3679 SDValue LHS = N->getOperand(Num: 0);
3680 SDValue RHS = N->getOperand(Num: 1);
3681 EVT MulVT = MVT::v8i16;
3682
3683 if (VT == MVT::v8i8) {
3684 SDValue PromotedLHS = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v16i8, N1: LHS,
3685 N2: DAG.getUNDEF(VT: MVT::v8i8));
3686 SDValue PromotedRHS = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v16i8, N1: RHS,
3687 N2: DAG.getUNDEF(VT: MVT::v8i8));
3688 SDValue LowLHS =
3689 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: PromotedLHS);
3690 SDValue LowRHS =
3691 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: PromotedRHS);
3692 SDValue MulLow = DAG.getBitcast(
3693 VT: MVT::v16i8, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: LowLHS, N2: LowRHS));
3694 // Take the low byte of each lane.
3695 SDValue Shuffle = DAG.getVectorShuffle(
3696 VT: MVT::v16i8, dl: DL, N1: MulLow, N2: DAG.getUNDEF(VT: MVT::v16i8),
3697 Mask: {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});
3698 return extractSubVector(Vec: Shuffle, IdxVal: 0, DAG, DL, VectorWidth: 64);
3699 } else {
3700 assert(VT == MVT::v16i8 && "Expected v16i8");
3701 SDValue LowLHS = DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: LHS);
3702 SDValue LowRHS = DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: RHS);
3703 SDValue HighLHS =
3704 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_HIGH_U, DL, VT: MulVT, Operand: LHS);
3705 SDValue HighRHS =
3706 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_HIGH_U, DL, VT: MulVT, Operand: RHS);
3707
3708 SDValue MulLow =
3709 DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: LowLHS, N2: LowRHS));
3710 SDValue MulHigh =
3711 DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: HighLHS, N2: HighRHS));
3712
3713 // Take the low byte of each lane.
3714 return DAG.getVectorShuffle(
3715 VT, dl: DL, N1: MulLow, N2: MulHigh,
3716 Mask: {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
3717 }
3718}
3719
3720SDValue DoubleVectorWidth(SDValue In, unsigned RequiredNumElems,
3721 SelectionDAG &DAG) {
3722 SDLoc DL(In);
3723 LLVMContext &Ctx = *DAG.getContext();
3724 EVT InVT = In.getValueType();
3725 unsigned NumElems = InVT.getVectorNumElements() * 2;
3726 EVT OutVT = EVT::getVectorVT(Context&: Ctx, VT: InVT.getVectorElementType(), NumElements: NumElems);
3727 SDValue Concat =
3728 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, N1: In, N2: DAG.getPOISON(VT: InVT));
3729 if (NumElems < RequiredNumElems) {
3730 return DoubleVectorWidth(In: Concat, RequiredNumElems, DAG);
3731 }
3732 return Concat;
3733}
3734
3735SDValue performConvertFPCombine(SDNode *N, SelectionDAG &DAG) {
3736 EVT OutVT = N->getValueType(ResNo: 0);
3737 if (!OutVT.isVector())
3738 return SDValue();
3739
3740 EVT OutElTy = OutVT.getVectorElementType();
3741 if (OutElTy != MVT::i8 && OutElTy != MVT::i16)
3742 return SDValue();
3743
3744 unsigned NumElems = OutVT.getVectorNumElements();
3745 if (!isPowerOf2_32(Value: NumElems))
3746 return SDValue();
3747
3748 EVT FPVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3749 if (FPVT.getVectorElementType() != MVT::f32)
3750 return SDValue();
3751
3752 SDLoc DL(N);
3753
3754 // First, convert to i32.
3755 LLVMContext &Ctx = *DAG.getContext();
3756 EVT IntVT = EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: NumElems);
3757 SDValue ToInt = DAG.getNode(Opcode: N->getOpcode(), DL, VT: IntVT, Operand: N->getOperand(Num: 0));
3758 APInt Mask = APInt::getLowBitsSet(numBits: IntVT.getScalarSizeInBits(),
3759 loBitsSet: OutVT.getScalarSizeInBits());
3760 // Mask out the top MSBs.
3761 SDValue Masked =
3762 DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ToInt, N2: DAG.getConstant(Val: Mask, DL, VT: IntVT));
3763
3764 if (OutVT.getSizeInBits() < 128) {
3765 // Create a wide enough vector that we can use narrow.
3766 EVT NarrowedVT = OutElTy == MVT::i8 ? MVT::v16i8 : MVT::v8i16;
3767 unsigned NumRequiredElems = NarrowedVT.getVectorNumElements();
3768 SDValue WideVector = DoubleVectorWidth(In: Masked, RequiredNumElems: NumRequiredElems, DAG);
3769 SDValue Trunc = truncateVectorWithNARROW(DstVT: NarrowedVT, In: WideVector, DL, DAG);
3770 return DAG.getBitcast(
3771 VT: OutVT, V: extractSubVector(Vec: Trunc, IdxVal: 0, DAG, DL, VectorWidth: OutVT.getSizeInBits()));
3772 } else {
3773 return truncateVectorWithNARROW(DstVT: OutVT, In: Masked, DL, DAG);
3774 }
3775 return SDValue();
3776}
3777
3778// Wide vector shift operations such as v8i32 with sign-extended
3779// operands cause Type Legalizer crashes because the target-specific
3780// extension nodes cannot be directly mapped to the 256-bit size.
3781//
3782// To resolve the crash and optimize performance, we intercept the
3783// illegal v8i32 shift in DAGCombine. We convert the shift amounts
3784// into multipliers and manually split the vector into two v4i32 halves.
3785//
3786// Before: t1: v8i32 = shl (sign_extend v8i16), const_vec
3787// After : t2: v4i32 = mul (ext_low_s v8i16), (ext_low_s narrow_vec)
3788// t3: v4i32 = mul (ext_high_s v8i16), (ext_high_s narrow_vec)
3789// t4: v8i32 = concat_vectors t2, t3
3790static SDValue performShiftCombine(SDNode *N,
3791 TargetLowering::DAGCombinerInfo &DCI) {
3792 SelectionDAG &DAG = DCI.DAG;
3793 assert(N->getOpcode() == ISD::SHL);
3794 EVT VT = N->getValueType(ResNo: 0);
3795 if (VT != MVT::v8i32)
3796 return SDValue();
3797
3798 SDValue LHS = N->getOperand(Num: 0);
3799 SDValue RHS = N->getOperand(Num: 1);
3800 unsigned ExtOpc = LHS.getOpcode();
3801 if (ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND)
3802 return SDValue();
3803
3804 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
3805 return SDValue();
3806
3807 SDLoc DL(N);
3808 SDValue ExtendIn = LHS.getOperand(i: 0);
3809 EVT FromVT = ExtendIn.getValueType();
3810 if (FromVT != MVT::v8i16)
3811 return SDValue();
3812
3813 unsigned NumElts = VT.getVectorNumElements();
3814 unsigned BitWidth = FromVT.getScalarSizeInBits();
3815 bool IsSigned = (ExtOpc == ISD::SIGN_EXTEND);
3816 unsigned MaxValidShift = IsSigned ? (BitWidth - 1) : BitWidth;
3817 SmallVector<SDValue, 16> MulConsts;
3818 for (unsigned I = 0; I < NumElts; ++I) {
3819 auto *C = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: I));
3820 if (!C)
3821 return SDValue();
3822
3823 const APInt &ShiftAmt = C->getAPIntValue();
3824 if (ShiftAmt.uge(RHS: MaxValidShift))
3825 return SDValue();
3826
3827 APInt MulAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt.getZExtValue());
3828 MulConsts.push_back(Elt: DAG.getConstant(Val: MulAmt, DL, VT: FromVT.getScalarType(),
3829 /*isTarget=*/false, /*isOpaque=*/true));
3830 }
3831
3832 SDValue NarrowConst = DAG.getBuildVector(VT: FromVT, DL, Ops: MulConsts);
3833 unsigned ExtLowOpc =
3834 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3835 unsigned ExtHighOpc =
3836 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3837
3838 EVT HalfVT = MVT::v4i32;
3839 SDValue LHSLo = DAG.getNode(Opcode: ExtLowOpc, DL, VT: HalfVT, Operand: ExtendIn);
3840 SDValue LHSHi = DAG.getNode(Opcode: ExtHighOpc, DL, VT: HalfVT, Operand: ExtendIn);
3841 SDValue RHSLo = DAG.getNode(Opcode: ExtLowOpc, DL, VT: HalfVT, Operand: NarrowConst);
3842 SDValue RHSHi = DAG.getNode(Opcode: ExtHighOpc, DL, VT: HalfVT, Operand: NarrowConst);
3843 SDValue MulLo = DAG.getNode(Opcode: ISD::MUL, DL, VT: HalfVT, N1: LHSLo, N2: RHSLo);
3844 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL, VT: HalfVT, N1: LHSHi, N2: RHSHi);
3845 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: MulLo, N2: MulHi);
3846}
3847
3848SDValue
3849WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
3850 DAGCombinerInfo &DCI) const {
3851 switch (N->getOpcode()) {
3852 default:
3853 return SDValue();
3854 case ISD::BITCAST:
3855 return performBitcastCombine(N, DCI);
3856 case ISD::SETCC:
3857 return performSETCCCombine(N, DCI, Subtarget);
3858 case ISD::VECTOR_SHUFFLE:
3859 return performVECTOR_SHUFFLECombine(N, DCI);
3860 case ISD::SIGN_EXTEND:
3861 case ISD::ZERO_EXTEND:
3862 return performVectorExtendCombine(N, DCI);
3863 case ISD::UINT_TO_FP:
3864 if (auto ExtCombine = performVectorExtendToFPCombine(N, DCI))
3865 return ExtCombine;
3866 return performVectorNonNegToFPCombine(N, DCI);
3867 case ISD::SINT_TO_FP:
3868 return performVectorExtendToFPCombine(N, DCI);
3869 case ISD::FP_TO_SINT_SAT:
3870 case ISD::FP_TO_UINT_SAT:
3871 case ISD::FP_ROUND:
3872 case ISD::CONCAT_VECTORS:
3873 return performVectorTruncZeroCombine(N, DCI);
3874 case ISD::FP_TO_SINT:
3875 case ISD::FP_TO_UINT:
3876 return performConvertFPCombine(N, DAG&: DCI.DAG);
3877 case ISD::TRUNCATE:
3878 return performTruncateCombine(N, DCI);
3879 case ISD::INTRINSIC_WO_CHAIN: {
3880 if (SDValue V = performBitmaskCombine(N, DAG&: DCI.DAG))
3881 return V;
3882 return performAnyAllCombine(N, DAG&: DCI.DAG);
3883 }
3884 case ISD::MUL:
3885 return performMulCombine(N, DCI);
3886 case ISD::SHL:
3887 return performShiftCombine(N, DCI);
3888 }
3889}
3890