1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "WebAssemblyMachineFunctionInfo.h"
18#include "WebAssemblySubtarget.h"
19#include "WebAssemblyTargetMachine.h"
20#include "WebAssemblyUtilities.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineFrameInfo.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineJumpTableInfo.h"
25#include "llvm/CodeGen/MachineModuleInfo.h"
26#include "llvm/CodeGen/MachineRegisterInfo.h"
27#include "llvm/CodeGen/SDPatternMatch.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/DiagnosticPrinter.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/KnownBits.h"
37#include "llvm/Support/MathExtras.h"
38#include "llvm/Target/TargetOptions.h"
39using namespace llvm;
40
41#define DEBUG_TYPE "wasm-lower"
42
43WebAssemblyTargetLowering::WebAssemblyTargetLowering(
44 const TargetMachine &TM, const WebAssemblySubtarget &STI)
45 : TargetLowering(TM, STI), Subtarget(&STI) {
46 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
47
48 // Set the load count for memcmp expand optimization
49 MaxLoadsPerMemcmp = 8;
50 MaxLoadsPerMemcmpOptSize = 4;
51
52 // Booleans always contain 0 or 1.
53 setBooleanContents(ZeroOrOneBooleanContent);
54 // Except in SIMD vectors
55 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
56 // We don't know the microarchitecture here, so just reduce register pressure.
57 setSchedulingPreference(Sched::RegPressure);
58 // Tell ISel that we have a stack pointer.
59 setStackPointerRegisterToSaveRestore(
60 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
61 // Set up the register classes.
62 addRegisterClass(VT: MVT::i32, RC: &WebAssembly::I32RegClass);
63 addRegisterClass(VT: MVT::i64, RC: &WebAssembly::I64RegClass);
64 addRegisterClass(VT: MVT::f32, RC: &WebAssembly::F32RegClass);
65 addRegisterClass(VT: MVT::f64, RC: &WebAssembly::F64RegClass);
66 if (Subtarget->hasSIMD128()) {
67 addRegisterClass(VT: MVT::v16i8, RC: &WebAssembly::V128RegClass);
68 addRegisterClass(VT: MVT::v8i16, RC: &WebAssembly::V128RegClass);
69 addRegisterClass(VT: MVT::v4i32, RC: &WebAssembly::V128RegClass);
70 addRegisterClass(VT: MVT::v4f32, RC: &WebAssembly::V128RegClass);
71 addRegisterClass(VT: MVT::v2i64, RC: &WebAssembly::V128RegClass);
72 addRegisterClass(VT: MVT::v2f64, RC: &WebAssembly::V128RegClass);
73 }
74 if (Subtarget->hasFP16()) {
75 addRegisterClass(VT: MVT::v8f16, RC: &WebAssembly::V128RegClass);
76 }
77 if (Subtarget->hasReferenceTypes()) {
78 addRegisterClass(VT: MVT::externref, RC: &WebAssembly::EXTERNREFRegClass);
79 addRegisterClass(VT: MVT::funcref, RC: &WebAssembly::FUNCREFRegClass);
80 if (Subtarget->hasExceptionHandling()) {
81 addRegisterClass(VT: MVT::exnref, RC: &WebAssembly::EXNREFRegClass);
82 }
83 }
84 // Compute derived properties from the register classes.
85 computeRegisterProperties(TRI: Subtarget->getRegisterInfo());
86
87 // Transform loads and stores to pointers in address space 1 to loads and
88 // stores to WebAssembly global variables, outside linear memory.
89 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
90 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
91 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
92 }
93 if (Subtarget->hasSIMD128()) {
94 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
95 MVT::v2f64}) {
96 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
97 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
98 }
99 }
100 if (Subtarget->hasFP16()) {
101 setOperationAction(Op: ISD::LOAD, VT: MVT::v8f16, Action: Custom);
102 setOperationAction(Op: ISD::STORE, VT: MVT::v8f16, Action: Custom);
103 }
104 if (Subtarget->hasReferenceTypes()) {
105 // We need custom load and store lowering for both externref, funcref and
106 // Other. The MVT::Other here represents tables of reference types.
107 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
108 setOperationAction(Op: ISD::LOAD, VT: T, Action: Custom);
109 setOperationAction(Op: ISD::STORE, VT: T, Action: Custom);
110 }
111 }
112
113 setOperationAction(Op: ISD::GlobalAddress, VT: MVTPtr, Action: Custom);
114 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVTPtr, Action: Custom);
115 setOperationAction(Op: ISD::ExternalSymbol, VT: MVTPtr, Action: Custom);
116 setOperationAction(Op: ISD::JumpTable, VT: MVTPtr, Action: Custom);
117 setOperationAction(Op: ISD::BlockAddress, VT: MVTPtr, Action: Custom);
118 setOperationAction(Op: ISD::BRIND, VT: MVT::Other, Action: Custom);
119 setOperationAction(Op: ISD::CLEAR_CACHE, VT: MVT::Other, Action: Custom);
120
121 // Take the default expansion for va_arg, va_copy, and va_end. There is no
122 // default action for va_start, so we do that custom.
123 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
124 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
125 setOperationAction(Op: ISD::VACOPY, VT: MVT::Other, Action: Expand);
126 setOperationAction(Op: ISD::VAEND, VT: MVT::Other, Action: Expand);
127
128 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {
129 if (!Subtarget->hasFP16() && T == MVT::v8f16) {
130 continue;
131 }
132 // Don't expand the floating-point types to constant pools.
133 setOperationAction(Op: ISD::ConstantFP, VT: T, Action: Legal);
134 // Expand floating-point comparisons.
135 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
136 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
137 setCondCodeAction(CCs: CC, VT: T, Action: Expand);
138 // Expand floating-point library function operators.
139 for (auto Op : {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FMA})
140 setOperationAction(Op, VT: T, Action: Expand);
141 // Expand vector FREM, but use a libcall rather than an expansion for scalar
142 if (MVT(T).isVector())
143 setOperationAction(Op: ISD::FREM, VT: T, Action: Expand);
144 else
145 setOperationAction(Op: ISD::FREM, VT: T, Action: LibCall);
146 // Note supported floating-point library function operators that otherwise
147 // default to expand.
148 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
149 ISD::FRINT, ISD::FROUNDEVEN})
150 setOperationAction(Op, VT: T, Action: Legal);
151 // Support minimum and maximum, which otherwise default to expand.
152 setOperationAction(Op: ISD::FMINIMUM, VT: T, Action: Legal);
153 setOperationAction(Op: ISD::FMAXIMUM, VT: T, Action: Legal);
154 if (Subtarget->hasSIMD128() && MVT(T).isVector()) {
155 setOperationAction(Op: ISD::PSEUDO_FMIN, VT: T, Action: Legal);
156 setOperationAction(Op: ISD::PSEUDO_FMAX, VT: T, Action: Legal);
157 }
158 // When experimental v8f16 support is enabled these instructions don't need
159 // to be expanded.
160 if (T != MVT::v8f16) {
161 setOperationAction(Op: ISD::FP16_TO_FP, VT: T, Action: Expand);
162 setOperationAction(Op: ISD::FP_TO_FP16, VT: T, Action: Expand);
163 }
164 if (Subtarget->hasFP16() && T == MVT::f32) {
165 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: T, MemVT: MVT::f16, Action: Legal);
166 setTruncStoreAction(ValVT: T, MemVT: MVT::f16, Action: Legal);
167 } else {
168 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: T, MemVT: MVT::f16, Action: Expand);
169 setTruncStoreAction(ValVT: T, MemVT: MVT::f16, Action: Expand);
170 }
171 }
172
173 // Expand unavailable integer operations.
174 for (auto Op :
175 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
176 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
177 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
178 for (auto T : {MVT::i32, MVT::i64})
179 setOperationAction(Op, VT: T, Action: Expand);
180 if (Subtarget->hasSIMD128())
181 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
182 setOperationAction(Op, VT: T, Action: Expand);
183 }
184
185 if (Subtarget->hasWideArithmetic()) {
186 setOperationAction(Op: ISD::ADD, VT: MVT::i128, Action: Custom);
187 setOperationAction(Op: ISD::SUB, VT: MVT::i128, Action: Custom);
188 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Custom);
189 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Custom);
190 setOperationAction(Op: ISD::UADDO, VT: MVT::i64, Action: Custom);
191 }
192
193 if (Subtarget->hasNontrappingFPToInt())
194 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
195 for (auto T : {MVT::i32, MVT::i64})
196 setOperationAction(Op, VT: T, Action: Custom);
197
198 if (Subtarget->hasRelaxedSIMD()) {
199 setOperationAction(
200 Ops: {ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM},
201 VTs: {MVT::v4f32, MVT::v2f64}, Action: Custom);
202 }
203
204 // Combine expands these operations, because wasi-libc and emscripten do not
205 // yet have the dedicated libcalls.
206 setTargetDAGCombine(
207 {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM});
208
209 // SIMD-specific configuration
210 if (Subtarget->hasSIMD128()) {
211
212 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
213
214 // Combine wide-vector muls, with extend inputs, to extmul_half.
215 setTargetDAGCombine(ISD::MUL);
216 setTargetDAGCombine(ISD::SHL);
217
218 // Combine vector mask reductions into alltrue/anytrue
219 setTargetDAGCombine(ISD::SETCC);
220
221 // Convert vector to integer bitcasts to bitmask
222 setTargetDAGCombine(ISD::BITCAST);
223
224 // Hoist bitcasts out of shuffles
225 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
226
227 // Combine extends of extract_subvectors into widening ops
228 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND});
229
230 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
231 // conversions ops
232 setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND,
233 ISD::EXTRACT_SUBVECTOR});
234
235 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
236 // into conversion ops
237 setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
238 ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_ROUND,
239 ISD::CONCAT_VECTORS});
240
241 setTargetDAGCombine(ISD::TRUNCATE);
242
243 // Support saturating add/sub for i8x16 and i16x8
244 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
245 for (auto T : {MVT::v16i8, MVT::v8i16})
246 setOperationAction(Op, VT: T, Action: Legal);
247
248 // Support integer abs
249 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
250 setOperationAction(Op: ISD::ABS, VT: T, Action: Legal);
251
252 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
253 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
254 MVT::v2f64})
255 setOperationAction(Op: ISD::BUILD_VECTOR, VT: T, Action: Custom);
256
257 if (Subtarget->hasFP16()) {
258 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
259 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::f16, Action: Custom);
260 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::v4f16, Action: Custom);
261 }
262
263 // We have custom shuffle lowering to expose the shuffle mask
264 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
265 MVT::v2f64})
266 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: T, Action: Custom);
267
268 if (Subtarget->hasFP16())
269 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v8f16, Action: Custom);
270
271 // Support splatting
272 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
273 MVT::v2f64})
274 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: T, Action: Legal);
275
276 setOperationAction(Ops: ISD::AVGCEILU, VTs: {MVT::v8i16, MVT::v16i8}, Action: Legal);
277
278 // Custom lowering since wasm shifts must have a scalar shift amount
279 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
280 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
281 setOperationAction(Op, VT: T, Action: Custom);
282
283 // Custom lower lane accesses to expand out variable indices
284 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
285 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
286 MVT::v2f64})
287 setOperationAction(Op, VT: T, Action: Custom);
288
289 // There is no i8x16.mul instruction
290 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Expand);
291
292 // Expand integer operations supported for scalars but not SIMD
293 for (auto Op :
294 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
295 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
296 setOperationAction(Op, VT: T, Action: Expand);
297
298 // But we do have integer min and max operations
299 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
300 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
301 setOperationAction(Op, VT: T, Action: Legal);
302
303 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
304 setOperationAction(Op: ISD::CTPOP, VT: MVT::v16i8, Action: Legal);
305 setOperationAction(Op: ISD::CTLZ, VT: MVT::v16i8, Action: Expand);
306 setOperationAction(Op: ISD::CTTZ, VT: MVT::v16i8, Action: Expand);
307
308 // Custom lower bit counting operations for other types to scalarize them.
309 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
310 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
311 setOperationAction(Op, VT: T, Action: Custom);
312
313 // Expand float operations supported for scalars but not SIMD
314 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
315 ISD::FEXP, ISD::FEXP2, ISD::FEXP10})
316 for (auto T : {MVT::v4f32, MVT::v2f64})
317 setOperationAction(Op, VT: T, Action: Expand);
318
319 // Unsigned comparison operations are unavailable for i64x2 vectors.
320 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
321 setCondCodeAction(CCs: CC, VT: MVT::v2i64, Action: Custom);
322
323 // 64x2 conversions are not in the spec
324 for (auto Op :
325 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
326 for (auto T : {MVT::v2i64, MVT::v2f64})
327 setOperationAction(Op, VT: T, Action: Expand);
328
329 // But saturating fp_to_int conversions are
330 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) {
331 setOperationAction(Op, VT: MVT::v4i32, Action: Custom);
332 if (Subtarget->hasFP16()) {
333 setOperationAction(Op, VT: MVT::v8i16, Action: Custom);
334 }
335 }
336
337 // Support vector extending
338 for (auto T : MVT::integer_fixedlen_vector_valuetypes()) {
339 setOperationAction(Op: ISD::ANY_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
340 setOperationAction(Op: ISD::SIGN_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
341 setOperationAction(Op: ISD::ZERO_EXTEND_VECTOR_INREG, VT: T, Action: Custom);
342 }
343
344 if (Subtarget->hasFP16()) {
345 setOperationAction(Op: ISD::FMA, VT: MVT::v8f16, Action: Legal);
346 }
347
348 if (Subtarget->hasRelaxedSIMD()) {
349 setOperationAction(Op: ISD::FMULADD, VT: MVT::v4f32, Action: Legal);
350 setOperationAction(Op: ISD::FMULADD, VT: MVT::v2f64, Action: Legal);
351 }
352
353 // Partial MLA reductions.
354 for (auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {
355 setPartialReduceMLAAction(Opc: Op, AccVT: MVT::v4i32, InputVT: MVT::v16i8, Action: Legal);
356 setPartialReduceMLAAction(Opc: Op, AccVT: MVT::v4i32, InputVT: MVT::v8i16, Action: Legal);
357 }
358 }
359
360 // As a special case, these operators use the type to mean the type to
361 // sign-extend from.
362 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
363 if (!Subtarget->hasSignExt()) {
364 // Sign extends are legal only when extending a vector extract
365 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
366 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
367 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action);
368 }
369 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
370 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: T, Action: Expand);
371
372 // Dynamic stack allocation: use the default expansion.
373 setOperationAction(Op: ISD::STACKSAVE, VT: MVT::Other, Action: Expand);
374 setOperationAction(Op: ISD::STACKRESTORE, VT: MVT::Other, Action: Expand);
375 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVTPtr, Action: Expand);
376
377 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i32, Action: Custom);
378 setOperationAction(Op: ISD::FrameIndex, VT: MVT::i64, Action: Custom);
379 setOperationAction(Op: ISD::CopyToReg, VT: MVT::Other, Action: Custom);
380
381 // Expand these forms; we pattern-match the forms that we can handle in isel.
382 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
383 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
384 setOperationAction(Op, VT: T, Action: Expand);
385
386 if (Subtarget->hasReferenceTypes())
387 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
388 for (auto T : {MVT::externref, MVT::funcref})
389 setOperationAction(Op, VT: T, Action: Expand);
390
391 // There is no vector conditional select instruction
392 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
393 MVT::v2f64, MVT::v8f16})
394 setOperationAction(Op: ISD::SELECT_CC, VT: T, Action: Expand);
395
396 // We have custom switch handling.
397 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Custom);
398
399 // WebAssembly doesn't have:
400 // - Floating-point extending loads.
401 // - Floating-point truncating stores.
402 // - i1 extending loads.
403 // - truncating SIMD stores and most extending loads
404 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
405 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
406 for (auto T : MVT::integer_valuetypes())
407 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
408 setLoadExtAction(ExtType: Ext, ValVT: T, MemVT: MVT::i1, Action: Promote);
409 if (Subtarget->hasSIMD128()) {
410 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
411 MVT::v2f64}) {
412 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
413 if (MVT(T) != MemT) {
414 setTruncStoreAction(ValVT: T, MemVT: MemT, Action: Expand);
415 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
416 setLoadExtAction(ExtType: Ext, ValVT: T, MemVT: MemT, Action: Expand);
417 }
418 }
419 }
420 // But some vector extending loads are legal
421 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
422 setLoadExtAction(ExtType: Ext, ValVT: MVT::v8i16, MemVT: MVT::v8i8, Action: Legal);
423 setLoadExtAction(ExtType: Ext, ValVT: MVT::v4i32, MemVT: MVT::v4i16, Action: Legal);
424 setLoadExtAction(ExtType: Ext, ValVT: MVT::v2i64, MemVT: MVT::v2i32, Action: Legal);
425 }
426 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::v2f64, MemVT: MVT::v2f32, Action: Legal);
427 }
428
429 // Don't do anything clever with build_pairs
430 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
431
432 // Trap lowers to wasm unreachable
433 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
434 setOperationAction(Op: ISD::DEBUGTRAP, VT: MVT::Other, Action: Legal);
435
436 // Exception handling intrinsics
437 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
438 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
439 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
440
441 setMaxAtomicSizeInBitsSupported(64);
442
443 // Always convert switches to br_tables unless there is only one case, which
444 // is equivalent to a simple branch. This reduces code size for wasm, and we
445 // defer possible jump table optimizations to the VM.
446 setMinimumJumpTableEntries(2);
447}
448
449TargetLowering::AtomicExpansionKind
450WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(
451 const AtomicRMWInst *AI) const {
452 // We have wasm instructions for these
453 switch (AI->getOperation()) {
454 case AtomicRMWInst::Add:
455 case AtomicRMWInst::Sub:
456 case AtomicRMWInst::And:
457 case AtomicRMWInst::Or:
458 case AtomicRMWInst::Xor:
459 case AtomicRMWInst::Xchg:
460 return AtomicExpansionKind::None;
461 default:
462 break;
463 }
464 return AtomicExpansionKind::CmpXChg;
465}
466
467bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
468 // Implementation copied from X86TargetLowering.
469 unsigned Opc = VecOp.getOpcode();
470
471 // Assume target opcodes can't be scalarized.
472 // TODO - do we have any exceptions?
473 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
474 return false;
475
476 // If the vector op is not supported, try to convert to scalar.
477 EVT VecVT = VecOp.getValueType();
478 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
479 return true;
480
481 // If the vector op is supported, but the scalar op is not, the transform may
482 // not be worthwhile.
483 EVT ScalarVT = VecVT.getScalarType();
484 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT);
485}
486
487FastISel *WebAssemblyTargetLowering::createFastISel(
488 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
489 const LibcallLoweringInfo *LibcallLowering) const {
490 return WebAssembly::createFastISel(funcInfo&: FuncInfo, libInfo: LibInfo, libcallLowering: LibcallLowering);
491}
492
493MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
494 EVT VT) const {
495 unsigned BitWidth = NextPowerOf2(A: VT.getSizeInBits() - 1);
496 if (BitWidth > 1 && BitWidth < 8)
497 BitWidth = 8;
498
499 if (BitWidth > 64) {
500 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
501 // the count to be an i32.
502 BitWidth = 32;
503 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&
504 "32-bit shift counts ought to be enough for anyone");
505 }
506
507 MVT Result = MVT::getIntegerVT(BitWidth);
508 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&
509 "Unable to represent scalar shift amount type");
510 return Result;
511}
512
513// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
514// undefined result on invalid/overflow, to the WebAssembly opcode, which
515// traps on invalid/overflow.
516static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
517 MachineBasicBlock *BB,
518 const TargetInstrInfo &TII,
519 bool IsUnsigned, bool Int64,
520 bool Float64, unsigned LoweredOpcode) {
521 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
522
523 Register OutReg = MI.getOperand(i: 0).getReg();
524 Register InReg = MI.getOperand(i: 1).getReg();
525
526 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
527 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
528 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
529 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
530 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
531 unsigned Eqz = WebAssembly::EQZ_I32;
532 unsigned And = WebAssembly::AND_I32;
533 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
534 int64_t Substitute = IsUnsigned ? 0 : Limit;
535 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
536 auto &Context = BB->getParent()->getFunction().getContext();
537 Type *Ty = Float64 ? Type::getDoubleTy(C&: Context) : Type::getFloatTy(C&: Context);
538
539 const BasicBlock *LLVMBB = BB->getBasicBlock();
540 MachineFunction *F = BB->getParent();
541 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
542 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
543 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
544
545 MachineFunction::iterator It = ++BB->getIterator();
546 F->insert(MBBI: It, MBB: FalseMBB);
547 F->insert(MBBI: It, MBB: TrueMBB);
548 F->insert(MBBI: It, MBB: DoneMBB);
549
550 // Transfer the remainder of BB and its successor edges to DoneMBB.
551 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
552 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
553
554 BB->addSuccessor(Succ: TrueMBB);
555 BB->addSuccessor(Succ: FalseMBB);
556 TrueMBB->addSuccessor(Succ: DoneMBB);
557 FalseMBB->addSuccessor(Succ: DoneMBB);
558
559 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
560 Tmp0 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
561 Tmp1 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
562 CmpReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
563 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
564 FalseReg = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: OutReg));
565 TrueReg = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: OutReg));
566
567 MI.eraseFromParent();
568 // For signed numbers, we can do a single comparison to determine whether
569 // fabs(x) is within range.
570 if (IsUnsigned) {
571 Tmp0 = InReg;
572 } else {
573 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Abs), DestReg: Tmp0).addReg(RegNo: InReg);
574 }
575 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: FConst), DestReg: Tmp1)
576 .addFPImm(Val: cast<ConstantFP>(Val: ConstantFP::get(Ty, V: CmpVal)));
577 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: LT), DestReg: CmpReg).addReg(RegNo: Tmp0).addReg(RegNo: Tmp1);
578
579 // For unsigned numbers, we have to do a separate comparison with zero.
580 if (IsUnsigned) {
581 Tmp1 = MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: InReg));
582 Register SecondCmpReg =
583 MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
584 Register AndReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
585 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: FConst), DestReg: Tmp1)
586 .addFPImm(Val: cast<ConstantFP>(Val: ConstantFP::get(Ty, V: 0.0)));
587 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: GE), DestReg: SecondCmpReg).addReg(RegNo: Tmp0).addReg(RegNo: Tmp1);
588 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: And), DestReg: AndReg).addReg(RegNo: CmpReg).addReg(RegNo: SecondCmpReg);
589 CmpReg = AndReg;
590 }
591
592 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).addReg(RegNo: CmpReg);
593
594 // Create the CFG diamond to select between doing the conversion or using
595 // the substitute value.
596 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: TrueMBB).addReg(RegNo: EqzReg);
597 BuildMI(BB: FalseMBB, MIMD: DL, MCID: TII.get(Opcode: LoweredOpcode), DestReg: FalseReg).addReg(RegNo: InReg);
598 BuildMI(BB: FalseMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
599 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: IConst), DestReg: TrueReg).addImm(Val: Substitute);
600 BuildMI(BB&: *DoneMBB, I: DoneMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: TargetOpcode::PHI), DestReg: OutReg)
601 .addReg(RegNo: FalseReg)
602 .addMBB(MBB: FalseMBB)
603 .addReg(RegNo: TrueReg)
604 .addMBB(MBB: TrueMBB);
605
606 return DoneMBB;
607}
608
609// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`
610// instruction to handle the zero-length case.
611static MachineBasicBlock *LowerMemcpy(MachineInstr &MI, DebugLoc DL,
612 MachineBasicBlock *BB,
613 const TargetInstrInfo &TII, bool Int64) {
614 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
615
616 MachineOperand DstMem = MI.getOperand(i: 0);
617 MachineOperand SrcMem = MI.getOperand(i: 1);
618 MachineOperand Dst = MI.getOperand(i: 2);
619 MachineOperand Src = MI.getOperand(i: 3);
620 MachineOperand Len = MI.getOperand(i: 4);
621
622 // If the length is a constant, we don't actually need the check.
623 if (MachineInstr *Def = MRI.getVRegDef(Reg: Len.getReg())) {
624 if (Def->getOpcode() == WebAssembly::CONST_I32 ||
625 Def->getOpcode() == WebAssembly::CONST_I64) {
626 if (Def->getOperand(i: 1).getImm() == 0) {
627 // A zero-length memcpy is a no-op.
628 MI.eraseFromParent();
629 return BB;
630 }
631 // A non-zero-length memcpy doesn't need a zero check.
632 unsigned MemoryCopy =
633 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
634 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: MemoryCopy))
635 .add(MO: DstMem)
636 .add(MO: SrcMem)
637 .add(MO: Dst)
638 .add(MO: Src)
639 .add(MO: Len);
640 MI.eraseFromParent();
641 return BB;
642 }
643 }
644
645 // We're going to add an extra use to `Len` to test if it's zero; that
646 // use shouldn't be a kill, even if the original use is.
647 MachineOperand NoKillLen = Len;
648 NoKillLen.setIsKill(false);
649
650 // Decide on which `MachineInstr` opcode we're going to use.
651 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
652 unsigned MemoryCopy =
653 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
654
655 // Create two new basic blocks; one for the new `memory.fill` that we can
656 // branch over, and one for the rest of the instructions after the original
657 // `memory.fill`.
658 const BasicBlock *LLVMBB = BB->getBasicBlock();
659 MachineFunction *F = BB->getParent();
660 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
661 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
662
663 MachineFunction::iterator It = ++BB->getIterator();
664 F->insert(MBBI: It, MBB: TrueMBB);
665 F->insert(MBBI: It, MBB: DoneMBB);
666
667 // Transfer the remainder of BB and its successor edges to DoneMBB.
668 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
669 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
670
671 // Connect the CFG edges.
672 BB->addSuccessor(Succ: TrueMBB);
673 BB->addSuccessor(Succ: DoneMBB);
674 TrueMBB->addSuccessor(Succ: DoneMBB);
675
676 // Create a virtual register for the `Eqz` result.
677 unsigned EqzReg;
678 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
679
680 // Erase the original `memory.copy`.
681 MI.eraseFromParent();
682
683 // Test if `Len` is zero.
684 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).add(MO: NoKillLen);
685
686 // Insert a new `memory.copy`.
687 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: MemoryCopy))
688 .add(MO: DstMem)
689 .add(MO: SrcMem)
690 .add(MO: Dst)
691 .add(MO: Src)
692 .add(MO: Len);
693
694 // Create the CFG triangle.
695 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: DoneMBB).addReg(RegNo: EqzReg);
696 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
697
698 return DoneMBB;
699}
700
701// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`
702// instruction to handle the zero-length case.
703static MachineBasicBlock *LowerMemset(MachineInstr &MI, DebugLoc DL,
704 MachineBasicBlock *BB,
705 const TargetInstrInfo &TII, bool Int64) {
706 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
707
708 MachineOperand Mem = MI.getOperand(i: 0);
709 MachineOperand Dst = MI.getOperand(i: 1);
710 MachineOperand Val = MI.getOperand(i: 2);
711 MachineOperand Len = MI.getOperand(i: 3);
712
713 // If the length is a constant, we don't actually need the check.
714 if (MachineInstr *Def = MRI.getVRegDef(Reg: Len.getReg())) {
715 if (Def->getOpcode() == WebAssembly::CONST_I32 ||
716 Def->getOpcode() == WebAssembly::CONST_I64) {
717 if (Def->getOperand(i: 1).getImm() == 0) {
718 // A zero-length memset is a no-op.
719 MI.eraseFromParent();
720 return BB;
721 }
722 // A non-zero-length memset doesn't need a zero check.
723 unsigned MemoryFill =
724 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
725 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: MemoryFill))
726 .add(MO: Mem)
727 .add(MO: Dst)
728 .add(MO: Val)
729 .add(MO: Len);
730 MI.eraseFromParent();
731 return BB;
732 }
733 }
734
735 // We're going to add an extra use to `Len` to test if it's zero; that
736 // use shouldn't be a kill, even if the original use is.
737 MachineOperand NoKillLen = Len;
738 NoKillLen.setIsKill(false);
739
740 // Decide on which `MachineInstr` opcode we're going to use.
741 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
742 unsigned MemoryFill =
743 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
744
745 // Create two new basic blocks; one for the new `memory.fill` that we can
746 // branch over, and one for the rest of the instructions after the original
747 // `memory.fill`.
748 const BasicBlock *LLVMBB = BB->getBasicBlock();
749 MachineFunction *F = BB->getParent();
750 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
751 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB: LLVMBB);
752
753 MachineFunction::iterator It = ++BB->getIterator();
754 F->insert(MBBI: It, MBB: TrueMBB);
755 F->insert(MBBI: It, MBB: DoneMBB);
756
757 // Transfer the remainder of BB and its successor edges to DoneMBB.
758 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB, From: std::next(x: MI.getIterator()), To: BB->end());
759 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
760
761 // Connect the CFG edges.
762 BB->addSuccessor(Succ: TrueMBB);
763 BB->addSuccessor(Succ: DoneMBB);
764 TrueMBB->addSuccessor(Succ: DoneMBB);
765
766 // Create a virtual register for the `Eqz` result.
767 unsigned EqzReg;
768 EqzReg = MRI.createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
769
770 // Erase the original `memory.fill`.
771 MI.eraseFromParent();
772
773 // Test if `Len` is zero.
774 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: Eqz), DestReg: EqzReg).add(MO: NoKillLen);
775
776 // Insert a new `memory.copy`.
777 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: MemoryFill)).add(MO: Mem).add(MO: Dst).add(MO: Val).add(MO: Len);
778
779 // Create the CFG triangle.
780 BuildMI(BB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR_IF)).addMBB(MBB: DoneMBB).addReg(RegNo: EqzReg);
781 BuildMI(BB: TrueMBB, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::BR)).addMBB(MBB: DoneMBB);
782
783 return DoneMBB;
784}
785
786static MachineBasicBlock *
787LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
788 const WebAssemblySubtarget *Subtarget,
789 const TargetInstrInfo &TII) {
790 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
791 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
792 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
793 MachineInstr &CallParams = *CallResults.getPrevNode();
794 assert(CallParams.getOpcode() ==
795 (IsRetCall ? WebAssembly::RET_CALL_PARAMS : WebAssembly::CALL_PARAMS));
796
797 bool IsIndirect =
798 CallParams.getOperand(i: 0).isReg() || CallParams.getOperand(i: 0).isFI();
799
800 bool IsFuncrefCall = false;
801 if (IsIndirect && CallParams.getOperand(i: 0).isReg()) {
802 Register Reg = CallParams.getOperand(i: 0).getReg();
803 const MachineFunction *MF = BB->getParent();
804 const MachineRegisterInfo &MRI = MF->getRegInfo();
805 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
806 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
807 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
808 }
809
810 unsigned CallOp;
811 if (IsIndirect && IsRetCall) {
812 CallOp = WebAssembly::RET_CALL_INDIRECT;
813 } else if (IsIndirect) {
814 CallOp = WebAssembly::CALL_INDIRECT;
815 } else if (IsRetCall) {
816 CallOp = WebAssembly::RET_CALL;
817 } else {
818 CallOp = WebAssembly::CALL;
819 }
820
821 MachineFunction &MF = *BB->getParent();
822 const MCInstrDesc &MCID = TII.get(Opcode: CallOp);
823 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
824
825 // Move the function pointer to the end of the arguments for indirect calls
826 if (IsIndirect) {
827 auto FnPtr = CallParams.getOperand(i: 0);
828 CallParams.removeOperand(OpNo: 0);
829
830 // For funcrefs, call_indirect is done through __funcref_call_table and the
831 // funcref is always installed in slot 0 of the table, therefore instead of
832 // having the function pointer added at the end of the params list, a zero
833 // (the index in
834 // __funcref_call_table is added).
835 if (IsFuncrefCall) {
836 Register RegZero =
837 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
838 MachineInstrBuilder MIBC0 =
839 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::CONST_I32), DestReg: RegZero).addImm(Val: 0);
840
841 BB->insert(I: CallResults.getIterator(), M: MIBC0);
842 MachineInstrBuilder(MF, CallParams).addReg(RegNo: RegZero);
843 } else
844 CallParams.addOperand(Op: FnPtr);
845 }
846
847 for (auto Def : CallResults.defs())
848 MIB.add(MO: Def);
849
850 if (IsIndirect) {
851 // Placeholder for the type index.
852 // This gets replaced with the correct value in WebAssemblyMCInstLower.cpp
853 MIB.addImm(Val: 0);
854 // The table into which this call_indirect indexes.
855 MCSymbolWasm *Table = IsFuncrefCall
856 ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
857 Ctx&: MF.getContext(), Subtarget)
858 : WebAssembly::getOrCreateFunctionTableSymbol(
859 Ctx&: MF.getContext(), Subtarget);
860 if (Subtarget->hasCallIndirectOverlong()) {
861 MIB.addSym(Sym: Table);
862 } else {
863 // For the MVP there is at most one table whose number is 0, but we can't
864 // write a table symbol or issue relocations. Instead we just ensure the
865 // table is live and write a zero.
866 Table->setNoStrip();
867 MIB.addImm(Val: 0);
868 }
869 }
870
871 // Avoid duplicating the implicit operands.
872 for (auto Use : CallParams.explicit_uses())
873 MIB.add(MO: Use);
874
875 BB->insert(I: CallResults.getIterator(), M: MIB);
876 CallParams.eraseFromParent();
877 CallResults.eraseFromParent();
878
879 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
880 // table slot with ref.null upon call_indirect return.
881 //
882 // This generates the following code, which comes right after a call_indirect
883 // of a funcref:
884 //
885 // i32.const 0
886 // ref.null func
887 // table.set __funcref_call_table
888 if (IsIndirect && IsFuncrefCall) {
889 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
890 Ctx&: MF.getContext(), Subtarget);
891 Register RegZero =
892 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::I32RegClass);
893 MachineInstr *Const0 =
894 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::CONST_I32), DestReg: RegZero).addImm(Val: 0);
895 BB->insertAfter(I: MIB.getInstr()->getIterator(), MI: Const0);
896
897 Register RegFuncref =
898 MF.getRegInfo().createVirtualRegister(RegClass: &WebAssembly::FUNCREFRegClass);
899 MachineInstr *RefNull =
900 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::REF_NULL_FUNCREF), DestReg: RegFuncref);
901 BB->insertAfter(I: Const0->getIterator(), MI: RefNull);
902
903 MachineInstr *TableSet =
904 BuildMI(MF, MIMD: DL, MCID: TII.get(Opcode: WebAssembly::TABLE_SET_FUNCREF))
905 .addSym(Sym: Table)
906 .addReg(RegNo: RegZero)
907 .addReg(RegNo: RegFuncref);
908 BB->insertAfter(I: RefNull->getIterator(), MI: TableSet);
909 }
910
911 return BB;
912}
913
914MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
915 MachineInstr &MI, MachineBasicBlock *BB) const {
916 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
917 DebugLoc DL = MI.getDebugLoc();
918
919 switch (MI.getOpcode()) {
920 default:
921 llvm_unreachable("Unexpected instr type to insert");
922 case WebAssembly::FP_TO_SINT_I32_F32:
923 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: false, Float64: false,
924 LoweredOpcode: WebAssembly::I32_TRUNC_S_F32);
925 case WebAssembly::FP_TO_UINT_I32_F32:
926 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: false, Float64: false,
927 LoweredOpcode: WebAssembly::I32_TRUNC_U_F32);
928 case WebAssembly::FP_TO_SINT_I64_F32:
929 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: true, Float64: false,
930 LoweredOpcode: WebAssembly::I64_TRUNC_S_F32);
931 case WebAssembly::FP_TO_UINT_I64_F32:
932 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: true, Float64: false,
933 LoweredOpcode: WebAssembly::I64_TRUNC_U_F32);
934 case WebAssembly::FP_TO_SINT_I32_F64:
935 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: false, Float64: true,
936 LoweredOpcode: WebAssembly::I32_TRUNC_S_F64);
937 case WebAssembly::FP_TO_UINT_I32_F64:
938 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: false, Float64: true,
939 LoweredOpcode: WebAssembly::I32_TRUNC_U_F64);
940 case WebAssembly::FP_TO_SINT_I64_F64:
941 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: false, Int64: true, Float64: true,
942 LoweredOpcode: WebAssembly::I64_TRUNC_S_F64);
943 case WebAssembly::FP_TO_UINT_I64_F64:
944 return LowerFPToInt(MI, DL, BB, TII, IsUnsigned: true, Int64: true, Float64: true,
945 LoweredOpcode: WebAssembly::I64_TRUNC_U_F64);
946 case WebAssembly::MEMCPY_A32:
947 return LowerMemcpy(MI, DL, BB, TII, Int64: false);
948 case WebAssembly::MEMCPY_A64:
949 return LowerMemcpy(MI, DL, BB, TII, Int64: true);
950 case WebAssembly::MEMSET_A32:
951 return LowerMemset(MI, DL, BB, TII, Int64: false);
952 case WebAssembly::MEMSET_A64:
953 return LowerMemset(MI, DL, BB, TII, Int64: true);
954 case WebAssembly::CALL_RESULTS:
955 case WebAssembly::RET_CALL_RESULTS:
956 return LowerCallResults(CallResults&: MI, DL, BB, Subtarget, TII);
957 }
958}
959
960std::pair<unsigned, const TargetRegisterClass *>
961WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
962 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
963 // First, see if this is a constraint that directly corresponds to a
964 // WebAssembly register class.
965 if (Constraint.size() == 1) {
966 switch (Constraint[0]) {
967 case 'r':
968 assert(VT != MVT::iPTR && "Pointer MVT not expected here");
969 if (Subtarget->hasSIMD128() && VT.isVector()) {
970 if (VT.getSizeInBits() == 128)
971 return std::make_pair(x: 0U, y: &WebAssembly::V128RegClass);
972 }
973 if (VT.isInteger() && !VT.isVector()) {
974 if (VT.getSizeInBits() <= 32)
975 return std::make_pair(x: 0U, y: &WebAssembly::I32RegClass);
976 if (VT.getSizeInBits() <= 64)
977 return std::make_pair(x: 0U, y: &WebAssembly::I64RegClass);
978 }
979 if (VT.isFloatingPoint() && !VT.isVector()) {
980 switch (VT.getSizeInBits()) {
981 case 32:
982 return std::make_pair(x: 0U, y: &WebAssembly::F32RegClass);
983 case 64:
984 return std::make_pair(x: 0U, y: &WebAssembly::F64RegClass);
985 default:
986 break;
987 }
988 }
989 break;
990 default:
991 break;
992 }
993 }
994
995 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
996}
997
998bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
999 // Assume ctz is a relatively cheap operation.
1000 return true;
1001}
1002
1003bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
1004 // Assume clz is a relatively cheap operation.
1005 return true;
1006}
1007
1008bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1009 const AddrMode &AM,
1010 Type *Ty, unsigned AS,
1011 Instruction *I) const {
1012 // WebAssembly offsets are added as unsigned without wrapping. The
1013 // isLegalAddressingMode gives us no way to determine if wrapping could be
1014 // happening, so we approximate this by accepting only non-negative offsets.
1015 if (AM.BaseOffs < 0)
1016 return false;
1017
1018 // WebAssembly has no scale register operands.
1019 if (AM.Scale != 0)
1020 return false;
1021
1022 // Everything else is legal.
1023 return true;
1024}
1025
1026bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
1027 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
1028 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
1029 // WebAssembly supports unaligned accesses, though it should be declared
1030 // with the p2align attribute on loads and stores which do so, and there
1031 // may be a performance impact. We tell LLVM they're "fast" because
1032 // for the kinds of things that LLVM uses this for (merging adjacent stores
1033 // of constants, etc.), WebAssembly implementations will either want the
1034 // unaligned access or they'll split anyway.
1035 if (Fast)
1036 *Fast = 1;
1037 return true;
1038}
1039
1040bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
1041 AttributeList Attr) const {
1042 // The current thinking is that wasm engines will perform this optimization,
1043 // so we can save on code size.
1044 return true;
1045}
1046
1047bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
1048 EVT ExtT = ExtVal.getValueType();
1049 SDValue N0 = peekThroughFreeze(V: ExtVal->getOperand(Num: 0));
1050 auto *Load = dyn_cast<LoadSDNode>(Val&: N0);
1051 if (!Load)
1052 return false;
1053 EVT MemT = Load->getValueType(ResNo: 0);
1054 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
1055 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
1056 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
1057}
1058
1059bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
1060 const GlobalAddressSDNode *GA) const {
1061 // Wasm doesn't support function addresses with offsets
1062 const GlobalValue *GV = GA->getGlobal();
1063 return isa<Function>(Val: GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
1064}
1065
1066EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
1067 LLVMContext &C,
1068 EVT VT) const {
1069 if (VT.isVector()) {
1070 if (VT.getVectorElementType() == MVT::f16 && !Subtarget->hasFP16())
1071 return VT.changeElementType(Context&: C, EltVT: MVT::i1);
1072
1073 return VT.changeVectorElementTypeToInteger();
1074 }
1075
1076 // So far, all branch instructions in Wasm take an I32 condition.
1077 // The default TargetLowering::getSetCCResultType returns the pointer size,
1078 // which would be useful to reduce instruction counts when testing
1079 // against 64-bit pointers/values if at some point Wasm supports that.
1080 return EVT::getIntegerVT(Context&: C, BitWidth: 32);
1081}
1082
1083void WebAssemblyTargetLowering::getTgtMemIntrinsic(
1084 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
1085 MachineFunction &MF, unsigned Intrinsic) const {
1086 IntrinsicInfo Info;
1087 switch (Intrinsic) {
1088 case Intrinsic::wasm_memory_atomic_notify:
1089 Info.opc = ISD::INTRINSIC_W_CHAIN;
1090 Info.memVT = MVT::i32;
1091 Info.ptrVal = I.getArgOperand(i: 0);
1092 Info.offset = 0;
1093 Info.align = Align(4);
1094 // atomic.notify instruction does not really load the memory specified with
1095 // this argument, but MachineMemOperand should either be load or store, so
1096 // we set this to a load.
1097 // FIXME Volatile isn't really correct, but currently all LLVM atomic
1098 // instructions are treated as volatiles in the backend, so we should be
1099 // consistent. The same applies for wasm_atomic_wait intrinsics too.
1100 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1101 Infos.push_back(Elt: Info);
1102 return;
1103 case Intrinsic::wasm_memory_atomic_wait32:
1104 Info.opc = ISD::INTRINSIC_W_CHAIN;
1105 Info.memVT = MVT::i32;
1106 Info.ptrVal = I.getArgOperand(i: 0);
1107 Info.offset = 0;
1108 Info.align = Align(4);
1109 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1110 Infos.push_back(Elt: Info);
1111 return;
1112 case Intrinsic::wasm_memory_atomic_wait64:
1113 Info.opc = ISD::INTRINSIC_W_CHAIN;
1114 Info.memVT = MVT::i64;
1115 Info.ptrVal = I.getArgOperand(i: 0);
1116 Info.offset = 0;
1117 Info.align = Align(8);
1118 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
1119 Infos.push_back(Elt: Info);
1120 return;
1121 case Intrinsic::wasm_loadf16_f32:
1122 Info.opc = ISD::INTRINSIC_W_CHAIN;
1123 Info.memVT = MVT::f16;
1124 Info.ptrVal = I.getArgOperand(i: 0);
1125 Info.offset = 0;
1126 Info.align = Align(2);
1127 Info.flags = MachineMemOperand::MOLoad;
1128 Infos.push_back(Elt: Info);
1129 return;
1130 case Intrinsic::wasm_storef16_f32:
1131 Info.opc = ISD::INTRINSIC_VOID;
1132 Info.memVT = MVT::f16;
1133 Info.ptrVal = I.getArgOperand(i: 1);
1134 Info.offset = 0;
1135 Info.align = Align(2);
1136 Info.flags = MachineMemOperand::MOStore;
1137 Infos.push_back(Elt: Info);
1138 return;
1139 default:
1140 return;
1141 }
1142}
1143
1144void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
1145 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
1146 const SelectionDAG &DAG, unsigned Depth) const {
1147 switch (Op.getOpcode()) {
1148 default:
1149 break;
1150 case ISD::INTRINSIC_WO_CHAIN: {
1151 unsigned IntNo = Op.getConstantOperandVal(i: 0);
1152 switch (IntNo) {
1153 default:
1154 break;
1155 case Intrinsic::wasm_bitmask: {
1156 unsigned BitWidth = Known.getBitWidth();
1157 EVT VT = Op.getOperand(i: 1).getSimpleValueType();
1158 unsigned PossibleBits = VT.getVectorNumElements();
1159 APInt ZeroMask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - PossibleBits);
1160 Known.Zero |= ZeroMask;
1161 break;
1162 }
1163 }
1164 break;
1165 }
1166 case WebAssemblyISD::EXTEND_LOW_U:
1167 case WebAssemblyISD::EXTEND_HIGH_U: {
1168 // We know the high half, of each destination vector element, will be zero.
1169 SDValue SrcOp = Op.getOperand(i: 0);
1170 EVT VT = SrcOp.getSimpleValueType();
1171 unsigned BitWidth = Known.getBitWidth();
1172 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1173 assert(BitWidth >= 8 && "Unexpected width!");
1174 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 8);
1175 Known.Zero |= Mask;
1176 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1177 assert(BitWidth >= 16 && "Unexpected width!");
1178 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 16);
1179 Known.Zero |= Mask;
1180 } else if (VT == MVT::v2i32 || VT == MVT::v4i32) {
1181 assert(BitWidth >= 32 && "Unexpected width!");
1182 APInt Mask = APInt::getHighBitsSet(numBits: BitWidth, hiBitsSet: BitWidth - 32);
1183 Known.Zero |= Mask;
1184 }
1185 break;
1186 }
1187 // For 128-bit addition if the upper bits are all zero then it's known that
1188 // the upper bits of the result will have all bits guaranteed zero except the
1189 // first.
1190 case WebAssemblyISD::I64_ADD128:
1191 if (Op.getResNo() == 1) {
1192 SDValue LHS_HI = Op.getOperand(i: 1);
1193 SDValue RHS_HI = Op.getOperand(i: 3);
1194 if (isNullConstant(V: LHS_HI) && isNullConstant(V: RHS_HI))
1195 Known.Zero.setBitsFrom(1);
1196 }
1197 break;
1198 }
1199}
1200
1201TargetLoweringBase::LegalizeTypeAction
1202WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
1203 if (VT.isFixedLengthVector()) {
1204 MVT EltVT = VT.getVectorElementType();
1205 // We have legal vector types with these lane types, so widening the
1206 // vector would let us use some of the lanes directly without having to
1207 // extend or truncate values.
1208 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
1209 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
1210 return TypeWidenVector;
1211 }
1212
1213 return TargetLoweringBase::getPreferredVectorAction(VT);
1214}
1215
1216bool WebAssemblyTargetLowering::isFMAFasterThanFMulAndFAdd(
1217 const MachineFunction &MF, EVT VT) const {
1218 if (!Subtarget->hasFP16() || !VT.isVector())
1219 return false;
1220
1221 EVT ScalarVT = VT.getScalarType();
1222 if (!ScalarVT.isSimple())
1223 return false;
1224
1225 return ScalarVT.getSimpleVT().SimpleTy == MVT::f16;
1226}
1227
1228bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
1229 SDValue Op, const TargetLoweringOpt &TLO) const {
1230 // ISel process runs DAGCombiner after legalization; this step is called
1231 // SelectionDAG optimization phase. This post-legalization combining process
1232 // runs DAGCombiner on each node, and if there was a change to be made,
1233 // re-runs legalization again on it and its user nodes to make sure
1234 // everythiing is in a legalized state.
1235 //
1236 // The legalization calls lowering routines, and we do our custom lowering for
1237 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
1238 // into zeros. But there is a set of routines in DAGCombiner that turns unused
1239 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
1240 // turns unused vector elements into undefs. But this routine does not work
1241 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
1242 // combination can result in a infinite loop, in which undefs are converted to
1243 // zeros in legalization and back to undefs in combining.
1244 //
1245 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
1246 // running for build_vectors.
1247 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
1248 return false;
1249 return true;
1250}
1251
1252//===----------------------------------------------------------------------===//
1253// WebAssembly Lowering private implementation.
1254//===----------------------------------------------------------------------===//
1255
1256//===----------------------------------------------------------------------===//
1257// Lowering Code
1258//===----------------------------------------------------------------------===//
1259
1260static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
1261 MachineFunction &MF = DAG.getMachineFunction();
1262 DAG.getContext()->diagnose(
1263 DI: DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
1264}
1265
1266// Test whether the given calling convention is supported.
1267static bool callingConvSupported(CallingConv::ID CallConv) {
1268 // We currently support the language-independent target-independent
1269 // conventions. We don't yet have a way to annotate calls with properties like
1270 // "cold", and we don't have any call-clobbered registers, so these are mostly
1271 // all handled the same.
1272 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
1273 CallConv == CallingConv::Cold ||
1274 CallConv == CallingConv::PreserveMost ||
1275 CallConv == CallingConv::PreserveAll ||
1276 CallConv == CallingConv::CXX_FAST_TLS ||
1277 CallConv == CallingConv::WASM_EmscriptenInvoke ||
1278 CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail;
1279}
1280
1281SDValue
1282WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
1283 SmallVectorImpl<SDValue> &InVals) const {
1284 SelectionDAG &DAG = CLI.DAG;
1285 SDLoc DL = CLI.DL;
1286 SDValue Chain = CLI.Chain;
1287 SDValue Callee = CLI.Callee;
1288 MachineFunction &MF = DAG.getMachineFunction();
1289 auto Layout = MF.getDataLayout();
1290
1291 // A call through a funcref is expressed in IR as a call through the pointer
1292 // produced by the llvm.wasm.funcref.to_ptr intrinsic. Detect this here and
1293 // recover the underlying funcref value so the call can be lowered to a
1294 // table.set + call_indirect through the dedicated __funcref_call_table.
1295 bool IsFuncrefCall = false;
1296 if (Callee.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1297 Callee.getConstantOperandVal(i: 0) == Intrinsic::wasm_funcref_to_ptr) {
1298 Callee = Callee.getOperand(i: 1);
1299 IsFuncrefCall = true;
1300 }
1301
1302 CallingConv::ID CallConv = CLI.CallConv;
1303 if (!callingConvSupported(CallConv))
1304 fail(DL, DAG,
1305 Msg: "WebAssembly doesn't support language-specific or target-specific "
1306 "calling conventions yet");
1307 if (CLI.IsPatchPoint)
1308 fail(DL, DAG, Msg: "WebAssembly doesn't support patch point yet");
1309
1310 if (CLI.IsTailCall) {
1311 auto NoTail = [&](const char *Msg) {
1312 if (CLI.CB && CLI.CB->isMustTailCall())
1313 fail(DL, DAG, Msg);
1314 CLI.IsTailCall = false;
1315 };
1316
1317 if (!Subtarget->hasTailCall())
1318 NoTail("WebAssembly 'tail-call' feature not enabled");
1319
1320 // Varargs calls cannot be tail calls because the buffer is on the stack
1321 if (CLI.IsVarArg)
1322 NoTail("WebAssembly does not support varargs tail calls");
1323
1324 // Do not tail call unless caller and callee return types match
1325 const Function &F = MF.getFunction();
1326 const TargetMachine &TM = getTargetMachine();
1327 Type *RetTy = F.getReturnType();
1328 SmallVector<MVT, 4> CallerRetTys;
1329 SmallVector<MVT, 4> CalleeRetTys;
1330 computeLegalValueVTs(F, TM, Ty: RetTy, ValueVTs&: CallerRetTys);
1331 computeLegalValueVTs(F, TM, Ty: CLI.RetTy, ValueVTs&: CalleeRetTys);
1332 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1333 std::equal(first1: CallerRetTys.begin(), last1: CallerRetTys.end(),
1334 first2: CalleeRetTys.begin());
1335 if (!TypesMatch)
1336 NoTail("WebAssembly tail call requires caller and callee return types to "
1337 "match");
1338
1339 // If pointers to local stack values are passed, we cannot tail call
1340 if (CLI.CB) {
1341 for (auto &Arg : CLI.CB->args()) {
1342 Value *Val = Arg.get();
1343 // Trace the value back through pointer operations
1344 while (true) {
1345 Value *Src = Val->stripPointerCastsAndAliases();
1346 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: Src))
1347 Src = GEP->getPointerOperand();
1348 if (Val == Src)
1349 break;
1350 Val = Src;
1351 }
1352 if (isa<AllocaInst>(Val)) {
1353 NoTail(
1354 "WebAssembly does not support tail calling with stack arguments");
1355 break;
1356 }
1357 }
1358 }
1359
1360 // A byval argument is copied into this function's stack frame below, and
1361 // a tail call releases that frame before the callee reads the copy.
1362 if (llvm::any_of(Range&: CLI.Outs, P: [](const ISD::OutputArg &Out) {
1363 return Out.Flags.isByVal() && Out.Flags.getByValSize() != 0;
1364 }))
1365 NoTail("WebAssembly does not support tail calling with byval arguments");
1366 }
1367
1368 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1369 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1370 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1371
1372 // The generic code may have added an sret argument. If we're lowering an
1373 // invoke function, the ABI requires that the function pointer be the first
1374 // argument, so we may have to swap the arguments.
1375 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1376 Outs[0].Flags.isSRet()) {
1377 std::swap(a&: Outs[0], b&: Outs[1]);
1378 std::swap(a&: OutVals[0], b&: OutVals[1]);
1379 }
1380
1381 bool HasSwiftSelfArg = false;
1382 bool HasSwiftErrorArg = false;
1383 bool HasSwiftAsyncArg = false;
1384 unsigned NumFixedArgs = 0;
1385 for (unsigned I = 0; I < Outs.size(); ++I) {
1386 const ISD::OutputArg &Out = Outs[I];
1387 SDValue &OutVal = OutVals[I];
1388 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1389 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1390 HasSwiftAsyncArg |= Out.Flags.isSwiftAsync();
1391 if (Out.Flags.isNest())
1392 fail(DL, DAG, Msg: "WebAssembly hasn't implemented nest arguments");
1393 if (Out.Flags.isInAlloca())
1394 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca arguments");
1395 if (Out.Flags.isInConsecutiveRegs())
1396 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs arguments");
1397 if (Out.Flags.isInConsecutiveRegsLast())
1398 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last arguments");
1399 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1400 auto &MFI = MF.getFrameInfo();
1401 int FI = MFI.CreateStackObject(Size: Out.Flags.getByValSize(),
1402 Alignment: Out.Flags.getNonZeroByValAlign(),
1403 /*isSS=*/isSpillSlot: false);
1404 SDValue SizeNode =
1405 DAG.getConstant(Val: Out.Flags.getByValSize(), DL, VT: MVT::i32);
1406 SDValue FINode = DAG.getFrameIndex(FI, VT: getPointerTy(DL: Layout));
1407 Align Alignment = Out.Flags.getNonZeroByValAlign();
1408 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FINode, Src: OutVal, Size: SizeNode, DstAlign: Alignment,
1409 SrcAlign: Alignment,
1410 /*isVolatile*/ isVol: false, /*AlwaysInline=*/false,
1411 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(),
1412 SrcPtrInfo: MachinePointerInfo());
1413 OutVal = FINode;
1414 }
1415 // Count the number of fixed args *after* legalization.
1416 NumFixedArgs += !Out.Flags.isVarArg();
1417 }
1418
1419 bool IsVarArg = CLI.IsVarArg;
1420 auto PtrVT = getPointerTy(DL: Layout);
1421
1422 // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
1423 // (for swifttailcc) swiftasync arguments if there aren't. These additional
1424 // arguments are also added for callee signature. They are necessary to match
1425 // callee and caller signature for indirect call.
1426 if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
1427 Type *PtrTy = PointerType::getUnqual(C&: *DAG.getContext());
1428 if (!HasSwiftSelfArg) {
1429 NumFixedArgs++;
1430 ISD::ArgFlagsTy Flags;
1431 Flags.setSwiftSelf();
1432 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1433 CLI.Outs.push_back(Elt: Arg);
1434 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1435 CLI.OutVals.push_back(Elt: ArgVal);
1436 }
1437 if (!HasSwiftErrorArg) {
1438 NumFixedArgs++;
1439 ISD::ArgFlagsTy Flags;
1440 Flags.setSwiftError();
1441 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1442 CLI.Outs.push_back(Elt: Arg);
1443 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1444 CLI.OutVals.push_back(Elt: ArgVal);
1445 }
1446 if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
1447 NumFixedArgs++;
1448 ISD::ArgFlagsTy Flags;
1449 Flags.setSwiftAsync();
1450 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1451 CLI.Outs.push_back(Elt: Arg);
1452 SDValue ArgVal = DAG.getUNDEF(VT: PtrVT);
1453 CLI.OutVals.push_back(Elt: ArgVal);
1454 }
1455 }
1456
1457 // Analyze operands of the call, assigning locations to each operand.
1458 SmallVector<CCValAssign, 16> ArgLocs;
1459 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1460
1461 if (IsVarArg) {
1462 // Outgoing non-fixed arguments are placed in a buffer. First
1463 // compute their offsets and the total amount of buffer space needed.
1464 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1465 const ISD::OutputArg &Out = Outs[I];
1466 SDValue &Arg = OutVals[I];
1467 EVT VT = Arg.getValueType();
1468 assert(VT != MVT::iPTR && "Legalized args should be concrete");
1469 Type *Ty = VT.getTypeForEVT(Context&: *DAG.getContext());
1470 Align Alignment =
1471 std::max(a: Out.Flags.getNonZeroOrigAlign(), b: Layout.getABITypeAlign(Ty));
1472 unsigned Offset =
1473 CCInfo.AllocateStack(Size: Layout.getTypeAllocSize(Ty), Alignment);
1474 CCInfo.addLoc(V: CCValAssign::getMem(ValNo: ArgLocs.size(), ValVT: VT.getSimpleVT(),
1475 Offset, LocVT: VT.getSimpleVT(),
1476 HTP: CCValAssign::Full));
1477 }
1478 }
1479
1480 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1481
1482 SDValue FINode;
1483 if (IsVarArg && NumBytes) {
1484 // For non-fixed arguments, next emit stores to store the argument values
1485 // to the stack buffer at the offsets computed above.
1486 MaybeAlign StackAlign = Layout.getStackAlignment();
1487 assert(StackAlign && "data layout string is missing stack alignment");
1488 int FI = MF.getFrameInfo().CreateStackObject(Size: NumBytes, Alignment: *StackAlign,
1489 /*isSS=*/isSpillSlot: false);
1490 unsigned ValNo = 0;
1491 SmallVector<SDValue, 8> Chains;
1492 for (SDValue Arg : drop_begin(RangeOrContainer&: OutVals, N: NumFixedArgs)) {
1493 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1494 "ArgLocs should remain in order and only hold varargs args");
1495 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1496 FINode = DAG.getFrameIndex(FI, VT: getPointerTy(DL: Layout));
1497 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: FINode,
1498 N2: DAG.getConstant(Val: Offset, DL, VT: PtrVT));
1499 Chains.push_back(
1500 Elt: DAG.getStore(Chain, dl: DL, Val: Arg, Ptr: Add,
1501 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset)));
1502 }
1503 if (!Chains.empty())
1504 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: Chains);
1505 } else if (IsVarArg) {
1506 FINode = DAG.getIntPtrConstant(Val: 0, DL);
1507 }
1508
1509 if (Callee->getOpcode() == ISD::GlobalAddress) {
1510 // If the callee is a GlobalAddress node (quite common, every direct call
1511 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1512 // doesn't at MO_GOT which is not needed for direct calls.
1513 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Callee);
1514 Callee = DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL,
1515 VT: getPointerTy(DL: DAG.getDataLayout()),
1516 offset: GA->getOffset());
1517 Callee = DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL,
1518 VT: getPointerTy(DL: DAG.getDataLayout()), Operand: Callee);
1519 }
1520
1521 // Compute the operands for the CALLn node.
1522 SmallVector<SDValue, 16> Ops;
1523 Ops.push_back(Elt: Chain);
1524 Ops.push_back(Elt: Callee);
1525
1526 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1527 // isn't reliable.
1528 Ops.append(in_start: OutVals.begin(),
1529 in_end: IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1530 // Add a pointer to the vararg buffer.
1531 if (IsVarArg)
1532 Ops.push_back(Elt: FINode);
1533
1534 SmallVector<EVT, 8> InTys;
1535 for (const auto &In : Ins) {
1536 assert(!In.Flags.isByVal() && "byval is not valid for return values");
1537 assert(!In.Flags.isNest() && "nest is not valid for return values");
1538 if (In.Flags.isInAlloca())
1539 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca return values");
1540 if (In.Flags.isInConsecutiveRegs())
1541 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs return values");
1542 if (In.Flags.isInConsecutiveRegsLast())
1543 fail(DL, DAG,
1544 Msg: "WebAssembly hasn't implemented cons regs last return values");
1545 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1546 // registers.
1547 InTys.push_back(Elt: In.VT);
1548 }
1549
1550 // Lastly, if this is a call to a funcref we need to add an instruction
1551 // table.set to the chain and transform the call.
1552 if (IsFuncrefCall) {
1553 // In the absence of function references proposal where a funcref call is
1554 // lowered to call_ref, using reference types we generate a table.set to set
1555 // the funcref to a special table used solely for this purpose, followed by
1556 // a call_indirect. Here we just generate the table set, and return the
1557 // SDValue of the table.set so that LowerCall can finalize the lowering by
1558 // generating the call_indirect.
1559 SDValue Chain = Ops[0];
1560
1561 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
1562 Ctx&: MF.getContext(), Subtarget);
1563 SDValue Sym = DAG.getMCSymbol(Sym: Table, VT: PtrVT);
1564 SDValue TableSlot = DAG.getConstant(Val: 0, DL, VT: MVT::i32);
1565 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1566 SDValue TableSet = DAG.getMemIntrinsicNode(
1567 Opcode: WebAssemblyISD::TABLE_SET, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: TableSetOps,
1568 MemVT: MVT::funcref, PtrInfo: MachinePointerInfo(), Alignment: Align(1),
1569 Flags: MachineMemOperand::MOStore);
1570
1571 Ops[0] = TableSet; // The new chain is the TableSet itself
1572 }
1573
1574 if (CLI.IsTailCall) {
1575 // ret_calls do not return values to the current frame
1576 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
1577 return DAG.getNode(Opcode: WebAssemblyISD::RET_CALL, DL, VTList: NodeTys, Ops);
1578 }
1579
1580 InTys.push_back(Elt: MVT::Other);
1581 SDVTList InTyList = DAG.getVTList(VTs: InTys);
1582 SDValue Res = DAG.getNode(Opcode: WebAssemblyISD::CALL, DL, VTList: InTyList, Ops);
1583
1584 for (size_t I = 0; I < Ins.size(); ++I)
1585 InVals.push_back(Elt: Res.getValue(R: I));
1586
1587 // Return the chain
1588 return Res.getValue(R: Ins.size());
1589}
1590
1591bool WebAssemblyTargetLowering::CanLowerReturn(
1592 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1593 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext & /*Context*/,
1594 const Type *RetTy) const {
1595 // WebAssembly can only handle returning tuples with multivalue enabled
1596 return WebAssembly::canLowerReturn(ResultSize: Outs.size(), Subtarget);
1597}
1598
1599SDValue WebAssemblyTargetLowering::LowerReturn(
1600 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1601 const SmallVectorImpl<ISD::OutputArg> &Outs,
1602 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1603 SelectionDAG &DAG) const {
1604 assert(WebAssembly::canLowerReturn(Outs.size(), Subtarget) &&
1605 "MVP WebAssembly can only return up to one value");
1606 if (!callingConvSupported(CallConv))
1607 fail(DL, DAG, Msg: "WebAssembly doesn't support non-C calling conventions");
1608
1609 SmallVector<SDValue, 4> RetOps(1, Chain);
1610 RetOps.append(in_start: OutVals.begin(), in_end: OutVals.end());
1611 Chain = DAG.getNode(Opcode: WebAssemblyISD::RETURN, DL, VT: MVT::Other, Ops: RetOps);
1612
1613 // Record the number and types of the return values.
1614 for (const ISD::OutputArg &Out : Outs) {
1615 assert(!Out.Flags.isByVal() && "byval is not valid for return values");
1616 assert(!Out.Flags.isNest() && "nest is not valid for return values");
1617 assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");
1618 if (Out.Flags.isInAlloca())
1619 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca results");
1620 if (Out.Flags.isInConsecutiveRegs())
1621 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs results");
1622 if (Out.Flags.isInConsecutiveRegsLast())
1623 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last results");
1624 }
1625
1626 return Chain;
1627}
1628
1629SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1630 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1631 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1632 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1633 if (!callingConvSupported(CallConv))
1634 fail(DL, DAG, Msg: "WebAssembly doesn't support non-C calling conventions");
1635
1636 MachineFunction &MF = DAG.getMachineFunction();
1637 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1638
1639 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1640 // of the incoming values before they're represented by virtual registers.
1641 MF.getRegInfo().addLiveIn(Reg: WebAssembly::ARGUMENTS);
1642
1643 bool HasSwiftErrorArg = false;
1644 bool HasSwiftSelfArg = false;
1645 bool HasSwiftAsyncArg = false;
1646 for (const ISD::InputArg &In : Ins) {
1647 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1648 HasSwiftErrorArg |= In.Flags.isSwiftError();
1649 HasSwiftAsyncArg |= In.Flags.isSwiftAsync();
1650 if (In.Flags.isInAlloca())
1651 fail(DL, DAG, Msg: "WebAssembly hasn't implemented inalloca arguments");
1652 if (In.Flags.isNest())
1653 fail(DL, DAG, Msg: "WebAssembly hasn't implemented nest arguments");
1654 if (In.Flags.isInConsecutiveRegs())
1655 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs arguments");
1656 if (In.Flags.isInConsecutiveRegsLast())
1657 fail(DL, DAG, Msg: "WebAssembly hasn't implemented cons regs last arguments");
1658 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1659 // registers.
1660 InVals.push_back(Elt: In.Used ? DAG.getNode(Opcode: WebAssemblyISD::ARGUMENT, DL, VT: In.VT,
1661 Operand: DAG.getTargetConstant(Val: InVals.size(),
1662 DL, VT: MVT::i32))
1663 : DAG.getUNDEF(VT: In.VT));
1664
1665 // Record the number and types of arguments.
1666 MFI->addParam(VT: In.VT);
1667 }
1668
1669 // For swiftcc and swifttailcc, emit additional swiftself, swifterror, and
1670 // (for swifttailcc) swiftasync arguments if there aren't. These additional
1671 // arguments are also added for callee signature. They are necessary to match
1672 // callee and caller signature for indirect call.
1673 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
1674 if (CallConv == CallingConv::Swift || CallConv == CallingConv::SwiftTail) {
1675 if (!HasSwiftSelfArg) {
1676 MFI->addParam(VT: PtrVT);
1677 }
1678 if (!HasSwiftErrorArg) {
1679 MFI->addParam(VT: PtrVT);
1680 }
1681 if (CallConv == CallingConv::SwiftTail && !HasSwiftAsyncArg) {
1682 MFI->addParam(VT: PtrVT);
1683 }
1684 }
1685 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1686 // the buffer is passed as an argument.
1687 if (IsVarArg) {
1688 MVT PtrVT = getPointerTy(DL: MF.getDataLayout());
1689 Register VarargVreg =
1690 MF.getRegInfo().createVirtualRegister(RegClass: getRegClassFor(VT: PtrVT));
1691 MFI->setVarargBufferVreg(VarargVreg);
1692 Chain = DAG.getCopyToReg(
1693 Chain, dl: DL, Reg: VarargVreg,
1694 N: DAG.getNode(Opcode: WebAssemblyISD::ARGUMENT, DL, VT: PtrVT,
1695 Operand: DAG.getTargetConstant(Val: Ins.size(), DL, VT: MVT::i32)));
1696 MFI->addParam(VT: PtrVT);
1697 }
1698
1699 // Record the number and types of arguments and results.
1700 SmallVector<MVT, 4> Params;
1701 SmallVector<MVT, 4> Results;
1702 computeSignatureVTs(Ty: MF.getFunction().getFunctionType(), TargetFunc: &MF.getFunction(),
1703 ContextFunc: MF.getFunction(), TM: DAG.getTarget(), Params, Results);
1704 for (MVT VT : Results)
1705 MFI->addResult(VT);
1706 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1707 // the param logic here with ComputeSignatureVTs
1708 assert(MFI->getParams().size() == Params.size() &&
1709 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1710 Params.begin()));
1711
1712 return Chain;
1713}
1714
1715void WebAssemblyTargetLowering::ReplaceNodeResults(
1716 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1717 switch (N->getOpcode()) {
1718 case ISD::SIGN_EXTEND_INREG:
1719 // Do not add any results, signifying that N should not be custom lowered
1720 // after all. This happens because simd128 turns on custom lowering for
1721 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1722 // illegal type.
1723 break;
1724 case ISD::ANY_EXTEND_VECTOR_INREG:
1725 case ISD::SIGN_EXTEND_VECTOR_INREG:
1726 case ISD::ZERO_EXTEND_VECTOR_INREG:
1727 // Do not add any results, signifying that N should not be custom lowered.
1728 // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
1729 break;
1730 case ISD::FP_ROUND: {
1731 EVT VT = N->getValueType(ResNo: 0);
1732 SDValue Src = N->getOperand(Num: 0);
1733 if (VT == MVT::v4f16 && Src.getValueType() == MVT::v4f32) {
1734 Results.push_back(
1735 Elt: DAG.getNode(Opcode: WebAssemblyISD::DEMOTE_ZERO, DL: SDLoc(N), VT: MVT::v8f16, Operand: Src));
1736 }
1737 break;
1738 }
1739 case ISD::ADD:
1740 case ISD::SUB:
1741 Results.push_back(Elt: Replace128Op(N, DAG));
1742 break;
1743 default:
1744 llvm_unreachable(
1745 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1746 }
1747}
1748
1749//===----------------------------------------------------------------------===//
1750// Custom lowering hooks.
1751//===----------------------------------------------------------------------===//
1752
1753SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1754 SelectionDAG &DAG) const {
1755 SDLoc DL(Op);
1756 switch (Op.getOpcode()) {
1757 default:
1758 llvm_unreachable("unimplemented operation lowering");
1759 return SDValue();
1760 case ISD::FrameIndex:
1761 return LowerFrameIndex(Op, DAG);
1762 case ISD::GlobalAddress:
1763 return LowerGlobalAddress(Op, DAG);
1764 case ISD::GlobalTLSAddress:
1765 return LowerGlobalTLSAddress(Op, DAG);
1766 case ISD::ExternalSymbol:
1767 return LowerExternalSymbol(Op, DAG);
1768 case ISD::JumpTable:
1769 return LowerJumpTable(Op, DAG);
1770 case ISD::BR_JT:
1771 return LowerBR_JT(Op, DAG);
1772 case ISD::VASTART:
1773 return LowerVASTART(Op, DAG);
1774 case ISD::BlockAddress:
1775 case ISD::BRIND:
1776 fail(DL, DAG, Msg: "WebAssembly hasn't implemented computed gotos");
1777 return SDValue();
1778 case ISD::RETURNADDR:
1779 return LowerRETURNADDR(Op, DAG);
1780 case ISD::FRAMEADDR:
1781 return LowerFRAMEADDR(Op, DAG);
1782 case ISD::CopyToReg:
1783 return LowerCopyToReg(Op, DAG);
1784 case ISD::EXTRACT_VECTOR_ELT:
1785 case ISD::INSERT_VECTOR_ELT:
1786 return LowerAccessVectorElement(Op, DAG);
1787 case ISD::INTRINSIC_VOID:
1788 case ISD::INTRINSIC_WO_CHAIN:
1789 case ISD::INTRINSIC_W_CHAIN:
1790 return LowerIntrinsic(Op, DAG);
1791 case ISD::SIGN_EXTEND_INREG:
1792 return LowerSIGN_EXTEND_INREG(Op, DAG);
1793 case ISD::ZERO_EXTEND_VECTOR_INREG:
1794 case ISD::SIGN_EXTEND_VECTOR_INREG:
1795 case ISD::ANY_EXTEND_VECTOR_INREG:
1796 return LowerEXTEND_VECTOR_INREG(Op, DAG);
1797 case ISD::BUILD_VECTOR:
1798 return LowerBUILD_VECTOR(Op, DAG);
1799 case ISD::VECTOR_SHUFFLE:
1800 return LowerVECTOR_SHUFFLE(Op, DAG);
1801 case ISD::SETCC:
1802 return LowerSETCC(Op, DAG);
1803 case ISD::SHL:
1804 case ISD::SRA:
1805 case ISD::SRL:
1806 return LowerShift(Op, DAG);
1807 case ISD::FP_TO_SINT_SAT:
1808 case ISD::FP_TO_UINT_SAT:
1809 return LowerFP_TO_INT_SAT(Op, DAG);
1810 case ISD::FMINNUM:
1811 case ISD::FMINIMUMNUM:
1812 return LowerFMIN(Op, DAG);
1813 case ISD::FMAXNUM:
1814 case ISD::FMAXIMUMNUM:
1815 return LowerFMAX(Op, DAG);
1816 case ISD::LOAD:
1817 return LowerLoad(Op, DAG);
1818 case ISD::STORE:
1819 return LowerStore(Op, DAG);
1820 case ISD::CTPOP:
1821 case ISD::CTLZ:
1822 case ISD::CTTZ:
1823 return DAG.UnrollVectorOp(N: Op.getNode());
1824 case ISD::CLEAR_CACHE:
1825 // Report this as a diagnostic rather than aborting, like the other
1826 // unsupported features in this target. Pass the chain through so that
1827 // codegen can reach the point where the diagnostic is emitted.
1828 fail(DL: SDLoc(Op), DAG, Msg: "llvm.clear_cache is not supported on wasm");
1829 return Op.getOperand(i: 0);
1830 case ISD::SMUL_LOHI:
1831 case ISD::UMUL_LOHI:
1832 return LowerMUL_LOHI(Op, DAG);
1833 case ISD::UADDO:
1834 return LowerUADDO(Op, DAG);
1835 }
1836}
1837
1838static bool IsWebAssemblyGlobal(SDValue Op) {
1839 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val&: Op))
1840 return WebAssembly::isWasmVarAddressSpace(AS: GA->getAddressSpace());
1841
1842 return false;
1843}
1844
1845static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
1846 SelectionDAG &DAG) {
1847 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op);
1848 if (!FI)
1849 return std::nullopt;
1850
1851 auto &MF = DAG.getMachineFunction();
1852 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FrameIndex: FI->getIndex());
1853}
1854
1855SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1856 SelectionDAG &DAG) const {
1857 SDLoc DL(Op);
1858 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
1859 const SDValue &Value = SN->getValue();
1860 const SDValue &Base = SN->getBasePtr();
1861 const SDValue &Offset = SN->getOffset();
1862
1863 if (IsWebAssemblyGlobal(Op: Base)) {
1864 if (!Offset->isUndef())
1865 report_fatal_error(reason: "unexpected offset when storing to webassembly global",
1866 gen_crash_diag: false);
1867
1868 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
1869 SDValue Ops[] = {SN->getChain(), Value, Base};
1870 return DAG.getMemIntrinsicNode(Opcode: WebAssemblyISD::GLOBAL_SET, dl: DL, VTList: Tys, Ops,
1871 MemVT: SN->getMemoryVT(), MMO: SN->getMemOperand());
1872 }
1873
1874 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Op: Base, DAG)) {
1875 if (!Offset->isUndef())
1876 report_fatal_error(reason: "unexpected offset when storing to webassembly local",
1877 gen_crash_diag: false);
1878
1879 SDValue Idx = DAG.getTargetConstant(Val: *Local, DL: Base, VT: MVT::i32);
1880 SDVTList Tys = DAG.getVTList(VT: MVT::Other); // The chain.
1881 SDValue Ops[] = {SN->getChain(), Idx, Value};
1882 return DAG.getNode(Opcode: WebAssemblyISD::LOCAL_SET, DL, VTList: Tys, Ops);
1883 }
1884
1885 if (WebAssembly::isWasmVarAddressSpace(AS: SN->getAddressSpace()))
1886 report_fatal_error(
1887 reason: "Encountered an unlowerable store to the wasm_var address space",
1888 gen_crash_diag: false);
1889
1890 return Op;
1891}
1892
1893SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1894 SelectionDAG &DAG) const {
1895 SDLoc DL(Op);
1896 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
1897 const SDValue &Base = LN->getBasePtr();
1898 const SDValue &Offset = LN->getOffset();
1899
1900 if (IsWebAssemblyGlobal(Op: Base)) {
1901 if (!Offset->isUndef())
1902 report_fatal_error(
1903 reason: "unexpected offset when loading from webassembly global", gen_crash_diag: false);
1904
1905 SDVTList Tys = DAG.getVTList(VT1: LN->getValueType(ResNo: 0), VT2: MVT::Other);
1906 SDValue Ops[] = {LN->getChain(), Base};
1907 return DAG.getMemIntrinsicNode(Opcode: WebAssemblyISD::GLOBAL_GET, dl: DL, VTList: Tys, Ops,
1908 MemVT: LN->getMemoryVT(), MMO: LN->getMemOperand());
1909 }
1910
1911 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Op: Base, DAG)) {
1912 if (!Offset->isUndef())
1913 report_fatal_error(
1914 reason: "unexpected offset when loading from webassembly local", gen_crash_diag: false);
1915
1916 SDValue Idx = DAG.getTargetConstant(Val: *Local, DL: Base, VT: MVT::i32);
1917 EVT LocalVT = LN->getValueType(ResNo: 0);
1918 return DAG.getNode(Opcode: WebAssemblyISD::LOCAL_GET, DL, ResultTys: {LocalVT, MVT::Other},
1919 Ops: {LN->getChain(), Idx});
1920 }
1921
1922 if (WebAssembly::isWasmVarAddressSpace(AS: LN->getAddressSpace()))
1923 report_fatal_error(
1924 reason: "Encountered an unlowerable load from the wasm_var address space",
1925 gen_crash_diag: false);
1926
1927 return Op;
1928}
1929
1930SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
1931 SelectionDAG &DAG) const {
1932 assert(Subtarget->hasWideArithmetic());
1933 assert(Op.getValueType() == MVT::i64);
1934 SDLoc DL(Op);
1935 unsigned Opcode;
1936 switch (Op.getOpcode()) {
1937 case ISD::UMUL_LOHI:
1938 Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
1939 break;
1940 case ISD::SMUL_LOHI:
1941 Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
1942 break;
1943 default:
1944 llvm_unreachable("unexpected opcode");
1945 }
1946 SDValue LHS = Op.getOperand(i: 0);
1947 SDValue RHS = Op.getOperand(i: 1);
1948 SDValue Lo =
1949 DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64), N1: LHS, N2: RHS);
1950 SDValue Hi(Lo.getNode(), 1);
1951 SDValue Ops[] = {Lo, Hi};
1952 return DAG.getMergeValues(Ops, dl: DL);
1953}
1954
1955// Lowers `UADDO` intrinsics to an `i64.add128` instruction when it's enabled.
1956//
1957// This enables generating a single wasm instruction for this operation where
1958// the upper half of both operands are constant zeros. The upper half of the
1959// result is then whether the overflow happened.
1960SDValue WebAssemblyTargetLowering::LowerUADDO(SDValue Op,
1961 SelectionDAG &DAG) const {
1962 assert(Subtarget->hasWideArithmetic());
1963 assert(Op.getValueType() == MVT::i64);
1964 assert(Op.getOpcode() == ISD::UADDO);
1965 SDLoc DL(Op);
1966 SDValue LHS = Op.getOperand(i: 0);
1967 SDValue RHS = Op.getOperand(i: 1);
1968 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
1969 SDValue Result =
1970 DAG.getNode(Opcode: WebAssemblyISD::I64_ADD128, DL,
1971 VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64), N1: LHS, N2: Zero, N3: RHS, N4: Zero);
1972 SDValue CarryI64(Result.getNode(), 1);
1973 SDValue CarryI32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: CarryI64);
1974 SDValue Ops[] = {Result, CarryI32};
1975 return DAG.getMergeValues(Ops, dl: DL);
1976}
1977
1978SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
1979 SelectionDAG &DAG) const {
1980 assert(Subtarget->hasWideArithmetic());
1981 assert(N->getValueType(0) == MVT::i128);
1982 SDLoc DL(N);
1983 unsigned Opcode;
1984 switch (N->getOpcode()) {
1985 case ISD::ADD:
1986 Opcode = WebAssemblyISD::I64_ADD128;
1987 break;
1988 case ISD::SUB:
1989 Opcode = WebAssemblyISD::I64_SUB128;
1990 break;
1991 default:
1992 llvm_unreachable("unexpected opcode");
1993 }
1994 SDValue LHS = N->getOperand(Num: 0);
1995 SDValue RHS = N->getOperand(Num: 1);
1996
1997 SDValue C0 = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
1998 SDValue C1 = DAG.getConstant(Val: 1, DL, VT: MVT::i64);
1999 SDValue LHS_0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: LHS, N2: C0);
2000 SDValue LHS_1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: LHS, N2: C1);
2001 SDValue RHS_0 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: RHS, N2: C0);
2002 SDValue RHS_1 = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i64, N1: RHS, N2: C1);
2003 SDValue Result_LO = DAG.getNode(Opcode, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64),
2004 N1: LHS_0, N2: LHS_1, N3: RHS_0, N4: RHS_1);
2005 SDValue Result_HI(Result_LO.getNode(), 1);
2006 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VTList: N->getVTList(), N1: Result_LO, N2: Result_HI);
2007}
2008
2009SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
2010 SelectionDAG &DAG) const {
2011 SDValue Src = Op.getOperand(i: 2);
2012 if (isa<FrameIndexSDNode>(Val: Src.getNode())) {
2013 // CopyToReg nodes don't support FrameIndex operands. Other targets select
2014 // the FI to some LEA-like instruction, but since we don't have that, we
2015 // need to insert some kind of instruction that can take an FI operand and
2016 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
2017 // local.copy between Op and its FI operand.
2018 SDValue Chain = Op.getOperand(i: 0);
2019 SDLoc DL(Op);
2020 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i: 1))->getReg();
2021 EVT VT = Src.getValueType();
2022 SDValue Copy(DAG.getMachineNode(Opcode: VT == MVT::i32 ? WebAssembly::COPY_I32
2023 : WebAssembly::COPY_I64,
2024 dl: DL, VT, Op1: Src),
2025 0);
2026 return Op.getNode()->getNumValues() == 1
2027 ? DAG.getCopyToReg(Chain, dl: DL, Reg, N: Copy)
2028 : DAG.getCopyToReg(Chain, dl: DL, Reg, N: Copy,
2029 Glue: Op.getNumOperands() == 4 ? Op.getOperand(i: 3)
2030 : SDValue());
2031 }
2032 return SDValue();
2033}
2034
2035SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
2036 SelectionDAG &DAG) const {
2037 int FI = cast<FrameIndexSDNode>(Val&: Op)->getIndex();
2038 return DAG.getTargetFrameIndex(FI, VT: Op.getValueType());
2039}
2040
2041SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
2042 SelectionDAG &DAG) const {
2043 SDLoc DL(Op);
2044
2045 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
2046 fail(DL, DAG,
2047 Msg: "Non-Emscripten WebAssembly hasn't implemented "
2048 "__builtin_return_address");
2049 return SDValue();
2050 }
2051
2052 unsigned Depth = Op.getConstantOperandVal(i: 0);
2053 MakeLibCallOptions CallOptions;
2054 return makeLibCall(DAG, LC: RTLIB::RETURN_ADDRESS, RetVT: Op.getValueType(),
2055 Ops: {DAG.getConstant(Val: Depth, DL, VT: MVT::i32)}, CallOptions, dl: DL)
2056 .first;
2057}
2058
2059SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
2060 SelectionDAG &DAG) const {
2061 // Non-zero depths are not supported by WebAssembly currently. Use the
2062 // legalizer's default expansion, which is to return 0 (what this function is
2063 // documented to do).
2064 if (Op.getConstantOperandVal(i: 0) > 0)
2065 return SDValue();
2066
2067 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
2068 EVT VT = Op.getValueType();
2069 Register FP =
2070 Subtarget->getRegisterInfo()->getFrameRegister(MF: DAG.getMachineFunction());
2071 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: SDLoc(Op), Reg: FP, VT);
2072}
2073
2074SDValue
2075WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
2076 SelectionDAG &DAG) const {
2077 SDLoc DL(Op);
2078 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
2079
2080 MachineFunction &MF = DAG.getMachineFunction();
2081 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
2082 report_fatal_error(reason: "cannot use thread-local storage without bulk memory",
2083 gen_crash_diag: false);
2084
2085 const GlobalValue *GV = GA->getGlobal();
2086
2087 // Currently only Emscripten supports dynamic linking with threads. Therefore,
2088 // on other targets, if we have thread-local storage, only the local-exec
2089 // model is possible.
2090 auto model = Subtarget->getTargetTriple().isOSEmscripten()
2091 ? GV->getThreadLocalMode()
2092 : GlobalValue::LocalExecTLSModel;
2093
2094 // Unsupported TLS modes
2095 assert(model != GlobalValue::NotThreadLocal);
2096 assert(model != GlobalValue::InitialExecTLSModel);
2097
2098 if (model == GlobalValue::LocalExecTLSModel ||
2099 model == GlobalValue::LocalDynamicTLSModel ||
2100 (model == GlobalValue::GeneralDynamicTLSModel &&
2101 getTargetMachine().shouldAssumeDSOLocal(GV))) {
2102 // For DSO-local TLS variables we use offset from __tls_base, or
2103 // __wasm_get_tls_base() if using libcall thread context.
2104
2105 MVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
2106 SDValue BaseAddr(WebAssembly::getTLSBase(DAG, DL, Subtarget), 0);
2107
2108 SDValue TLSOffset = DAG.getTargetGlobalAddress(
2109 GV, DL, VT: PtrVT, offset: GA->getOffset(), TargetFlags: WebAssemblyII::MO_TLS_BASE_REL);
2110 SDValue SymOffset =
2111 DAG.getNode(Opcode: WebAssemblyISD::WrapperREL, DL, VT: PtrVT, Operand: TLSOffset);
2112
2113 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BaseAddr, N2: SymOffset);
2114 }
2115
2116 assert(model == GlobalValue::GeneralDynamicTLSModel);
2117
2118 EVT VT = Op.getValueType();
2119 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2120 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT,
2121 offset: GA->getOffset(),
2122 TargetFlags: WebAssemblyII::MO_GOT_TLS));
2123}
2124
2125SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
2126 SelectionDAG &DAG) const {
2127 SDLoc DL(Op);
2128 const auto *GA = cast<GlobalAddressSDNode>(Val&: Op);
2129 EVT VT = Op.getValueType();
2130 assert(GA->getTargetFlags() == 0 &&
2131 "Unexpected target flags on generic GlobalAddressSDNode");
2132 if (!WebAssembly::isValidAddressSpace(AS: GA->getAddressSpace()))
2133 fail(DL, DAG, Msg: "Invalid address space for WebAssembly target");
2134
2135 unsigned OperandFlags = 0;
2136 const GlobalValue *GV = GA->getGlobal();
2137 // Since WebAssembly tables cannot yet be shared across modules, we don't
2138 // need special treatment for tables in PIC mode.
2139 if (isPositionIndependent() &&
2140 !WebAssembly::isWebAssemblyTableType(Ty: GV->getValueType())) {
2141 if (getTargetMachine().shouldAssumeDSOLocal(GV)) {
2142 MachineFunction &MF = DAG.getMachineFunction();
2143 MVT PtrVT = getPointerTy(DL: MF.getDataLayout());
2144 const char *BaseName;
2145 if (GV->getValueType()->isFunctionTy()) {
2146 BaseName = MF.createExternalSymbolName(Name: "__table_base");
2147 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
2148 } else {
2149 BaseName = MF.createExternalSymbolName(Name: "__memory_base");
2150 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
2151 }
2152 SDValue BaseAddr =
2153 DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT,
2154 Operand: DAG.getTargetExternalSymbol(Sym: BaseName, VT: PtrVT));
2155
2156 SDValue SymAddr = DAG.getNode(
2157 Opcode: WebAssemblyISD::WrapperREL, DL, VT,
2158 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT, offset: GA->getOffset(),
2159 TargetFlags: OperandFlags));
2160
2161 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: BaseAddr, N2: SymAddr);
2162 }
2163 OperandFlags = WebAssemblyII::MO_GOT;
2164 }
2165
2166 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2167 Operand: DAG.getTargetGlobalAddress(GV: GA->getGlobal(), DL, VT,
2168 offset: GA->getOffset(), TargetFlags: OperandFlags));
2169}
2170
2171SDValue
2172WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
2173 SelectionDAG &DAG) const {
2174 SDLoc DL(Op);
2175 const auto *ES = cast<ExternalSymbolSDNode>(Val&: Op);
2176 EVT VT = Op.getValueType();
2177 assert(ES->getTargetFlags() == 0 &&
2178 "Unexpected target flags on generic ExternalSymbolSDNode");
2179 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT,
2180 Operand: DAG.getTargetExternalSymbol(Sym: ES->getSymbol(), VT));
2181}
2182
2183SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
2184 SelectionDAG &DAG) const {
2185 // There's no need for a Wrapper node because we always incorporate a jump
2186 // table operand into a BR_TABLE instruction, rather than ever
2187 // materializing it in a register.
2188 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
2189 return DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Op.getValueType(),
2190 TargetFlags: JT->getTargetFlags());
2191}
2192
2193SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
2194 SelectionDAG &DAG) const {
2195 SDLoc DL(Op);
2196 SDValue Chain = Op.getOperand(i: 0);
2197 const auto *JT = cast<JumpTableSDNode>(Val: Op.getOperand(i: 1));
2198 SDValue Index = Op.getOperand(i: 2);
2199 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
2200
2201 SmallVector<SDValue, 8> Ops;
2202 Ops.push_back(Elt: Chain);
2203 Ops.push_back(Elt: Index);
2204
2205 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
2206 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
2207
2208 // Add an operand for each case.
2209 for (auto *MBB : MBBs)
2210 Ops.push_back(Elt: DAG.getBasicBlock(MBB));
2211
2212 // Add the first MBB as a dummy default target for now. This will be replaced
2213 // with the proper default target (and the preceding range check eliminated)
2214 // if possible by WebAssemblyFixBrTableDefaults.
2215 Ops.push_back(Elt: DAG.getBasicBlock(MBB: *MBBs.begin()));
2216 return DAG.getNode(Opcode: WebAssemblyISD::BR_TABLE, DL, VT: MVT::Other, Ops);
2217}
2218
2219SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
2220 SelectionDAG &DAG) const {
2221 SDLoc DL(Op);
2222 EVT PtrVT = getPointerTy(DL: DAG.getMachineFunction().getDataLayout());
2223
2224 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
2225 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
2226
2227 SDValue ArgN = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
2228 Reg: MFI->getVarargBufferVreg(), VT: PtrVT);
2229 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: ArgN, Ptr: Op.getOperand(i: 1),
2230 PtrInfo: MachinePointerInfo(SV));
2231}
2232
2233SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
2234 SelectionDAG &DAG) const {
2235 MachineFunction &MF = DAG.getMachineFunction();
2236 unsigned IntNo;
2237 switch (Op.getOpcode()) {
2238 case ISD::INTRINSIC_VOID:
2239 case ISD::INTRINSIC_W_CHAIN:
2240 IntNo = Op.getConstantOperandVal(i: 1);
2241 break;
2242 case ISD::INTRINSIC_WO_CHAIN:
2243 IntNo = Op.getConstantOperandVal(i: 0);
2244 break;
2245 default:
2246 llvm_unreachable("Invalid intrinsic");
2247 }
2248 SDLoc DL(Op);
2249
2250 switch (IntNo) {
2251 default:
2252 return SDValue(); // Don't custom lower most intrinsics.
2253
2254 case Intrinsic::wasm_lsda: {
2255 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
2256 const char *SymName = MF.createExternalSymbolName(
2257 Name: "GCC_except_table" + std::to_string(val: MF.getFunctionNumber()));
2258 if (isPositionIndependent()) {
2259 SDValue Node = DAG.getTargetExternalSymbol(
2260 Sym: SymName, VT: PtrVT, TargetFlags: WebAssemblyII::MO_MEMORY_BASE_REL);
2261 const char *BaseName = MF.createExternalSymbolName(Name: "__memory_base");
2262 SDValue BaseAddr =
2263 DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT,
2264 Operand: DAG.getTargetExternalSymbol(Sym: BaseName, VT: PtrVT));
2265 SDValue SymAddr =
2266 DAG.getNode(Opcode: WebAssemblyISD::WrapperREL, DL, VT: PtrVT, Operand: Node);
2267 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: BaseAddr, N2: SymAddr);
2268 }
2269 SDValue Node = DAG.getTargetExternalSymbol(Sym: SymName, VT: PtrVT);
2270 return DAG.getNode(Opcode: WebAssemblyISD::Wrapper, DL, VT: PtrVT, Operand: Node);
2271 }
2272
2273 case Intrinsic::wasm_shuffle: {
2274 // Drop in-chain and replace undefs, but otherwise pass through unchanged
2275 SDValue Ops[18];
2276 size_t OpIdx = 0;
2277 Ops[OpIdx++] = Op.getOperand(i: 1);
2278 Ops[OpIdx++] = Op.getOperand(i: 2);
2279 while (OpIdx < 18) {
2280 const SDValue &MaskIdx = Op.getOperand(i: OpIdx + 1);
2281 if (MaskIdx.isUndef() || MaskIdx.getNode()->getAsZExtVal() >= 32) {
2282 bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;
2283 Ops[OpIdx++] = DAG.getConstant(Val: 0, DL, VT: MVT::i32, isTarget);
2284 } else {
2285 Ops[OpIdx++] = MaskIdx;
2286 }
2287 }
2288 return DAG.getNode(Opcode: WebAssemblyISD::SHUFFLE, DL, VT: Op.getValueType(), Ops);
2289 }
2290
2291 case Intrinsic::wasm_funcref_to_ptr: {
2292 // llvm.wasm.funcref.to_ptr only has a defined lowering when its result
2293 // feeds directly into an indirect call. Reaching here means the pointer
2294 // escapes a direct call. We haven't implemented conversion of a funcref
2295 // into a real function pointer so we crash if we get here.
2296 fail(DL, DAG,
2297 Msg: "a funcref can only be converted to a pointer to be directly called; "
2298 "the resulting pointer cannot otherwise be used");
2299 return DAG.getPOISON(VT: Op.getValueType());
2300 }
2301
2302 case Intrinsic::thread_pointer: {
2303 return SDValue(WebAssembly::getTLSBase(DAG, DL, Subtarget), 0);
2304 }
2305 }
2306}
2307
2308SDValue
2309WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2310 SelectionDAG &DAG) const {
2311 SDLoc DL(Op);
2312 // If sign extension operations are disabled, allow sext_inreg only if operand
2313 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
2314 // extension operations, but allowing sext_inreg in this context lets us have
2315 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
2316 // everywhere would be simpler in this file, but would necessitate large and
2317 // brittle patterns to undo the expansion and select extract_lane_s
2318 // instructions.
2319 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
2320 if (Op.getOperand(i: 0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2321 return SDValue();
2322
2323 const SDValue &Extract = Op.getOperand(i: 0);
2324 MVT VecT = Extract.getOperand(i: 0).getSimpleValueType();
2325 if (VecT.getVectorElementType().getSizeInBits() > 32)
2326 return SDValue();
2327 MVT ExtractedLaneT =
2328 cast<VTSDNode>(Val: Op.getOperand(i: 1).getNode())->getVT().getSimpleVT();
2329 MVT ExtractedVecT =
2330 MVT::getVectorVT(VT: ExtractedLaneT, NumElements: 128 / ExtractedLaneT.getSizeInBits());
2331 if (ExtractedVecT == VecT)
2332 return Op;
2333
2334 // Bitcast vector to appropriate type to ensure ISel pattern coverage
2335 const SDNode *Index = Extract.getOperand(i: 1).getNode();
2336 if (!isa<ConstantSDNode>(Val: Index))
2337 return SDValue();
2338 unsigned IndexVal = Index->getAsZExtVal();
2339 unsigned Scale =
2340 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
2341 assert(Scale > 1);
2342 SDValue NewIndex =
2343 DAG.getConstant(Val: IndexVal * Scale, DL, VT: Index->getValueType(ResNo: 0));
2344 SDValue NewExtract = DAG.getNode(
2345 Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: Extract.getValueType(),
2346 N1: DAG.getBitcast(VT: ExtractedVecT, V: Extract.getOperand(i: 0)), N2: NewIndex);
2347 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: Op.getValueType(), N1: NewExtract,
2348 N2: Op.getOperand(i: 1));
2349}
2350
2351static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT,
2352 SelectionDAG &DAG) {
2353 SDValue Source = peekThroughBitcasts(V: Op);
2354 if (Source.getOpcode() != ISD::VECTOR_SHUFFLE)
2355 return SDValue();
2356
2357 assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||
2358 UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&
2359 "expected extend_low");
2360 auto *Shuffle = cast<ShuffleVectorSDNode>(Val: Source.getNode());
2361
2362 ArrayRef<int> Mask = Shuffle->getMask();
2363 // Look for a shuffle which moves from the high half to the low half.
2364 size_t FirstIdx = Mask.size() / 2;
2365 for (size_t i = 0; i < Mask.size() / 2; ++i) {
2366 if (Mask[i] != static_cast<int>(FirstIdx + i)) {
2367 return SDValue();
2368 }
2369 }
2370
2371 SDLoc DL(Op);
2372 unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S
2373 ? WebAssemblyISD::EXTEND_HIGH_S
2374 : WebAssemblyISD::EXTEND_HIGH_U;
2375 SDValue ShuffleSrc = Shuffle->getOperand(Num: 0);
2376 if (Op.getOpcode() == ISD::BITCAST)
2377 ShuffleSrc = DAG.getBitcast(VT: Op.getValueType(), V: ShuffleSrc);
2378
2379 return DAG.getNode(Opcode: Opc, DL, VT, Operand: ShuffleSrc);
2380}
2381
2382SDValue
2383WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,
2384 SelectionDAG &DAG) const {
2385 SDLoc DL(Op);
2386 EVT VT = Op.getValueType();
2387 SDValue Src = Op.getOperand(i: 0);
2388 EVT SrcVT = Src.getValueType();
2389
2390 if (SrcVT.getVectorElementType() == MVT::i1 ||
2391 SrcVT.getVectorElementType() == MVT::i64)
2392 return SDValue();
2393
2394 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
2395 "Unexpected extension factor.");
2396 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
2397
2398 if (Scale != 2 && Scale != 4 && Scale != 8)
2399 return SDValue();
2400
2401 unsigned Ext;
2402 switch (Op.getOpcode()) {
2403 default:
2404 llvm_unreachable("unexpected opcode");
2405 case ISD::ANY_EXTEND_VECTOR_INREG:
2406 case ISD::ZERO_EXTEND_VECTOR_INREG:
2407 Ext = WebAssemblyISD::EXTEND_LOW_U;
2408 break;
2409 case ISD::SIGN_EXTEND_VECTOR_INREG:
2410 Ext = WebAssemblyISD::EXTEND_LOW_S;
2411 break;
2412 }
2413
2414 if (Scale == 2) {
2415 // See if we can use EXTEND_HIGH.
2416 if (auto ExtendHigh = GetExtendHigh(Op: Op.getOperand(i: 0), UserOpc: Ext, VT, DAG))
2417 return ExtendHigh;
2418 }
2419
2420 SDValue Ret = Src;
2421 while (Scale != 1) {
2422 Ret = DAG.getNode(Opcode: Ext, DL,
2423 VT: Ret.getValueType()
2424 .widenIntegerVectorElementType(Context&: *DAG.getContext())
2425 .getHalfNumVectorElementsVT(Context&: *DAG.getContext()),
2426 Operand: Ret);
2427 Scale /= 2;
2428 }
2429 assert(Ret.getValueType() == VT);
2430 return Ret;
2431}
2432
2433static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
2434 SDLoc DL(Op);
2435 if (Op.getValueType() != MVT::v2f64 && Op.getValueType() != MVT::v4f32)
2436 return SDValue();
2437
2438 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
2439 unsigned &Index) -> bool {
2440 switch (Op.getOpcode()) {
2441 case ISD::SINT_TO_FP:
2442 Opcode = WebAssemblyISD::CONVERT_LOW_S;
2443 break;
2444 case ISD::UINT_TO_FP:
2445 Opcode = WebAssemblyISD::CONVERT_LOW_U;
2446 break;
2447 case ISD::FP_EXTEND:
2448 case ISD::FP16_TO_FP:
2449 Opcode = WebAssemblyISD::PROMOTE_LOW;
2450 break;
2451 default:
2452 return false;
2453 }
2454
2455 auto ExtractVector = Op.getOperand(i: 0);
2456 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2457 return false;
2458
2459 if (!isa<ConstantSDNode>(Val: ExtractVector.getOperand(i: 1).getNode()))
2460 return false;
2461
2462 SrcVec = ExtractVector.getOperand(i: 0);
2463 Index = ExtractVector.getConstantOperandVal(i: 1);
2464 return true;
2465 };
2466
2467 unsigned NumLanes = Op.getValueType() == MVT::v2f64 ? 2 : 4;
2468 unsigned FirstOpcode = 0, SecondOpcode = 0, ThirdOpcode = 0, FourthOpcode = 0;
2469 unsigned FirstIndex = 0, SecondIndex = 0, ThirdIndex = 0, FourthIndex = 0;
2470 SDValue FirstSrcVec, SecondSrcVec, ThirdSrcVec, FourthSrcVec;
2471
2472 if (!GetConvertedLane(Op.getOperand(i: 0), FirstOpcode, FirstSrcVec,
2473 FirstIndex) ||
2474 !GetConvertedLane(Op.getOperand(i: 1), SecondOpcode, SecondSrcVec,
2475 SecondIndex))
2476 return SDValue();
2477
2478 // If we're converting to v4f32, check the third and fourth lanes, too.
2479 if (NumLanes == 4 && (!GetConvertedLane(Op.getOperand(i: 2), ThirdOpcode,
2480 ThirdSrcVec, ThirdIndex) ||
2481 !GetConvertedLane(Op.getOperand(i: 3), FourthOpcode,
2482 FourthSrcVec, FourthIndex)))
2483 return SDValue();
2484
2485 if (FirstOpcode != SecondOpcode)
2486 return SDValue();
2487
2488 // TODO Add an optimization similar to the v2f64 below for shuffling the
2489 // vectors when the lanes are in the wrong order or come from different src
2490 // vectors.
2491 if (NumLanes == 4 &&
2492 (FirstOpcode != ThirdOpcode || FirstOpcode != FourthOpcode ||
2493 FirstSrcVec != SecondSrcVec || FirstSrcVec != ThirdSrcVec ||
2494 FirstSrcVec != FourthSrcVec || FirstIndex != 0 || SecondIndex != 1 ||
2495 ThirdIndex != 2 || FourthIndex != 3))
2496 return SDValue();
2497
2498 MVT ExpectedSrcVT;
2499 switch (FirstOpcode) {
2500 case WebAssemblyISD::CONVERT_LOW_S:
2501 case WebAssemblyISD::CONVERT_LOW_U:
2502 ExpectedSrcVT = MVT::v4i32;
2503 break;
2504 case WebAssemblyISD::PROMOTE_LOW:
2505 ExpectedSrcVT = NumLanes == 2 ? MVT::v4f32 : MVT::v8i16;
2506 break;
2507 }
2508 if (FirstSrcVec.getValueType() != ExpectedSrcVT)
2509 return SDValue();
2510
2511 auto Src = FirstSrcVec;
2512 if (NumLanes == 2 &&
2513 (FirstIndex != 0 || SecondIndex != 1 || FirstSrcVec != SecondSrcVec)) {
2514 // Shuffle the source vector so that the converted lanes are the low lanes.
2515 Src = DAG.getVectorShuffle(VT: ExpectedSrcVT, dl: DL, N1: FirstSrcVec, N2: SecondSrcVec,
2516 Mask: {static_cast<int>(FirstIndex),
2517 static_cast<int>(SecondIndex) + 4, -1, -1});
2518 }
2519 return DAG.getNode(Opcode: FirstOpcode, DL, VT: NumLanes == 2 ? MVT::v2f64 : MVT::v4f32,
2520 Operand: Src);
2521}
2522
2523SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
2524 SelectionDAG &DAG) const {
2525 MVT VT = Op.getSimpleValueType();
2526 if (VT == MVT::v8f16) {
2527 // BUILD_VECTOR can't handle FP16 operands since Wasm doesn't have a scalar
2528 // FP16 type, so cast them to I16s.
2529 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
2530 SmallVector<SDValue, 8> NewOps;
2531 for (unsigned I = 0, E = Op.getNumOperands(); I < E; ++I)
2532 NewOps.push_back(Elt: DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: I)));
2533 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: SDLoc(), VT: IVT, Ops: NewOps);
2534 return DAG.getBitcast(VT, V: Res);
2535 }
2536
2537 if (auto ConvertLow = LowerConvertLow(Op, DAG))
2538 return ConvertLow;
2539
2540 SDLoc DL(Op);
2541 const EVT VecT = Op.getValueType();
2542 const EVT LaneT = Op.getOperand(i: 0).getValueType();
2543 const size_t Lanes = Op.getNumOperands();
2544 bool CanSwizzle = VecT == MVT::v16i8;
2545
2546 // BUILD_VECTORs are lowered to the instruction that initializes the highest
2547 // possible number of lanes at once followed by a sequence of replace_lane
2548 // instructions to individually initialize any remaining lanes.
2549
2550 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
2551 // swizzled lanes should be given greater weight.
2552
2553 // TODO: Investigate looping rather than always extracting/replacing specific
2554 // lanes to fill gaps.
2555
2556 auto IsConstant = [](const SDValue &V) {
2557 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
2558 };
2559
2560 // Returns the source vector and index vector pair if they exist. Checks for:
2561 // (extract_vector_elt
2562 // $src,
2563 // (sign_extend_inreg (extract_vector_elt $indices, $i))
2564 // )
2565 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
2566 auto Bail = std::make_pair(x: SDValue(), y: SDValue());
2567 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2568 return Bail;
2569 const SDValue &SwizzleSrc = Lane->getOperand(Num: 0);
2570 const SDValue &IndexExt = Lane->getOperand(Num: 1);
2571 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
2572 return Bail;
2573 const SDValue &Index = IndexExt->getOperand(Num: 0);
2574 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2575 return Bail;
2576 const SDValue &SwizzleIndices = Index->getOperand(Num: 0);
2577 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
2578 SwizzleIndices.getValueType() != MVT::v16i8 ||
2579 Index->getOperand(Num: 1)->getOpcode() != ISD::Constant ||
2580 Index->getConstantOperandVal(Num: 1) != I)
2581 return Bail;
2582 return std::make_pair(x: SwizzleSrc, y: SwizzleIndices);
2583 };
2584
2585 // If the lane is extracted from another vector at a constant index, return
2586 // that vector. The source vector must not have more lanes than the dest
2587 // because the shufflevector indices are in terms of the destination lanes and
2588 // would not be able to address the smaller individual source lanes.
2589 auto GetShuffleSrc = [&](const SDValue &Lane) {
2590 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2591 return SDValue();
2592 if (!isa<ConstantSDNode>(Val: Lane->getOperand(Num: 1).getNode()))
2593 return SDValue();
2594 if (Lane->getOperand(Num: 0).getValueType().getVectorNumElements() >
2595 VecT.getVectorNumElements())
2596 return SDValue();
2597 return Lane->getOperand(Num: 0);
2598 };
2599
2600 using ValueEntry = std::pair<SDValue, size_t>;
2601 SmallVector<ValueEntry, 16> SplatValueCounts;
2602
2603 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2604 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2605
2606 using ShuffleEntry = std::pair<SDValue, size_t>;
2607 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2608
2609 auto AddCount = [](auto &Counts, const auto &Val) {
2610 auto CountIt =
2611 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2612 if (CountIt == Counts.end()) {
2613 Counts.emplace_back(Val, 1);
2614 } else {
2615 CountIt->second++;
2616 }
2617 };
2618
2619 auto GetMostCommon = [](auto &Counts) {
2620 auto CommonIt = llvm::max_element(Counts, llvm::less_second());
2621 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
2622 return *CommonIt;
2623 };
2624
2625 size_t NumConstantLanes = 0;
2626
2627 // Count eligible lanes for each type of vector creation op
2628 for (size_t I = 0; I < Lanes; ++I) {
2629 const SDValue &Lane = Op->getOperand(Num: I);
2630 if (Lane.isUndef())
2631 continue;
2632
2633 AddCount(SplatValueCounts, Lane);
2634
2635 if (IsConstant(Lane))
2636 NumConstantLanes++;
2637 if (auto ShuffleSrc = GetShuffleSrc(Lane))
2638 AddCount(ShuffleCounts, ShuffleSrc);
2639 if (CanSwizzle) {
2640 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2641 if (SwizzleSrcs.first)
2642 AddCount(SwizzleCounts, SwizzleSrcs);
2643 }
2644 }
2645
2646 SDValue SplatValue;
2647 size_t NumSplatLanes;
2648 std::tie(args&: SplatValue, args&: NumSplatLanes) = GetMostCommon(SplatValueCounts);
2649
2650 SDValue SwizzleSrc;
2651 SDValue SwizzleIndices;
2652 size_t NumSwizzleLanes = 0;
2653 if (SwizzleCounts.size())
2654 std::forward_as_tuple(args: std::tie(args&: SwizzleSrc, args&: SwizzleIndices),
2655 args&: NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2656
2657 // Shuffles can draw from up to two vectors, so find the two most common
2658 // sources.
2659 SDValue ShuffleSrc1, ShuffleSrc2;
2660 size_t NumShuffleLanes = 0;
2661 if (ShuffleCounts.size()) {
2662 std::tie(args&: ShuffleSrc1, args&: NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2663 llvm::erase_if(C&: ShuffleCounts,
2664 P: [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2665 }
2666 if (ShuffleCounts.size()) {
2667 size_t AdditionalShuffleLanes;
2668 std::tie(args&: ShuffleSrc2, args&: AdditionalShuffleLanes) =
2669 GetMostCommon(ShuffleCounts);
2670 NumShuffleLanes += AdditionalShuffleLanes;
2671 }
2672
2673 // Predicate returning true if the lane is properly initialized by the
2674 // original instruction
2675 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2676 SDValue Result;
2677 // Prefer swizzles over shuffles over vector consts over splats
2678 if (NumSwizzleLanes >= NumShuffleLanes &&
2679 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2680 Result = DAG.getNode(Opcode: WebAssemblyISD::SWIZZLE, DL, VT: VecT, N1: SwizzleSrc,
2681 N2: SwizzleIndices);
2682 auto Swizzled = std::make_pair(x&: SwizzleSrc, y&: SwizzleIndices);
2683 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2684 return Swizzled == GetSwizzleSrcs(I, Lane);
2685 };
2686 } else if (NumShuffleLanes >= NumConstantLanes &&
2687 NumShuffleLanes >= NumSplatLanes) {
2688 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2689 size_t DestLaneCount = VecT.getVectorNumElements();
2690 size_t Scale1 = 1;
2691 size_t Scale2 = 1;
2692 SDValue Src1 = ShuffleSrc1;
2693 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VT: VecT);
2694 if (Src1.getValueType() != VecT) {
2695 size_t LaneSize =
2696 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2697 assert(LaneSize > DestLaneSize);
2698 Scale1 = LaneSize / DestLaneSize;
2699 Src1 = DAG.getBitcast(VT: VecT, V: Src1);
2700 }
2701 if (Src2.getValueType() != VecT) {
2702 size_t LaneSize =
2703 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2704 assert(LaneSize > DestLaneSize);
2705 Scale2 = LaneSize / DestLaneSize;
2706 Src2 = DAG.getBitcast(VT: VecT, V: Src2);
2707 }
2708
2709 int Mask[16];
2710 assert(DestLaneCount <= 16);
2711 for (size_t I = 0; I < DestLaneCount; ++I) {
2712 const SDValue &Lane = Op->getOperand(Num: I);
2713 SDValue Src = GetShuffleSrc(Lane);
2714 if (Src == ShuffleSrc1) {
2715 Mask[I] = Lane->getConstantOperandVal(Num: 1) * Scale1;
2716 } else if (Src && Src == ShuffleSrc2) {
2717 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(Num: 1) * Scale2;
2718 } else {
2719 Mask[I] = -1;
2720 }
2721 }
2722 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2723 Result = DAG.getVectorShuffle(VT: VecT, dl: DL, N1: Src1, N2: Src2, Mask: MaskRef);
2724 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2725 auto Src = GetShuffleSrc(Lane);
2726 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2727 };
2728 } else if (NumConstantLanes >= NumSplatLanes) {
2729 SmallVector<SDValue, 16> ConstLanes;
2730 for (const SDValue &Lane : Op->op_values()) {
2731 if (IsConstant(Lane)) {
2732 // Values may need to be fixed so that they will sign extend to be
2733 // within the expected range during ISel. Check whether the value is in
2734 // bounds based on the lane bit width and if it is out of bounds, lop
2735 // off the extra bits.
2736 uint64_t LaneBits = 128 / Lanes;
2737 if (auto *Const = dyn_cast<ConstantSDNode>(Val: Lane.getNode())) {
2738 ConstLanes.push_back(Elt: DAG.getConstant(
2739 Val: Const->getAPIntValue().trunc(width: LaneBits).getZExtValue(),
2740 DL: SDLoc(Lane), VT: LaneT));
2741 } else {
2742 ConstLanes.push_back(Elt: Lane);
2743 }
2744 } else if (LaneT.isFloatingPoint()) {
2745 ConstLanes.push_back(Elt: DAG.getConstantFP(Val: 0, DL, VT: LaneT));
2746 } else {
2747 ConstLanes.push_back(Elt: DAG.getConstant(Val: 0, DL, VT: LaneT));
2748 }
2749 }
2750 Result = DAG.getBuildVector(VT: VecT, DL, Ops: ConstLanes);
2751 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2752 return IsConstant(Lane);
2753 };
2754 } else {
2755 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits();
2756 if (NumSplatLanes == 1 && Op->getOperand(Num: 0) == SplatValue &&
2757 (DestLaneSize == 32 || DestLaneSize == 64)) {
2758 // Could be selected to load_zero.
2759 Result = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: VecT, Operand: SplatValue);
2760 } else {
2761 // Use a splat (which might be selected as a load splat)
2762 Result = DAG.getSplatBuildVector(VT: VecT, DL, Op: SplatValue);
2763 }
2764 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2765 return Lane == SplatValue;
2766 };
2767 }
2768
2769 assert(Result);
2770 assert(IsLaneConstructed);
2771
2772 // Add replace_lane instructions for any unhandled values
2773 for (size_t I = 0; I < Lanes; ++I) {
2774 const SDValue &Lane = Op->getOperand(Num: I);
2775 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2776 Result = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: VecT, N1: Result, N2: Lane,
2777 N3: DAG.getConstant(Val: I, DL, VT: MVT::i32));
2778 }
2779
2780 return Result;
2781}
2782
2783SDValue
2784WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2785 SelectionDAG &DAG) const {
2786 SDLoc DL(Op);
2787 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val: Op.getNode())->getMask();
2788 MVT VecType = Op.getOperand(i: 0).getSimpleValueType();
2789 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
2790 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2791
2792 // Space for two vector args and sixteen mask indices
2793 SDValue Ops[18];
2794 size_t OpIdx = 0;
2795 Ops[OpIdx++] = Op.getOperand(i: 0);
2796 Ops[OpIdx++] = Op.getOperand(i: 1);
2797
2798 // Expand mask indices to byte indices and materialize them as operands
2799 for (int M : Mask) {
2800 for (size_t J = 0; J < LaneBytes; ++J) {
2801 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2802 // whole lane of vector input, to allow further reduction at VM. E.g.
2803 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2804 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2805 Ops[OpIdx++] = DAG.getConstant(Val: ByteIndex, DL, VT: MVT::i32);
2806 }
2807 }
2808
2809 return DAG.getNode(Opcode: WebAssemblyISD::SHUFFLE, DL, VT: Op.getValueType(), Ops);
2810}
2811
2812SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2813 SelectionDAG &DAG) const {
2814 SDLoc DL(Op);
2815 // The legalizer does not know how to expand the unsupported comparison modes
2816 // of i64x2 vectors, so we manually unroll them here.
2817 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2818 SmallVector<SDValue, 2> LHS, RHS;
2819 DAG.ExtractVectorElements(Op: Op->getOperand(Num: 0), Args&: LHS);
2820 DAG.ExtractVectorElements(Op: Op->getOperand(Num: 1), Args&: RHS);
2821 const SDValue &CC = Op->getOperand(Num: 2);
2822 auto MakeLane = [&](unsigned I) {
2823 return DAG.getNode(Opcode: ISD::SELECT_CC, DL, VT: MVT::i64, N1: LHS[I], N2: RHS[I],
2824 N3: DAG.getConstant(Val: uint64_t(-1), DL, VT: MVT::i64),
2825 N4: DAG.getConstant(Val: uint64_t(0), DL, VT: MVT::i64), N5: CC);
2826 };
2827 return DAG.getBuildVector(VT: Op->getValueType(ResNo: 0), DL,
2828 Ops: {MakeLane(0), MakeLane(1)});
2829}
2830
2831SDValue
2832WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2833 SelectionDAG &DAG) const {
2834 if (Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
2835 Op.getValueType() == MVT::v8f16) {
2836 // INSERT_VECTOR_ELT can't handle FP16 operands since Wasm doesn't have a
2837 // scalar FP16 type, so cast them to I16s.
2838 SDLoc DL(Op);
2839 SDValue IntVector = DAG.getBitcast(VT: MVT::v8i16, V: Op.getOperand(i: 0));
2840 SDValue IntElement = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: 1));
2841 SDValue Inserted = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: MVT::v8i16,
2842 N1: IntVector, N2: IntElement, N3: Op.getOperand(i: 2));
2843 return DAG.getBitcast(VT: MVT::v8f16, V: Inserted);
2844 }
2845
2846 // Allow constant lane indices, expand variable lane indices
2847 SDNode *IdxNode = Op.getOperand(i: Op.getNumOperands() - 1).getNode();
2848 if (isa<ConstantSDNode>(Val: IdxNode)) {
2849 // Ensure the index type is i32 to match the tablegen patterns
2850 uint64_t Idx = IdxNode->getAsZExtVal();
2851 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
2852 Ops[Op.getNumOperands() - 1] =
2853 DAG.getConstant(Val: Idx, DL: SDLoc(IdxNode), VT: MVT::i32);
2854 return DAG.getNode(Opcode: Op.getOpcode(), DL: SDLoc(Op), VT: Op.getValueType(), Ops);
2855 }
2856 // Perform default expansion
2857 return SDValue();
2858}
2859
2860static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
2861 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2862 // 32-bit and 64-bit unrolled shifts will have proper semantics
2863 if (LaneT.bitsGE(VT: MVT::i32))
2864 return DAG.UnrollVectorOp(N: Op.getNode());
2865 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2866 SDLoc DL(Op);
2867 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2868 SDValue Mask = DAG.getConstant(Val: LaneT.getSizeInBits() - 1, DL, VT: MVT::i32);
2869 unsigned ShiftOpcode = Op.getOpcode();
2870 SmallVector<SDValue, 16> ShiftedElements;
2871 DAG.ExtractVectorElements(Op: Op.getOperand(i: 0), Args&: ShiftedElements, Start: 0, Count: 0, EltVT: MVT::i32);
2872 SmallVector<SDValue, 16> ShiftElements;
2873 DAG.ExtractVectorElements(Op: Op.getOperand(i: 1), Args&: ShiftElements, Start: 0, Count: 0, EltVT: MVT::i32);
2874 SmallVector<SDValue, 16> UnrolledOps;
2875 for (size_t i = 0; i < NumLanes; ++i) {
2876 SDValue MaskedShiftValue =
2877 DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: ShiftElements[i], N2: Mask);
2878 SDValue ShiftedValue = ShiftedElements[i];
2879 if (ShiftOpcode == ISD::SRA)
2880 ShiftedValue = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i32,
2881 N1: ShiftedValue, N2: DAG.getValueType(LaneT));
2882 UnrolledOps.push_back(
2883 Elt: DAG.getNode(Opcode: ShiftOpcode, DL, VT: MVT::i32, N1: ShiftedValue, N2: MaskedShiftValue));
2884 }
2885 return DAG.getBuildVector(VT: Op.getValueType(), DL, Ops: UnrolledOps);
2886}
2887
2888SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2889 SelectionDAG &DAG) const {
2890 SDLoc DL(Op);
2891 // Only manually lower vector shifts
2892 assert(Op.getSimpleValueType().isVector());
2893
2894 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
2895 auto ShiftVal = Op.getOperand(i: 1);
2896
2897 // Try to skip bitmask operation since it is implied inside shift instruction
2898 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
2899 if (MaskOp.getOpcode() != ISD::AND)
2900 return MaskOp;
2901 SDValue LHS = MaskOp.getOperand(i: 0);
2902 SDValue RHS = MaskOp.getOperand(i: 1);
2903 if (MaskOp.getValueType().isVector()) {
2904 APInt MaskVal;
2905 if (!ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: MaskVal))
2906 std::swap(a&: LHS, b&: RHS);
2907
2908 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: MaskVal) &&
2909 MaskVal == MaskBits)
2910 MaskOp = LHS;
2911 } else {
2912 if (!isa<ConstantSDNode>(Val: RHS.getNode()))
2913 std::swap(a&: LHS, b&: RHS);
2914
2915 auto ConstantRHS = dyn_cast<ConstantSDNode>(Val: RHS.getNode());
2916 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2917 MaskOp = LHS;
2918 }
2919
2920 return MaskOp;
2921 };
2922
2923 // Skip vector and operation
2924 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2925 ShiftVal = DAG.getSplatValue(V: ShiftVal);
2926 if (!ShiftVal)
2927 return unrollVectorShift(Op, DAG);
2928
2929 // Skip scalar and operation
2930 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2931 // Use anyext because none of the high bits can affect the shift
2932 ShiftVal = DAG.getAnyExtOrTrunc(Op: ShiftVal, DL, VT: MVT::i32);
2933
2934 unsigned Opcode;
2935 switch (Op.getOpcode()) {
2936 case ISD::SHL:
2937 Opcode = WebAssemblyISD::VEC_SHL;
2938 break;
2939 case ISD::SRA:
2940 Opcode = WebAssemblyISD::VEC_SHR_S;
2941 break;
2942 case ISD::SRL:
2943 Opcode = WebAssemblyISD::VEC_SHR_U;
2944 break;
2945 default:
2946 llvm_unreachable("unexpected opcode");
2947 }
2948
2949 return DAG.getNode(Opcode, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: ShiftVal);
2950}
2951
2952SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2953 SelectionDAG &DAG) const {
2954 EVT ResT = Op.getValueType();
2955 EVT SatVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
2956
2957 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2958 (SatVT == MVT::i32 || SatVT == MVT::i64))
2959 return Op;
2960
2961 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2962 return Op;
2963
2964 if (ResT == MVT::v8i16 && SatVT == MVT::i16)
2965 return Op;
2966
2967 return SDValue();
2968}
2969
2970static bool HasNoSignedZerosOrNaNs(SDValue Op, SelectionDAG &DAG) {
2971 return (Op->getFlags().hasNoNaNs() ||
2972 (DAG.isKnownNeverNaN(Op: Op->getOperand(Num: 0)) &&
2973 DAG.isKnownNeverNaN(Op: Op->getOperand(Num: 1)))) &&
2974 (Op->getFlags().hasNoSignedZeros() ||
2975 DAG.isKnownNeverLogicalZero(Op: Op->getOperand(Num: 0)) ||
2976 DAG.isKnownNeverLogicalZero(Op: Op->getOperand(Num: 1)));
2977}
2978
2979SDValue WebAssemblyTargetLowering::LowerFMIN(SDValue Op,
2980 SelectionDAG &DAG) const {
2981 if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
2982 return DAG.getNode(Opcode: WebAssemblyISD::RELAXED_FMIN, DL: SDLoc(Op),
2983 VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2984 }
2985 return SDValue();
2986}
2987
2988SDValue WebAssemblyTargetLowering::LowerFMAX(SDValue Op,
2989 SelectionDAG &DAG) const {
2990 if (Subtarget->hasRelaxedSIMD() && HasNoSignedZerosOrNaNs(Op, DAG)) {
2991 return DAG.getNode(Opcode: WebAssemblyISD::RELAXED_FMAX, DL: SDLoc(Op),
2992 VT: Op.getValueType(), N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
2993 }
2994 return SDValue();
2995}
2996
2997//===----------------------------------------------------------------------===//
2998// Custom DAG combine hooks
2999//===----------------------------------------------------------------------===//
3000static SDValue
3001performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
3002 auto &DAG = DCI.DAG;
3003 auto Shuffle = cast<ShuffleVectorSDNode>(Val: N);
3004
3005 // Hoist vector bitcasts that don't change the number of lanes out of unary
3006 // shuffles, where they are less likely to get in the way of other combines.
3007 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
3008 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
3009 SDValue Bitcast = N->getOperand(Num: 0);
3010 if (Bitcast.getOpcode() != ISD::BITCAST)
3011 return SDValue();
3012 if (!N->getOperand(Num: 1).isUndef())
3013 return SDValue();
3014 SDValue CastOp = Bitcast.getOperand(i: 0);
3015 EVT SrcType = CastOp.getValueType();
3016 EVT DstType = Bitcast.getValueType();
3017 if (!SrcType.is128BitVector() ||
3018 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
3019 return SDValue();
3020 SDValue NewShuffle = DAG.getVectorShuffle(
3021 VT: SrcType, dl: SDLoc(N), N1: CastOp, N2: DAG.getUNDEF(VT: SrcType), Mask: Shuffle->getMask());
3022 return DAG.getBitcast(VT: DstType, V: NewShuffle);
3023}
3024
3025/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
3026/// split up into scalar instructions during legalization, and the vector
3027/// extending instructions are selected in performVectorExtendCombine below.
3028static SDValue
3029performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
3030 const WebAssemblySubtarget *Subtarget) {
3031 auto &DAG = DCI.DAG;
3032 assert(N->getOpcode() == ISD::UINT_TO_FP ||
3033 N->getOpcode() == ISD::SINT_TO_FP);
3034
3035 EVT InVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3036 EVT ResVT = N->getValueType(ResNo: 0);
3037 MVT ExtVT;
3038 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
3039 ExtVT = MVT::v4i32;
3040 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
3041 ExtVT = MVT::v2i32;
3042 else if (Subtarget->hasFP16() && ResVT == MVT::v8f16 && InVT == MVT::v8i8)
3043 ExtVT = MVT::v8i16;
3044 else
3045 return SDValue();
3046
3047 unsigned Op =
3048 N->getOpcode() == ISD::UINT_TO_FP ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
3049 SDValue Conv = DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ExtVT, Operand: N->getOperand(Num: 0));
3050 return DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: ResVT, Operand: Conv);
3051}
3052
3053static SDValue
3054performVectorNonNegToFPCombine(SDNode *N,
3055 TargetLowering::DAGCombinerInfo &DCI) {
3056 auto &DAG = DCI.DAG;
3057
3058 SDNodeFlags Flags = N->getFlags();
3059 SDValue Op0 = N->getOperand(Num: 0);
3060 EVT VT = N->getValueType(ResNo: 0);
3061
3062 // Optimize uitofp to sitofp when the sign bit is known to be zero.
3063 // Depending on the target (runtime) backend, this might be performance
3064 // neutral (e.g. AArch64) or a significant improvement (e.g. x86_64).
3065 if (VT.isVector() && (Flags.hasNonNeg() || DAG.SignBitIsZero(Op: Op0))) {
3066 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL: SDLoc(N), VT, Operand: Op0);
3067 }
3068
3069 return SDValue();
3070}
3071
3072static SDValue
3073performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
3074 auto &DAG = DCI.DAG;
3075 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
3076 N->getOpcode() == ISD::ZERO_EXTEND);
3077
3078 EVT ResVT = N->getValueType(ResNo: 0);
3079 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
3080 SDLoc DL(N);
3081
3082 if (ResVT == MVT::v16i32 && N->getOperand(Num: 0)->getValueType(ResNo: 0) == MVT::v16i8) {
3083 // Use a tree of extend low/high to split and extend the input in two
3084 // layers to avoid doing several shuffles and even more extends.
3085 unsigned LowOp =
3086 IsSext ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3087 unsigned HighOp =
3088 IsSext ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3089 SDValue Input = N->getOperand(Num: 0);
3090 SDValue LowHalf = DAG.getNode(Opcode: LowOp, DL, VT: MVT::v8i16, Operand: Input);
3091 SDValue HighHalf = DAG.getNode(Opcode: HighOp, DL, VT: MVT::v8i16, Operand: Input);
3092 SDValue Subvectors[] = {
3093 DAG.getNode(Opcode: LowOp, DL, VT: MVT::v4i32, Operand: LowHalf),
3094 DAG.getNode(Opcode: HighOp, DL, VT: MVT::v4i32, Operand: LowHalf),
3095 DAG.getNode(Opcode: LowOp, DL, VT: MVT::v4i32, Operand: HighHalf),
3096 DAG.getNode(Opcode: HighOp, DL, VT: MVT::v4i32, Operand: HighHalf),
3097 };
3098 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ResVT, Ops: Subvectors);
3099 }
3100
3101 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
3102 // possible before the extract_subvector can be expanded.
3103 auto Extract = N->getOperand(Num: 0);
3104 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
3105 return SDValue();
3106 auto Source = Extract.getOperand(i: 0);
3107 auto *IndexNode = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
3108 if (IndexNode == nullptr)
3109 return SDValue();
3110 auto Index = IndexNode->getZExtValue();
3111
3112 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
3113 // extracted subvector is the low or high half of its source.
3114 if (ResVT == MVT::v8i16) {
3115 if (Extract.getValueType() != MVT::v8i8 ||
3116 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
3117 return SDValue();
3118 } else if (ResVT == MVT::v4i32) {
3119 if (Extract.getValueType() != MVT::v4i16 ||
3120 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
3121 return SDValue();
3122 } else if (ResVT == MVT::v2i64) {
3123 if (Extract.getValueType() != MVT::v2i32 ||
3124 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
3125 return SDValue();
3126 } else {
3127 return SDValue();
3128 }
3129
3130 bool IsLow = Index == 0;
3131
3132 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
3133 : WebAssemblyISD::EXTEND_HIGH_S)
3134 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
3135 : WebAssemblyISD::EXTEND_HIGH_U);
3136
3137 return DAG.getNode(Opcode: Op, DL, VT: ResVT, Operand: Source);
3138}
3139
3140static SDValue
3141performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
3142 auto &DAG = DCI.DAG;
3143
3144 auto GetWasmConversionOp = [](unsigned Op) {
3145 switch (Op) {
3146 case ISD::FP_TO_SINT_SAT:
3147 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
3148 case ISD::FP_TO_UINT_SAT:
3149 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
3150 case ISD::FP_ROUND:
3151 return WebAssemblyISD::DEMOTE_ZERO;
3152 }
3153 llvm_unreachable("unexpected op");
3154 };
3155
3156 auto IsZeroSplat = [](SDValue SplatVal) {
3157 auto *Splat = dyn_cast<BuildVectorSDNode>(Val: SplatVal.getNode());
3158 APInt SplatValue, SplatUndef;
3159 unsigned SplatBitSize;
3160 bool HasAnyUndefs;
3161 // Endianness doesn't matter in this context because we are looking for
3162 // an all-zero value.
3163 return Splat &&
3164 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
3165 HasAnyUndefs) &&
3166 SplatValue == 0;
3167 };
3168
3169 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
3170 // Combine this:
3171 //
3172 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
3173 //
3174 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
3175 //
3176 // Or this:
3177 //
3178 // (concat_vectors ({v2f32, v4f16} (fp_round ({v2f64, v4f32} $x))),
3179 // ({v2f32, v4f16} (splat 0)))
3180 //
3181 // into ({f32x4, f16x8}.demote_zero_{f64x2, f32x4} $x).
3182 EVT ResVT;
3183 EVT ExpectedConversionType;
3184 auto Conversion = N->getOperand(Num: 0);
3185 auto ConversionOp = Conversion.getOpcode();
3186 switch (ConversionOp) {
3187 case ISD::FP_TO_SINT_SAT:
3188 case ISD::FP_TO_UINT_SAT:
3189 ResVT = MVT::v4i32;
3190 ExpectedConversionType = MVT::v2i32;
3191 break;
3192 case ISD::FP_ROUND:
3193 if (Conversion.getValueType() == MVT::v2f32) {
3194 ResVT = MVT::v4f32;
3195 ExpectedConversionType = MVT::v2f32;
3196 } else if (Conversion.getValueType() == MVT::v4f16) {
3197 ResVT = MVT::v8f16;
3198 ExpectedConversionType = MVT::v4f16;
3199 } else {
3200 return SDValue();
3201 }
3202 break;
3203 default:
3204 return SDValue();
3205 }
3206
3207 if (N->getValueType(ResNo: 0) != ResVT)
3208 return SDValue();
3209
3210 if (Conversion.getValueType() != ExpectedConversionType)
3211 return SDValue();
3212
3213 auto Source = Conversion.getOperand(i: 0);
3214 if (!((Source.getValueType() == MVT::v2f64 && ResVT == MVT::v4f32) ||
3215 (Source.getValueType() == MVT::v2f64 && ResVT == MVT::v4i32) ||
3216 (Source.getValueType() == MVT::v4f32 && ResVT == MVT::v8f16)))
3217 return SDValue();
3218
3219 if (!IsZeroSplat(N->getOperand(Num: 1)) ||
3220 N->getOperand(Num: 1).getValueType() != ExpectedConversionType)
3221 return SDValue();
3222
3223 unsigned Op = GetWasmConversionOp(ConversionOp);
3224 return DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ResVT, Operand: Source);
3225 }
3226
3227 // Combine this:
3228 //
3229 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
3230 //
3231 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
3232 //
3233 // Or this:
3234 //
3235 // ({v4f32, v8f16} (fp_round (concat_vectors $x,
3236 // ({v2f64, v4f32} (splat 0)))))
3237 //
3238 // into ({f32x4, f16x8}.demote_zero_{f64x2, f32x4} $x).
3239 EVT ResVT;
3240 auto ConversionOp = N->getOpcode();
3241 switch (ConversionOp) {
3242 case ISD::FP_TO_SINT_SAT:
3243 case ISD::FP_TO_UINT_SAT:
3244 ResVT = MVT::v4i32;
3245 break;
3246 case ISD::FP_ROUND:
3247 ResVT = N->getValueType(ResNo: 0);
3248 break;
3249 default:
3250 llvm_unreachable("unexpected op");
3251 }
3252
3253 if (N->getValueType(ResNo: 0) != ResVT)
3254 return SDValue();
3255
3256 auto Concat = N->getOperand(Num: 0);
3257 if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
3258 return SDValue();
3259 EVT ConcatVT = Concat.getValueType();
3260 EVT SourceVT = Concat.getOperand(i: 0).getValueType();
3261
3262 if (!IsZeroSplat(Concat.getOperand(i: 1)))
3263 return SDValue();
3264
3265 if (ConversionOp == ISD::FP_ROUND) {
3266 bool IsF64ToF32 =
3267 ConcatVT == MVT::v4f64 && SourceVT == MVT::v2f64 && ResVT == MVT::v4f32;
3268 bool IsF32ToF16 =
3269 ConcatVT == MVT::v8f32 && SourceVT == MVT::v4f32 && ResVT == MVT::v8f16;
3270 if (!(IsF64ToF32 || IsF32ToF16))
3271 return SDValue();
3272 } else {
3273 if (ConcatVT != MVT::v4f64 || SourceVT != MVT::v2f64 || ResVT != MVT::v4i32)
3274 return SDValue();
3275 }
3276
3277 unsigned Op = GetWasmConversionOp(ConversionOp);
3278 return DAG.getNode(Opcode: Op, DL: SDLoc(N), VT: ResVT, Operand: Concat.getOperand(i: 0));
3279}
3280
3281// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
3282static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3283 const SDLoc &DL, unsigned VectorWidth) {
3284 EVT VT = Vec.getValueType();
3285 EVT ElVT = VT.getVectorElementType();
3286 unsigned Factor = VT.getSizeInBits() / VectorWidth;
3287 EVT ResultVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: ElVT,
3288 NumElements: VT.getVectorNumElements() / Factor);
3289
3290 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
3291 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
3292 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3293
3294 // This is the index of the first element of the VectorWidth-bit chunk
3295 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3296 IdxVal &= ~(ElemsPerChunk - 1);
3297
3298 // If the input is a buildvector just emit a smaller one.
3299 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3300 return DAG.getBuildVector(VT: ResultVT, DL,
3301 Ops: Vec->ops().slice(N: IdxVal, M: ElemsPerChunk));
3302
3303 SDValue VecIdx = DAG.getIntPtrConstant(Val: IdxVal, DL);
3304 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ResultVT, N1: Vec, N2: VecIdx);
3305}
3306
3307// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
3308// is the expected destination value type after recursion. In is the initial
3309// input. Note that the input should have enough leading zero bits to prevent
3310// NARROW_U from saturating results.
3311static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,
3312 SelectionDAG &DAG) {
3313 EVT SrcVT = In.getValueType();
3314
3315 // No truncation required, we might get here due to recursive calls.
3316 if (SrcVT == DstVT)
3317 return In;
3318
3319 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
3320 unsigned NumElems = SrcVT.getVectorNumElements();
3321 if (!isPowerOf2_32(Value: NumElems))
3322 return SDValue();
3323 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
3324 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
3325
3326 LLVMContext &Ctx = *DAG.getContext();
3327 EVT PackedSVT = EVT::getIntegerVT(Context&: Ctx, BitWidth: SrcVT.getScalarSizeInBits() / 2);
3328
3329 // Narrow to the largest type possible:
3330 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
3331 EVT InVT = MVT::i16, OutVT = MVT::i8;
3332 if (SrcVT.getScalarSizeInBits() > 16) {
3333 InVT = MVT::i32;
3334 OutVT = MVT::i16;
3335 }
3336 unsigned SubSizeInBits = SrcSizeInBits / 2;
3337 InVT = EVT::getVectorVT(Context&: Ctx, VT: InVT, NumElements: SubSizeInBits / InVT.getSizeInBits());
3338 OutVT = EVT::getVectorVT(Context&: Ctx, VT: OutVT, NumElements: SubSizeInBits / OutVT.getSizeInBits());
3339
3340 // Split lower/upper subvectors.
3341 SDValue Lo = extractSubVector(Vec: In, IdxVal: 0, DAG, DL, VectorWidth: SubSizeInBits);
3342 SDValue Hi = extractSubVector(Vec: In, IdxVal: NumElems / 2, DAG, DL, VectorWidth: SubSizeInBits);
3343
3344 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
3345 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
3346 Lo = DAG.getBitcast(VT: InVT, V: Lo);
3347 Hi = DAG.getBitcast(VT: InVT, V: Hi);
3348 SDValue Res = DAG.getNode(Opcode: WebAssemblyISD::NARROW_U, DL, VT: OutVT, N1: Lo, N2: Hi);
3349 return DAG.getBitcast(VT: DstVT, V: Res);
3350 }
3351
3352 // Recursively narrow lower/upper subvectors, concat result and narrow again.
3353 EVT PackedVT = EVT::getVectorVT(Context&: Ctx, VT: PackedSVT, NumElements: NumElems / 2);
3354 Lo = truncateVectorWithNARROW(DstVT: PackedVT, In: Lo, DL, DAG);
3355 Hi = truncateVectorWithNARROW(DstVT: PackedVT, In: Hi, DL, DAG);
3356
3357 PackedVT = EVT::getVectorVT(Context&: Ctx, VT: PackedSVT, NumElements: NumElems);
3358 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: PackedVT, N1: Lo, N2: Hi);
3359 return truncateVectorWithNARROW(DstVT, In: Res, DL, DAG);
3360}
3361
3362static SDValue performTruncateCombine(SDNode *N,
3363 TargetLowering::DAGCombinerInfo &DCI) {
3364 auto &DAG = DCI.DAG;
3365
3366 SDValue In = N->getOperand(Num: 0);
3367 EVT InVT = In.getValueType();
3368 if (!InVT.isSimple())
3369 return SDValue();
3370
3371 EVT OutVT = N->getValueType(ResNo: 0);
3372 if (!OutVT.isVector())
3373 return SDValue();
3374
3375 EVT OutSVT = OutVT.getVectorElementType();
3376 EVT InSVT = InVT.getVectorElementType();
3377 // Currently only cover truncate to v16i8 or v8i16.
3378 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
3379 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
3380 return SDValue();
3381
3382 SDLoc DL(N);
3383 APInt Mask = APInt::getLowBitsSet(numBits: InVT.getScalarSizeInBits(),
3384 loBitsSet: OutVT.getScalarSizeInBits());
3385 In = DAG.getNode(Opcode: ISD::AND, DL, VT: InVT, N1: In, N2: DAG.getConstant(Val: Mask, DL, VT: InVT));
3386 return truncateVectorWithNARROW(DstVT: OutVT, In, DL, DAG);
3387}
3388
3389static SDValue performBitcastCombine(SDNode *N,
3390 TargetLowering::DAGCombinerInfo &DCI) {
3391 using namespace llvm::SDPatternMatch;
3392 auto &DAG = DCI.DAG;
3393 SDLoc DL(N);
3394 SDValue Src = N->getOperand(Num: 0);
3395 EVT VT = N->getValueType(ResNo: 0);
3396 EVT SrcVT = Src.getValueType();
3397
3398 if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
3399 SrcVT.isFixedLengthVectorOf(EltVT: MVT::i1)))
3400 return SDValue();
3401
3402 unsigned NumElts = SrcVT.getVectorNumElements();
3403 EVT Width = MVT::getIntegerVT(BitWidth: 128 / NumElts);
3404
3405 // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
3406 // ==> bitmask
3407 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
3408 return DAG.getZExtOrTrunc(
3409 Op: DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3410 Ops: {DAG.getConstant(Val: Intrinsic::wasm_bitmask, DL, VT: MVT::i32),
3411 DAG.getSExtOrTrunc(Op: N->getOperand(Num: 0), DL,
3412 VT: SrcVT.changeVectorElementType(
3413 Context&: *DAG.getContext(), EltVT: Width))}),
3414 DL, VT);
3415 }
3416
3417 // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
3418 if (NumElts == 32 || NumElts == 64) {
3419 SDValue Concat, SetCCVector;
3420 ISD::CondCode SetCond;
3421
3422 if (!sd_match(N, P: m_BitCast(Op: m_c_SetCC(CC&: SetCond, LHS: m_Value(N&: Concat),
3423 RHS: m_Value(N&: SetCCVector)))))
3424 return SDValue();
3425 if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
3426 return SDValue();
3427
3428 // Reconstruct the wide bitmask from each CONCAT_VECTORS operand.
3429 // Derive the per-chunk mask/integer types from the actual operand type
3430 // instead of hardcoding v16i1 / i16 for every chunk.
3431 EVT ConcatOperandVT = Concat.getOperand(i: 0).getValueType();
3432 unsigned ConcatOperandNumElts = ConcatOperandVT.getVectorNumElements();
3433
3434 EVT ConcatOperandMaskVT =
3435 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
3436 EC: ElementCount::getFixed(MinVal: ConcatOperandNumElts));
3437 EVT ConcatOperandBitmaskVT =
3438 EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ConcatOperandNumElts);
3439 EVT ReturnVT = N->getValueType(ResNo: 0);
3440 SDValue ReconstructedBitmask = DAG.getConstant(Val: 0, DL, VT: ReturnVT);
3441 // Example:
3442 // v32i16 = concat(v8i16, v8i16, v8i16, v8i16)
3443 // -> v8i1 + v8i1 + v8i1 + v8i1
3444 // -> i8 + i8 + i8 + i8
3445 // -> reconstructed i32 bitmask
3446 for (size_t I = 0; I < Concat->ops().size(); ++I) {
3447 SDValue ConcatOperand = Concat.getOperand(i: I);
3448 assert(ConcatOperand.getValueType() == ConcatOperandVT &&
3449 "concat_vectors operands must have the same type");
3450
3451 SDValue SetCCVectorOperand =
3452 extractSubVector(Vec: SetCCVector, IdxVal: I * ConcatOperandNumElts, DAG, DL, VectorWidth: 128);
3453 if (!SetCCVectorOperand ||
3454 SetCCVectorOperand.getValueType() != ConcatOperandVT)
3455 return SDValue();
3456
3457 // Build the per-chunk mask using the correct chunk type:
3458 // v16i8 -> v16i1 -> i16
3459 // v8i16 -> v8i1 -> i8
3460 // v4i32 -> v4i1 -> i4
3461 // v2i64 -> v2i1 -> i2
3462 SDValue ConcatOperandMask = DAG.getSetCC(
3463 DL, VT: ConcatOperandMaskVT, LHS: ConcatOperand, RHS: SetCCVectorOperand, Cond: SetCond);
3464 SDValue ConcatOperandBitmask =
3465 DAG.getBitcast(VT: ConcatOperandBitmaskVT, V: ConcatOperandMask);
3466 SDValue ExtendedConcatOperandBitmask =
3467 DAG.getZExtOrTrunc(Op: ConcatOperandBitmask, DL, VT: ReturnVT);
3468
3469 // Shift each chunk's mask to its original lane position before merging it
3470 // into the result:
3471 // Result bit index = I * ConcatOperandNumElts + local lane index
3472 //
3473 // Example: four chunks, each containing 8 mask bits:
3474 // result = M0 | (M1 << 8) | (M2 << 16) | (M3 << 24)
3475 //
3476 SDValue PositionedChunkBitmask = ExtendedConcatOperandBitmask;
3477 if (I != 0) {
3478 PositionedChunkBitmask = DAG.getNode(
3479 Opcode: ISD::SHL, DL, VT: ReturnVT, N1: ExtendedConcatOperandBitmask,
3480 N2: DAG.getShiftAmountConstant(Val: I * ConcatOperandNumElts, VT: ReturnVT, DL));
3481 }
3482
3483 ReconstructedBitmask = DAG.getNode(
3484 Opcode: ISD::OR, DL, VT: ReturnVT, N1: ReconstructedBitmask, N2: PositionedChunkBitmask);
3485 }
3486
3487 return ReconstructedBitmask;
3488 }
3489
3490 return SDValue();
3491}
3492
3493static SDValue performBitmaskCombine(SDNode *N, SelectionDAG &DAG) {
3494 // bitmask (setcc <X>, 0, setlt) => bitmask X
3495 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
3496 using namespace llvm::SDPatternMatch;
3497
3498 if (N->getConstantOperandVal(Num: 0) != Intrinsic::wasm_bitmask)
3499 return SDValue();
3500
3501 SDValue LHS;
3502 if (!sd_match(N: N->getOperand(Num: 1),
3503 P: m_c_SpecificSetCC(CC: ISD::SETLT, LHS: m_Value(N&: LHS), RHS: m_Zero())))
3504 return SDValue();
3505
3506 SDLoc DL(N);
3507 return DAG.getNode(
3508 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: N->getValueType(ResNo: 0),
3509 Ops: {DAG.getConstant(Val: Intrinsic::wasm_bitmask, DL, VT: MVT::i32), LHS});
3510}
3511
3512static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG) {
3513 // any_true (setcc <X>, 0, eq) => (not (all_true X))
3514 // all_true (setcc <X>, 0, eq) => (not (any_true X))
3515 // any_true (setcc <X>, 0, ne) => (any_true X)
3516 // all_true (setcc <X>, 0, ne) => (all_true X)
3517 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
3518 using namespace llvm::SDPatternMatch;
3519
3520 SDValue LHS;
3521 if (N->getNumOperands() < 2 ||
3522 !sd_match(N: N->getOperand(Num: 1), P: m_c_SetCC(LHS: m_Value(N&: LHS), RHS: m_Zero())))
3523 return SDValue();
3524 EVT LT = LHS.getValueType();
3525 if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
3526 return SDValue();
3527
3528 auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
3529 ISD::CondCode SetType,
3530 Intrinsic::WASMIntrinsics InPost) {
3531 if (N->getConstantOperandVal(Num: 0) != InPre)
3532 return SDValue();
3533
3534 SDValue LHS;
3535 if (!sd_match(N: N->getOperand(Num: 1),
3536 P: m_c_SpecificSetCC(CC: SetType, LHS: m_Value(N&: LHS), RHS: m_Zero())))
3537 return SDValue();
3538
3539 SDLoc DL(N);
3540 SDValue Ret = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3541 Ops: {DAG.getConstant(Val: InPost, DL, VT: MVT::i32), LHS});
3542 if (SetType == ISD::SETEQ)
3543 Ret = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Ret,
3544 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
3545 return DAG.getZExtOrTrunc(Op: Ret, DL, VT: N->getValueType(ResNo: 0));
3546 };
3547
3548 if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
3549 Intrinsic::wasm_alltrue))
3550 return AnyTrueEQ;
3551 if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
3552 Intrinsic::wasm_anytrue))
3553 return AllTrueEQ;
3554 if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
3555 Intrinsic::wasm_anytrue))
3556 return AnyTrueNE;
3557 if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
3558 Intrinsic::wasm_alltrue))
3559 return AllTrueNE;
3560
3561 return SDValue();
3562}
3563
3564struct MaskReduceInfo {
3565 Intrinsic::ID IID;
3566 unsigned WideCombineOpcode;
3567 bool Invert;
3568};
3569
3570static SDValue combineSmallMaskReduction(SDNode *N, EVT FromVT,
3571 unsigned NumElts,
3572 const MaskReduceInfo &Info,
3573 SelectionDAG &DAG) {
3574 EVT VecVT = FromVT.changeVectorElementType(Context&: *DAG.getContext(),
3575 EltVT: MVT::getIntegerVT(BitWidth: 128 / NumElts));
3576 assert(VecVT.getSizeInBits() == 128 &&
3577 "mask reduction should be widened to a 128-bit vector");
3578
3579 SDLoc DL(N);
3580 SDValue Mask = N->getOperand(Num: 0)->getOperand(Num: 0);
3581 SDValue Ret = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3582 Ops: {DAG.getConstant(Val: Info.IID, DL, VT: MVT::i32),
3583 DAG.getSExtOrTrunc(Op: Mask, DL, VT: VecVT)});
3584 if (Info.Invert)
3585 Ret = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Ret,
3586 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
3587 return DAG.getZExtOrTrunc(Op: Ret, DL, VT: N->getValueType(ResNo: 0));
3588}
3589
3590static SDValue combineWideMaskReduction(SDNode *N, SDValue Mask, EVT MaskVT,
3591 unsigned NumElts,
3592 const MaskReduceInfo &Info,
3593 SelectionDAG &DAG) {
3594 assert((NumElts == 32 || NumElts == 64) &&
3595 "combineWideMaskReduction is only for wide masks");
3596 assert(MaskVT.isFixedLengthVector() &&
3597 MaskVT.getVectorElementType() == MVT::i1);
3598 SDLoc DL(N);
3599 unsigned ChunkElts = 16;
3600 EVT ChunkMaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
3601 EC: ElementCount::getFixed(MinVal: ChunkElts));
3602 EVT LegalVecVT = ChunkMaskVT.changeVectorElementType(
3603 Context&: *DAG.getContext(), EltVT: MVT::getIntegerVT(BitWidth: 128 / ChunkElts));
3604
3605 SmallVector<SDValue, 4> ChunkResults;
3606 // Split the wide mask into v16i1 chunks and reduce each chunk separately.
3607 // For example:
3608 // v32i1: [0..15] [16..31]
3609 // | |
3610 // v v
3611 // chunk0 chunk1
3612 //
3613 // v64i1: [0..15] [16..31] [32..47] [48..63]
3614 // | | | |
3615 // v v v v
3616 // chunk0 chunk1 chunk2 chunk3
3617 //
3618 // each chunk:
3619 // v16i1 -> v16i8 -> wasm_anytrue/alltrue -> i32 0/1
3620 for (unsigned I = 0; I < NumElts; I += ChunkElts) {
3621 SDValue ChunkMask = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ChunkMaskVT,
3622 N1: Mask, N2: DAG.getVectorIdxConstant(Val: I, DL));
3623 SDValue LegalMask = DAG.getSExtOrTrunc(Op: ChunkMask, DL, VT: LegalVecVT);
3624 SDValue Reduced =
3625 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3626 N1: DAG.getConstant(Val: Info.IID, DL, VT: MVT::i32), N2: LegalMask);
3627 ChunkResults.push_back(Elt: Reduced);
3628 }
3629
3630 SDValue Acc = ChunkResults[0];
3631 for (unsigned I = 1; I < ChunkResults.size(); ++I)
3632 Acc =
3633 DAG.getNode(Opcode: Info.WideCombineOpcode, DL, VT: MVT::i32, N1: Acc, N2: ChunkResults[I]);
3634
3635 if (Info.Invert)
3636 Acc = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Acc,
3637 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i32));
3638
3639 return DAG.getZExtOrTrunc(Op: Acc, DL, VT: N->getValueType(ResNo: 0));
3640}
3641
3642static std::optional<MaskReduceInfo> classifyMaskReduction(SDNode *N) {
3643 auto *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
3644 if (!C)
3645 return std::nullopt;
3646
3647 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3648
3649 // setcc (bitcast mask), 0, ne -> any_true(mask)
3650 if (C->isZero() && CC == ISD::SETNE)
3651 return MaskReduceInfo{.IID: Intrinsic::wasm_anytrue, .WideCombineOpcode: ISD::OR, .Invert: false};
3652
3653 // setcc (bitcast mask), 0, eq -> !any_true(mask)
3654 if (C->isZero() && CC == ISD::SETEQ)
3655 return MaskReduceInfo{.IID: Intrinsic::wasm_anytrue, .WideCombineOpcode: ISD::OR, .Invert: true};
3656
3657 // setcc (bitcast mask), -1, eq -> all_true(mask)
3658 if (C->isAllOnes() && CC == ISD::SETEQ)
3659 return MaskReduceInfo{.IID: Intrinsic::wasm_alltrue, .WideCombineOpcode: ISD::AND, .Invert: false};
3660
3661 // setcc (bitcast mask), -1, ne -> !all_true(mask)
3662 if (C->isAllOnes() && CC == ISD::SETNE)
3663 return MaskReduceInfo{.IID: Intrinsic::wasm_alltrue, .WideCombineOpcode: ISD::AND, .Invert: true};
3664
3665 return std::nullopt;
3666}
3667
3668/// Try to convert a i128 comparison to a v16i8 comparison before type
3669/// legalization splits it up into chunks
3670static SDValue
3671combineVectorSizedSetCCEquality(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
3672 const WebAssemblySubtarget *Subtarget) {
3673
3674 SDLoc DL(N);
3675 SDValue X = N->getOperand(Num: 0);
3676 SDValue Y = N->getOperand(Num: 1);
3677 EVT VT = N->getValueType(ResNo: 0);
3678 EVT OpVT = X.getValueType();
3679
3680 SelectionDAG &DAG = DCI.DAG;
3681 if (DCI.DAG.getMachineFunction().getFunction().hasFnAttribute(
3682 Kind: Attribute::NoImplicitFloat))
3683 return SDValue();
3684
3685 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
3686 // We're looking for an oversized integer equality comparison with SIMD
3687 if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||
3688 !Subtarget->hasSIMD128() || !isIntEqualitySetCC(Code: CC))
3689 return SDValue();
3690
3691 // Don't perform this combine if constructing the vector will be expensive.
3692 auto IsVectorBitCastCheap = [](SDValue X) {
3693 X = peekThroughBitcasts(V: X);
3694 return isa<ConstantSDNode>(Val: X) || X.getOpcode() == ISD::LOAD;
3695 };
3696
3697 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
3698 return SDValue();
3699
3700 SDValue VecX = DAG.getBitcast(VT: MVT::v16i8, V: X);
3701 SDValue VecY = DAG.getBitcast(VT: MVT::v16i8, V: Y);
3702 SDValue Cmp = DAG.getSetCC(DL, VT: MVT::v16i8, LHS: VecX, RHS: VecY, Cond: CC);
3703
3704 SDValue Intr =
3705 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
3706 Ops: {DAG.getConstant(Val: CC == ISD::SETEQ ? Intrinsic::wasm_alltrue
3707 : Intrinsic::wasm_anytrue,
3708 DL, VT: MVT::i32),
3709 Cmp});
3710
3711 return DAG.getSetCC(DL, VT, LHS: Intr, RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
3712 Cond: ISD::SETNE);
3713}
3714
3715static SDValue performSETCCCombine(SDNode *N,
3716 TargetLowering::DAGCombinerInfo &DCI,
3717 const WebAssemblySubtarget *Subtarget) {
3718 if (!DCI.isBeforeLegalize())
3719 return SDValue();
3720
3721 EVT VT = N->getValueType(ResNo: 0);
3722 if (!VT.isScalarInteger())
3723 return SDValue();
3724
3725 if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))
3726 return V;
3727
3728 SDValue LHS = N->getOperand(Num: 0);
3729 if (LHS->getOpcode() != ISD::BITCAST)
3730 return SDValue();
3731
3732 EVT FromVT = LHS->getOperand(Num: 0).getValueType();
3733 if (!FromVT.isFixedLengthVectorOf(EltVT: MVT::i1))
3734 return SDValue();
3735
3736 unsigned NumElts = FromVT.getVectorNumElements();
3737 auto Info = classifyMaskReduction(N);
3738 if (!Info)
3739 return SDValue();
3740
3741 auto &DAG = DCI.DAG;
3742 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16)
3743 return combineSmallMaskReduction(N, FromVT, NumElts, Info: *Info, DAG);
3744
3745 if (NumElts == 32 || NumElts == 64)
3746 return combineWideMaskReduction(N, Mask: LHS.getOperand(i: 0), MaskVT: FromVT, NumElts,
3747 Info: *Info, DAG);
3748
3749 return SDValue();
3750}
3751
3752static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG) {
3753 EVT VT = N->getValueType(ResNo: 0);
3754 if (VT != MVT::v8i32 && VT != MVT::v16i32)
3755 return SDValue();
3756
3757 // Mul with extending inputs.
3758 SDValue LHS = N->getOperand(Num: 0);
3759 SDValue RHS = N->getOperand(Num: 1);
3760 if (LHS.getOpcode() != RHS.getOpcode())
3761 return SDValue();
3762
3763 if (LHS.getOpcode() != ISD::SIGN_EXTEND &&
3764 LHS.getOpcode() != ISD::ZERO_EXTEND)
3765 return SDValue();
3766
3767 if (LHS->getOperand(Num: 0).getValueType() != RHS->getOperand(Num: 0).getValueType())
3768 return SDValue();
3769
3770 EVT FromVT = LHS->getOperand(Num: 0).getValueType();
3771 EVT EltTy = FromVT.getVectorElementType();
3772 if (EltTy != MVT::i8)
3773 return SDValue();
3774
3775 // For an input DAG that looks like this
3776 // %a = input_type
3777 // %b = input_type
3778 // %lhs = extend %a to output_type
3779 // %rhs = extend %b to output_type
3780 // %mul = mul %lhs, %rhs
3781
3782 // input_type | output_type | instructions
3783 // v16i8 | v16i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3784 // | | %high = i16x8.extmul_high_i8x16_, %a, %b
3785 // | | %low_low = i32x4.ext_low_i16x8_ %low
3786 // | | %low_high = i32x4.ext_high_i16x8_ %low
3787 // | | %high_low = i32x4.ext_low_i16x8_ %high
3788 // | | %high_high = i32x4.ext_high_i16x8_ %high
3789 // | | %res = concat_vector(...)
3790 // v8i8 | v8i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3791 // | | %low_low = i32x4.ext_low_i16x8_ %low
3792 // | | %low_high = i32x4.ext_high_i16x8_ %low
3793 // | | %res = concat_vector(%low_low, %low_high)
3794
3795 SDLoc DL(N);
3796 unsigned NumElts = VT.getVectorNumElements();
3797 SDValue ExtendInLHS = LHS->getOperand(Num: 0);
3798 SDValue ExtendInRHS = RHS->getOperand(Num: 0);
3799 bool IsSigned = LHS->getOpcode() == ISD::SIGN_EXTEND;
3800 unsigned ExtendLowOpc =
3801 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3802 unsigned ExtendHighOpc =
3803 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3804
3805 auto GetExtendLow = [&DAG, &DL, &ExtendLowOpc](EVT VT, SDValue Op) {
3806 return DAG.getNode(Opcode: ExtendLowOpc, DL, VT, Operand: Op);
3807 };
3808 auto GetExtendHigh = [&DAG, &DL, &ExtendHighOpc](EVT VT, SDValue Op) {
3809 return DAG.getNode(Opcode: ExtendHighOpc, DL, VT, Operand: Op);
3810 };
3811
3812 if (NumElts == 16) {
3813 SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);
3814 SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);
3815 SDValue MulLow = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: LowLHS, N2: LowRHS);
3816 SDValue HighLHS = GetExtendHigh(MVT::v8i16, ExtendInLHS);
3817 SDValue HighRHS = GetExtendHigh(MVT::v8i16, ExtendInRHS);
3818 SDValue MulHigh = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: HighLHS, N2: HighRHS);
3819 SDValue SubVectors[] = {
3820 GetExtendLow(MVT::v4i32, MulLow),
3821 GetExtendHigh(MVT::v4i32, MulLow),
3822 GetExtendLow(MVT::v4i32, MulHigh),
3823 GetExtendHigh(MVT::v4i32, MulHigh),
3824 };
3825 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: SubVectors);
3826 } else {
3827 assert(NumElts == 8);
3828 SDValue LowLHS = DAG.getNode(Opcode: LHS->getOpcode(), DL, VT: MVT::v8i16, Operand: ExtendInLHS);
3829 SDValue LowRHS = DAG.getNode(Opcode: RHS->getOpcode(), DL, VT: MVT::v8i16, Operand: ExtendInRHS);
3830 SDValue MulLow = DAG.getNode(Opcode: ISD::MUL, DL, VT: MVT::v8i16, N1: LowLHS, N2: LowRHS);
3831 SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
3832 SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);
3833 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
3834 }
3835 return SDValue();
3836}
3837
3838static SDValue performMulCombine(SDNode *N,
3839 TargetLowering::DAGCombinerInfo &DCI) {
3840 assert(N->getOpcode() == ISD::MUL);
3841 EVT VT = N->getValueType(ResNo: 0);
3842 if (!VT.isVector())
3843 return SDValue();
3844
3845 if (auto Res = TryWideExtMulCombine(N, DAG&: DCI.DAG))
3846 return Res;
3847
3848 // We don't natively support v16i8 or v8i8 mul, but we do support v8i16. So,
3849 // extend them to v8i16.
3850 if (VT != MVT::v8i8 && VT != MVT::v16i8)
3851 return SDValue();
3852
3853 SDLoc DL(N);
3854 SelectionDAG &DAG = DCI.DAG;
3855 SDValue LHS = N->getOperand(Num: 0);
3856 SDValue RHS = N->getOperand(Num: 1);
3857 EVT MulVT = MVT::v8i16;
3858
3859 if (VT == MVT::v8i8) {
3860 SDValue PromotedLHS = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v16i8, N1: LHS,
3861 N2: DAG.getUNDEF(VT: MVT::v8i8));
3862 SDValue PromotedRHS = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v16i8, N1: RHS,
3863 N2: DAG.getUNDEF(VT: MVT::v8i8));
3864 SDValue LowLHS =
3865 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: PromotedLHS);
3866 SDValue LowRHS =
3867 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: PromotedRHS);
3868 SDValue MulLow = DAG.getBitcast(
3869 VT: MVT::v16i8, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: LowLHS, N2: LowRHS));
3870 // Take the low byte of each lane.
3871 SDValue Shuffle = DAG.getVectorShuffle(
3872 VT: MVT::v16i8, dl: DL, N1: MulLow, N2: DAG.getUNDEF(VT: MVT::v16i8),
3873 Mask: {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});
3874 return extractSubVector(Vec: Shuffle, IdxVal: 0, DAG, DL, VectorWidth: 64);
3875 } else {
3876 assert(VT == MVT::v16i8 && "Expected v16i8");
3877 SDValue LowLHS = DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: LHS);
3878 SDValue LowRHS = DAG.getNode(Opcode: WebAssemblyISD::EXTEND_LOW_U, DL, VT: MulVT, Operand: RHS);
3879 SDValue HighLHS =
3880 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_HIGH_U, DL, VT: MulVT, Operand: LHS);
3881 SDValue HighRHS =
3882 DAG.getNode(Opcode: WebAssemblyISD::EXTEND_HIGH_U, DL, VT: MulVT, Operand: RHS);
3883
3884 SDValue MulLow =
3885 DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: LowLHS, N2: LowRHS));
3886 SDValue MulHigh =
3887 DAG.getBitcast(VT, V: DAG.getNode(Opcode: ISD::MUL, DL, VT: MulVT, N1: HighLHS, N2: HighRHS));
3888
3889 // Take the low byte of each lane.
3890 return DAG.getVectorShuffle(
3891 VT, dl: DL, N1: MulLow, N2: MulHigh,
3892 Mask: {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
3893 }
3894}
3895
3896SDValue DoubleVectorWidth(SDValue In, unsigned RequiredNumElems,
3897 SelectionDAG &DAG) {
3898 SDLoc DL(In);
3899 LLVMContext &Ctx = *DAG.getContext();
3900 EVT InVT = In.getValueType();
3901 unsigned NumElems = InVT.getVectorNumElements() * 2;
3902 EVT OutVT = EVT::getVectorVT(Context&: Ctx, VT: InVT.getVectorElementType(), NumElements: NumElems);
3903 SDValue Concat =
3904 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: OutVT, N1: In, N2: DAG.getPOISON(VT: InVT));
3905 if (NumElems < RequiredNumElems) {
3906 return DoubleVectorWidth(In: Concat, RequiredNumElems, DAG);
3907 }
3908 return Concat;
3909}
3910
3911SDValue performConvertFPCombine(SDNode *N, SelectionDAG &DAG) {
3912 EVT OutVT = N->getValueType(ResNo: 0);
3913 if (!OutVT.isVector())
3914 return SDValue();
3915
3916 EVT OutElTy = OutVT.getVectorElementType();
3917 if (OutElTy != MVT::i8 && OutElTy != MVT::i16)
3918 return SDValue();
3919
3920 unsigned NumElems = OutVT.getVectorNumElements();
3921 if (!isPowerOf2_32(Value: NumElems))
3922 return SDValue();
3923
3924 EVT FPVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3925 if (FPVT.getVectorElementType() != MVT::f32)
3926 return SDValue();
3927
3928 SDLoc DL(N);
3929
3930 // First, convert to i32.
3931 LLVMContext &Ctx = *DAG.getContext();
3932 EVT IntVT = EVT::getVectorVT(Context&: Ctx, VT: MVT::i32, NumElements: NumElems);
3933 SDValue ToInt = DAG.getNode(Opcode: N->getOpcode(), DL, VT: IntVT, Operand: N->getOperand(Num: 0));
3934 APInt Mask = APInt::getLowBitsSet(numBits: IntVT.getScalarSizeInBits(),
3935 loBitsSet: OutVT.getScalarSizeInBits());
3936 // Mask out the top MSBs.
3937 SDValue Masked =
3938 DAG.getNode(Opcode: ISD::AND, DL, VT: IntVT, N1: ToInt, N2: DAG.getConstant(Val: Mask, DL, VT: IntVT));
3939
3940 if (OutVT.getSizeInBits() < 128) {
3941 // Create a wide enough vector that we can use narrow.
3942 EVT NarrowedVT = OutElTy == MVT::i8 ? MVT::v16i8 : MVT::v8i16;
3943 unsigned NumRequiredElems = NarrowedVT.getVectorNumElements();
3944 SDValue WideVector = DoubleVectorWidth(In: Masked, RequiredNumElems: NumRequiredElems, DAG);
3945 SDValue Trunc = truncateVectorWithNARROW(DstVT: NarrowedVT, In: WideVector, DL, DAG);
3946 return DAG.getBitcast(
3947 VT: OutVT, V: extractSubVector(Vec: Trunc, IdxVal: 0, DAG, DL, VectorWidth: OutVT.getSizeInBits()));
3948 } else {
3949 return truncateVectorWithNARROW(DstVT: OutVT, In: Masked, DL, DAG);
3950 }
3951 return SDValue();
3952}
3953
3954// Wide vector shift operations such as v8i32 with sign-extended
3955// operands cause Type Legalizer crashes because the target-specific
3956// extension nodes cannot be directly mapped to the 256-bit size.
3957//
3958// To resolve the crash and optimize performance, we intercept the
3959// illegal v8i32 shift in DAGCombine. We convert the shift amounts
3960// into multipliers and manually split the vector into two v4i32 halves.
3961//
3962// Before: t1: v8i32 = shl (sign_extend v8i16), const_vec
3963// After : t2: v4i32 = mul (ext_low_s v8i16), (ext_low_s narrow_vec)
3964// t3: v4i32 = mul (ext_high_s v8i16), (ext_high_s narrow_vec)
3965// t4: v8i32 = concat_vectors t2, t3
3966static SDValue performShiftCombine(SDNode *N,
3967 TargetLowering::DAGCombinerInfo &DCI) {
3968 SelectionDAG &DAG = DCI.DAG;
3969 assert(N->getOpcode() == ISD::SHL);
3970 EVT VT = N->getValueType(ResNo: 0);
3971 if (VT != MVT::v8i32)
3972 return SDValue();
3973
3974 SDValue LHS = N->getOperand(Num: 0);
3975 SDValue RHS = N->getOperand(Num: 1);
3976 unsigned ExtOpc = LHS.getOpcode();
3977 if (ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND)
3978 return SDValue();
3979
3980 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
3981 return SDValue();
3982
3983 SDLoc DL(N);
3984 SDValue ExtendIn = LHS.getOperand(i: 0);
3985 EVT FromVT = ExtendIn.getValueType();
3986 if (FromVT != MVT::v8i16)
3987 return SDValue();
3988
3989 unsigned NumElts = VT.getVectorNumElements();
3990 unsigned BitWidth = FromVT.getScalarSizeInBits();
3991 bool IsSigned = (ExtOpc == ISD::SIGN_EXTEND);
3992 unsigned MaxValidShift = IsSigned ? (BitWidth - 1) : BitWidth;
3993 SmallVector<SDValue, 16> MulConsts;
3994 for (unsigned I = 0; I < NumElts; ++I) {
3995 auto *C = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i: I));
3996 if (!C)
3997 return SDValue();
3998
3999 const APInt &ShiftAmt = C->getAPIntValue();
4000 if (ShiftAmt.uge(RHS: MaxValidShift))
4001 return SDValue();
4002
4003 APInt MulAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt.getZExtValue());
4004 MulConsts.push_back(Elt: DAG.getConstant(Val: MulAmt, DL, VT: FromVT.getScalarType(),
4005 /*isTarget=*/false, /*isOpaque=*/true));
4006 }
4007
4008 SDValue NarrowConst = DAG.getBuildVector(VT: FromVT, DL, Ops: MulConsts);
4009 unsigned ExtLowOpc =
4010 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
4011 unsigned ExtHighOpc =
4012 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
4013
4014 EVT HalfVT = MVT::v4i32;
4015 SDValue LHSLo = DAG.getNode(Opcode: ExtLowOpc, DL, VT: HalfVT, Operand: ExtendIn);
4016 SDValue LHSHi = DAG.getNode(Opcode: ExtHighOpc, DL, VT: HalfVT, Operand: ExtendIn);
4017 SDValue RHSLo = DAG.getNode(Opcode: ExtLowOpc, DL, VT: HalfVT, Operand: NarrowConst);
4018 SDValue RHSHi = DAG.getNode(Opcode: ExtHighOpc, DL, VT: HalfVT, Operand: NarrowConst);
4019 SDValue MulLo = DAG.getNode(Opcode: ISD::MUL, DL, VT: HalfVT, N1: LHSLo, N2: RHSLo);
4020 SDValue MulHi = DAG.getNode(Opcode: ISD::MUL, DL, VT: HalfVT, N1: LHSHi, N2: RHSHi);
4021 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: MulLo, N2: MulHi);
4022}
4023
4024static SDValue performMinMaxF128Combine(SDNode *N, SelectionDAG &DAG) {
4025 if (N->getValueType(ResNo: 0) != MVT::f128)
4026 return SDValue();
4027
4028 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4029 switch (N->getOpcode()) {
4030 // wasi-libc and emscripten do not currently define fminimuml and fmaximuml.
4031 case ISD::FMINIMUM:
4032 case ISD::FMAXIMUM:
4033 return TLI.expandFMINIMUM_FMAXIMUM(N, DAG);
4034
4035 // wasi-libc and emscripten do not currently define fminimum_numl and
4036 // fmaximum_numl.
4037 case ISD::FMINIMUMNUM:
4038 case ISD::FMAXIMUMNUM:
4039 return TLI.expandFMINIMUMNUM_FMAXIMUMNUM(N, DAG);
4040
4041 default:
4042 return SDValue();
4043 }
4044}
4045
4046SDValue
4047WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
4048 DAGCombinerInfo &DCI) const {
4049 switch (N->getOpcode()) {
4050 default:
4051 return SDValue();
4052 case ISD::BITCAST:
4053 return performBitcastCombine(N, DCI);
4054 case ISD::SETCC:
4055 return performSETCCCombine(N, DCI, Subtarget);
4056 case ISD::VECTOR_SHUFFLE:
4057 return performVECTOR_SHUFFLECombine(N, DCI);
4058 case ISD::SIGN_EXTEND:
4059 case ISD::ZERO_EXTEND:
4060 return performVectorExtendCombine(N, DCI);
4061 case ISD::UINT_TO_FP:
4062 if (auto ExtCombine = performVectorExtendToFPCombine(N, DCI, Subtarget))
4063 return ExtCombine;
4064 return performVectorNonNegToFPCombine(N, DCI);
4065 case ISD::SINT_TO_FP:
4066 return performVectorExtendToFPCombine(N, DCI, Subtarget);
4067 case ISD::FP_TO_SINT_SAT:
4068 case ISD::FP_TO_UINT_SAT:
4069 case ISD::FP_ROUND:
4070 case ISD::CONCAT_VECTORS:
4071 return performVectorTruncZeroCombine(N, DCI);
4072 case ISD::FP_TO_SINT:
4073 case ISD::FP_TO_UINT:
4074 return performConvertFPCombine(N, DAG&: DCI.DAG);
4075 case ISD::TRUNCATE:
4076 return performTruncateCombine(N, DCI);
4077 case ISD::INTRINSIC_WO_CHAIN: {
4078 if (SDValue V = performBitmaskCombine(N, DAG&: DCI.DAG))
4079 return V;
4080 return performAnyAllCombine(N, DAG&: DCI.DAG);
4081 }
4082 case ISD::MUL:
4083 return performMulCombine(N, DCI);
4084 case ISD::SHL:
4085 return performShiftCombine(N, DCI);
4086 case ISD::FMINIMUM:
4087 case ISD::FMAXIMUM:
4088 case ISD::FMINIMUMNUM:
4089 case ISD::FMAXIMUMNUM:
4090 return performMinMaxF128Combine(N, DAG&: DCI.DAG);
4091 }
4092}
4093