1//- WebAssemblyISelDAGToDAG.cpp - A dag to dag inst selector for WebAssembly -//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines an instruction selector for the WebAssembly target.
11///
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
15#include "WebAssembly.h"
16#include "WebAssemblyISelLowering.h"
17#include "WebAssemblyTargetMachine.h"
18#include "WebAssemblyUtilities.h"
19#include "llvm/BinaryFormat/Wasm.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/CodeGen/WasmEHInfo.h"
23#include "llvm/IR/DiagnosticInfo.h"
24#include "llvm/IR/Function.h" // To access function attributes.
25#include "llvm/IR/IntrinsicsWebAssembly.h"
26#include "llvm/MC/MCSymbolWasm.h"
27#include "llvm/Support/CodeGen.h"
28#include "llvm/Support/Debug.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Support/raw_ostream.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "wasm-isel"
35#define PASS_NAME "WebAssembly Instruction Selection"
36
37//===--------------------------------------------------------------------===//
38/// WebAssembly-specific code to select WebAssembly machine instructions for
39/// SelectionDAG operations.
40///
41namespace {
42class WebAssemblyDAGToDAGISel final : public SelectionDAGISel {
43 /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
44 /// right decision when generating code for different targets.
45 const WebAssemblySubtarget *Subtarget;
46
47public:
48 WebAssemblyDAGToDAGISel() = delete;
49
50 WebAssemblyDAGToDAGISel(WebAssemblyTargetMachine &TM,
51 CodeGenOptLevel OptLevel)
52 : SelectionDAGISel(TM, OptLevel), Subtarget(nullptr) {}
53
54 bool runOnMachineFunction(MachineFunction &MF) override {
55 LLVM_DEBUG(dbgs() << "********** ISelDAGToDAG **********\n"
56 "********** Function: "
57 << MF.getName() << '\n');
58
59 Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
60
61 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
62 }
63
64 void PreprocessISelDAG() override;
65
66 void Select(SDNode *Node) override;
67
68 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
69 InlineAsm::ConstraintCode ConstraintID,
70 std::vector<SDValue> &OutOps) override;
71
72 bool SelectAddrOperands32(SDValue Op, SDValue &Offset, SDValue &Addr);
73 bool SelectAddrOperands64(SDValue Op, SDValue &Offset, SDValue &Addr);
74 bool SelectAtomicAddrOperands(SDNode *Op, SDValue N, SDValue &Offset,
75 SDValue &Addr, SDValue &Order, bool Is64);
76 bool SelectAtomicAddrOperands32(SDNode *Op, SDValue N, SDValue &Offset,
77 SDValue &Addr, SDValue &Order);
78 bool SelectAtomicAddrOperands64(SDNode *Op, SDValue N, SDValue &Offset,
79 SDValue &Addr, SDValue &Order);
80
81// Include the pieces autogenerated from the target description.
82#include "WebAssemblyGenDAGISel.inc"
83
84private:
85 // add select functions here...
86
87 bool SelectAddrOperands(MVT AddrType, unsigned ConstOpc, SDValue Op,
88 SDValue &Offset, SDValue &Addr);
89 bool SelectAddrAddOperands(MVT OffsetType, SDValue N, SDValue &Offset,
90 SDValue &Addr);
91};
92
93class WebAssemblyDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
94public:
95 static char ID;
96 explicit WebAssemblyDAGToDAGISelLegacy(WebAssemblyTargetMachine &TM,
97 CodeGenOptLevel OptLevel)
98 : SelectionDAGISelLegacy(
99 ID, std::make_unique<WebAssemblyDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
100};
101} // end anonymous namespace
102
103char WebAssemblyDAGToDAGISelLegacy::ID;
104
105INITIALIZE_PASS(WebAssemblyDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false,
106 false)
107
108void WebAssemblyDAGToDAGISel::PreprocessISelDAG() {
109 // Stack objects that should be allocated to locals are hoisted to WebAssembly
110 // locals when they are first used. However for those without uses, we hoist
111 // them here. It would be nice if there were some hook to do this when they
112 // are added to the MachineFrameInfo, but that's not the case right now.
113 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
114 for (int Idx = 0; Idx < FrameInfo.getObjectIndexEnd(); Idx++)
115 WebAssemblyFrameLowering::getLocalForStackObject(MF&: *MF, FrameIndex: Idx);
116
117 SelectionDAGISel::PreprocessISelDAG();
118}
119
120static SDValue getTagSymNode(int Tag, SelectionDAG *DAG) {
121 assert(Tag == WebAssembly::CPP_EXCEPTION || Tag == WebAssembly::C_LONGJMP);
122 auto &MF = DAG->getMachineFunction();
123 const auto &TLI = DAG->getTargetLoweringInfo();
124 MVT PtrVT = TLI.getPointerTy(DL: DAG->getDataLayout());
125 const char *SymName = Tag == WebAssembly::CPP_EXCEPTION
126 ? MF.createExternalSymbolName(Name: "__cpp_exception")
127 : MF.createExternalSymbolName(Name: "__c_longjmp");
128 return DAG->getTargetExternalSymbol(Sym: SymName, VT: PtrVT);
129}
130
131static APInt encodeFunctionSignature(SelectionDAG *DAG, SDLoc &DL,
132 SmallVector<MVT, 4> &Returns,
133 SmallVector<MVT, 4> &Params) {
134 auto toWasmValType = [](MVT VT) {
135 if (VT == MVT::i32) {
136 return wasm::ValType::I32;
137 }
138 if (VT == MVT::i64) {
139 return wasm::ValType::I64;
140 }
141 if (VT == MVT::f32) {
142 return wasm::ValType::F32;
143 }
144 if (VT == MVT::f64) {
145 return wasm::ValType::F64;
146 }
147 if (VT == MVT::externref) {
148 return wasm::ValType::EXTERNREF;
149 }
150 if (VT == MVT::funcref) {
151 return wasm::ValType::FUNCREF;
152 }
153 if (VT == MVT::exnref) {
154 return wasm::ValType::EXNREF;
155 }
156 LLVM_DEBUG(errs() << "Unhandled type for llvm.wasm.ref.test.func: " << VT
157 << "\n");
158 llvm_unreachable("Unhandled type for llvm.wasm.ref.test.func");
159 };
160 auto NParams = Params.size();
161 auto NReturns = Returns.size();
162 auto BitWidth = (NParams + NReturns + 2) * 64;
163 auto Sig = APInt(BitWidth, 0);
164
165 // Annoying special case: if getSignificantBits() <= 64 then InstrEmitter will
166 // emit an Imm instead of a CImm. It simplifies WebAssemblyMCInstLower if we
167 // always emit a CImm. So xor NParams with 0x7ffffff to ensure
168 // getSignificantBits() > 64
169 Sig |= NReturns ^ 0x7ffffff;
170 for (auto &Return : Returns) {
171 auto V = toWasmValType(Return);
172 Sig <<= 64;
173 Sig |= (int64_t)V;
174 }
175 Sig <<= 64;
176 Sig |= NParams;
177 for (auto &Param : Params) {
178 auto V = toWasmValType(Param);
179 Sig <<= 64;
180 Sig |= (int64_t)V;
181 }
182 return Sig;
183}
184
185static unsigned getWebAssemblyMemoryOrder(AtomicOrdering Ordering) {
186 unsigned OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
187 switch (Ordering) {
188 case AtomicOrdering::Unordered:
189 case AtomicOrdering::Monotonic:
190 case AtomicOrdering::Acquire:
191 case AtomicOrdering::Release:
192 case AtomicOrdering::AcquireRelease:
193 OrderVal = wasm::WASM_MEM_ORDER_ACQ_REL;
194 break;
195 case AtomicOrdering::SequentiallyConsistent:
196 OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
197 break;
198 default:
199 llvm_unreachable("Invalid atomic ordering");
200 }
201 return OrderVal;
202}
203
204void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
205 // If we have a custom node, we already have selected!
206 if (Node->isMachineOpcode()) {
207 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
208 Node->setNodeId(-1);
209 return;
210 }
211
212 MVT PtrVT = TLI->getPointerTy(DL: CurDAG->getDataLayout());
213 auto GlobalGetIns = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
214 : WebAssembly::GLOBAL_GET_I32;
215
216 SDLoc DL(Node);
217 MachineFunction &MF = CurDAG->getMachineFunction();
218 switch (Node->getOpcode()) {
219 case ISD::ATOMIC_FENCE: {
220 if (!MF.getSubtarget<WebAssemblySubtarget>().hasAtomics())
221 break;
222
223 uint64_t SyncScopeID = Node->getConstantOperandVal(Num: 2);
224 MachineSDNode *Fence = nullptr;
225 switch (SyncScopeID) {
226 case SyncScope::SingleThread:
227 // We lower a single-thread fence to a pseudo compiler barrier instruction
228 // preventing instruction reordering. This will not be emitted in final
229 // binary.
230 Fence = CurDAG->getMachineNode(Opcode: WebAssembly::COMPILER_FENCE,
231 dl: DL, // debug loc
232 VT: MVT::Other, // outchain type
233 Op1: Node->getOperand(Num: 0) // inchain
234 );
235 break;
236 case SyncScope::System: {
237 unsigned Order = wasm::WASM_MEM_ORDER_SEQ_CST;
238 if (MF.getSubtarget<WebAssemblySubtarget>().hasRelaxedAtomics()) {
239 auto Ordering =
240 static_cast<AtomicOrdering>(Node->getConstantOperandVal(Num: 1));
241 Order = getWebAssemblyMemoryOrder(Ordering);
242 }
243 Fence = CurDAG->getMachineNode(
244 Opcode: WebAssembly::ATOMIC_FENCE,
245 dl: DL, // debug loc
246 VT: MVT::Other, // outchain type
247 Op1: CurDAG->getTargetConstant(Val: Order, DL, VT: MVT::i32), // order
248 Op2: Node->getOperand(Num: 0) // inchain
249 );
250 break;
251 }
252 default:
253 llvm_unreachable("Unknown scope!");
254 }
255
256 ReplaceNode(F: Node, T: Fence);
257 CurDAG->RemoveDeadNode(N: Node);
258 return;
259 }
260
261 case ISD::INTRINSIC_WO_CHAIN: {
262 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
263 switch (IntNo) {
264 case Intrinsic::wasm_tls_size: {
265 MachineSDNode *TLSSize = CurDAG->getMachineNode(
266 Opcode: GlobalGetIns, dl: DL, VT: PtrVT,
267 Op1: CurDAG->getTargetExternalSymbol(Sym: "__tls_size", VT: PtrVT));
268 ReplaceNode(F: Node, T: TLSSize);
269 return;
270 }
271
272 case Intrinsic::wasm_tls_align: {
273 MachineSDNode *TLSAlign = CurDAG->getMachineNode(
274 Opcode: GlobalGetIns, dl: DL, VT: PtrVT,
275 Op1: CurDAG->getTargetExternalSymbol(Sym: "__tls_align", VT: PtrVT));
276 ReplaceNode(F: Node, T: TLSAlign);
277 return;
278 }
279 case Intrinsic::wasm_ptr_to_funcref: {
280 // Convert a function pointer to a funcref by reading the corresponding
281 // entry from the __indirect_function_table.
282 MachineFunction &MF = CurDAG->getMachineFunction();
283 auto PtrVT = MVT::getIntegerVT(BitWidth: MF.getDataLayout().getPointerSizeInBits());
284 MCSymbol *Table = WebAssembly::getOrCreateFunctionTableSymbol(
285 Ctx&: MF.getContext(), Subtarget);
286 SDValue TableSym = CurDAG->getMCSymbol(Sym: Table, VT: PtrVT);
287 SDValue FuncPtr = Node->getOperand(Num: 1);
288 if (Subtarget->hasAddr64() && FuncPtr.getValueType() == MVT::i64) {
289 // table.get expects an i32 but on 64 bit platforms the function pointer
290 // is an i64. In that case, i32.wrap_i64 to convert.
291 FuncPtr = SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::I32_WRAP_I64, dl: DL,
292 VT: MVT::i32, Op1: FuncPtr),
293 0);
294 }
295 MachineSDNode *FuncRef = CurDAG->getMachineNode(
296 Opcode: WebAssembly::TABLE_GET_FUNCREF, dl: DL, VT: MVT::funcref, Op1: TableSym, Op2: FuncPtr);
297 ReplaceNode(F: Node, T: FuncRef);
298 return;
299 }
300 case Intrinsic::wasm_ref_test_func: {
301 // First emit the TABLE_GET instruction to convert function pointer ==>
302 // funcref
303 MachineFunction &MF = CurDAG->getMachineFunction();
304 auto PtrVT = MVT::getIntegerVT(BitWidth: MF.getDataLayout().getPointerSizeInBits());
305 MCSymbol *Table = WebAssembly::getOrCreateFunctionTableSymbol(
306 Ctx&: MF.getContext(), Subtarget);
307 SDValue TableSym = CurDAG->getMCSymbol(Sym: Table, VT: PtrVT);
308 SDValue FuncPtr = Node->getOperand(Num: 1);
309 if (Subtarget->hasAddr64() && FuncPtr.getValueType() == MVT::i64) {
310 // table.get expects an i32 but on 64 bit platforms the function pointer
311 // is an i64. In that case, i32.wrap_i64 to convert.
312 FuncPtr = SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::I32_WRAP_I64, dl: DL,
313 VT: MVT::i32, Op1: FuncPtr),
314 0);
315 }
316 SDValue FuncRef =
317 SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::TABLE_GET_FUNCREF, dl: DL,
318 VT: MVT::funcref, Op1: TableSym, Op2: FuncPtr),
319 0);
320
321 // Encode the signature information into the type index placeholder.
322 // This gets decoded and converted into the actual type signature in
323 // WebAssemblyMCInstLower.cpp.
324 SmallVector<MVT, 4> Params;
325 SmallVector<MVT, 4> Returns;
326
327 bool IsParam = false;
328 // Operand 0 is the return register, Operand 1 is the function pointer.
329 // The remaining operands encode the type of the function we are testing
330 // for.
331 for (unsigned I = 2, E = Node->getNumOperands(); I < E; ++I) {
332 MVT VT = Node->getOperand(Num: I).getValueType().getSimpleVT();
333 if (VT == MVT::Untyped) {
334 IsParam = true;
335 continue;
336 }
337 if (IsParam) {
338 Params.push_back(Elt: VT);
339 } else {
340 Returns.push_back(Elt: VT);
341 }
342 }
343 auto Sig = encodeFunctionSignature(DAG: CurDAG, DL, Returns, Params);
344
345 auto SigOp = CurDAG->getTargetConstant(
346 Val: Sig, DL, VT: EVT::getIntegerVT(Context&: *CurDAG->getContext(), BitWidth: Sig.getBitWidth()));
347 MachineSDNode *RefTestNode = CurDAG->getMachineNode(
348 Opcode: WebAssembly::REF_TEST_FUNCREF, dl: DL, VT: MVT::i32, Ops: {SigOp, FuncRef});
349 ReplaceNode(F: Node, T: RefTestNode);
350 return;
351 }
352 }
353 break;
354 }
355
356 case ISD::INTRINSIC_W_CHAIN: {
357 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
358 const auto &TLI = CurDAG->getTargetLoweringInfo();
359 MVT PtrVT = TLI.getPointerTy(DL: CurDAG->getDataLayout());
360 switch (IntNo) {
361 case Intrinsic::wasm_tls_base: {
362 MachineSDNode *TLSBase = llvm::WebAssembly::getTLSBase(
363 DAG&: *CurDAG, DL, Subtarget, Chain: Node->getOperand(Num: 0));
364 ReplaceNode(F: Node, T: TLSBase);
365 return;
366 }
367
368 case Intrinsic::wasm_catch: {
369 int Tag = Node->getConstantOperandVal(Num: 2);
370 SDValue SymNode = getTagSymNode(Tag, DAG: CurDAG);
371 unsigned CatchOpcode = WebAssembly::WasmUseLegacyEH
372 ? WebAssembly::CATCH_LEGACY
373 : WebAssembly::CATCH;
374 MachineSDNode *Catch =
375 CurDAG->getMachineNode(Opcode: CatchOpcode, dl: DL,
376 ResultTys: {
377 PtrVT, // exception pointer
378 MVT::Other // outchain type
379 },
380 Ops: {
381 SymNode, // exception symbol
382 Node->getOperand(Num: 0) // inchain
383 });
384 ReplaceNode(F: Node, T: Catch);
385 return;
386 }
387 }
388 break;
389 }
390
391 case ISD::INTRINSIC_VOID: {
392 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
393 switch (IntNo) {
394 case Intrinsic::wasm_throw: {
395 int Tag = Node->getConstantOperandVal(Num: 2);
396 SDValue SymNode = getTagSymNode(Tag, DAG: CurDAG);
397 MachineSDNode *Throw =
398 CurDAG->getMachineNode(Opcode: WebAssembly::THROW, dl: DL,
399 VT: MVT::Other, // outchain type
400 Ops: {
401 SymNode, // exception symbol
402 Node->getOperand(Num: 3), // thrown value
403 Node->getOperand(Num: 0) // inchain
404 });
405 ReplaceNode(F: Node, T: Throw);
406 return;
407 }
408 case Intrinsic::wasm_rethrow: {
409 // RETHROW's BB argument will be populated in LateEHPrepare. Just use a
410 // '0' as a placeholder for now.
411 MachineSDNode *Rethrow = CurDAG->getMachineNode(
412 Opcode: WebAssembly::RETHROW, dl: DL,
413 VT: MVT::Other, // outchain type
414 Ops: {
415 CurDAG->getConstant(Val: 0, DL, VT: MVT::i32), // placeholder
416 Node->getOperand(Num: 0) // inchain
417 });
418 ReplaceNode(F: Node, T: Rethrow);
419 return;
420 }
421 }
422 break;
423 }
424
425 case WebAssemblyISD::CALL:
426 case WebAssemblyISD::RET_CALL: {
427 // CALL has both variable operands and variable results, but ISel only
428 // supports one or the other. Split calls into two nodes glued together, one
429 // for the operands and one for the results. These two nodes will be
430 // recombined in a custom inserter hook into a single MachineInstr.
431 SmallVector<SDValue, 16> Ops;
432 for (size_t i = 1; i < Node->getNumOperands(); ++i) {
433 SDValue Op = Node->getOperand(Num: i);
434 // Remove the wrapper when the call target is a function, an external
435 // symbol (which will be lowered to a library function), or an alias of
436 // a function. If the target is not a function/external symbol, we
437 // shouldn't remove the wrapper, because we cannot call it directly and
438 // instead we want it to be loaded with a CONST instruction and called
439 // with a call_indirect later.
440 if (i == 1 && Op->getOpcode() == WebAssemblyISD::Wrapper) {
441 SDValue NewOp = Op->getOperand(Num: 0);
442 if (auto *GlobalOp = dyn_cast<GlobalAddressSDNode>(Val: NewOp.getNode())) {
443 if (isa<Function>(
444 Val: GlobalOp->getGlobal()->stripPointerCastsAndAliases()))
445 Op = NewOp;
446 } else if (isa<ExternalSymbolSDNode>(Val: NewOp.getNode())) {
447 Op = NewOp;
448 }
449 }
450 Ops.push_back(Elt: Op);
451 }
452
453 // Add the chain last
454 Ops.push_back(Elt: Node->getOperand(Num: 0));
455 MachineSDNode *CallParams =
456 CurDAG->getMachineNode(Opcode: WebAssembly::CALL_PARAMS, dl: DL, VT: MVT::Glue, Ops);
457
458 unsigned Results = Node->getOpcode() == WebAssemblyISD::CALL
459 ? WebAssembly::CALL_RESULTS
460 : WebAssembly::RET_CALL_RESULTS;
461
462 SDValue Link(CallParams, 0);
463 MachineSDNode *CallResults =
464 CurDAG->getMachineNode(Opcode: Results, dl: DL, VTs: Node->getVTList(), Ops: Link);
465 ReplaceNode(F: Node, T: CallResults);
466 return;
467 }
468
469 default:
470 break;
471 }
472
473 // Select the default instruction.
474 SelectCode(N: Node);
475}
476
477bool WebAssemblyDAGToDAGISel::SelectInlineAsmMemoryOperand(
478 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
479 std::vector<SDValue> &OutOps) {
480 switch (ConstraintID) {
481 case InlineAsm::ConstraintCode::m:
482 // We just support simple memory operands that just have a single address
483 // operand and need no special handling.
484 OutOps.push_back(x: Op);
485 return false;
486 default:
487 break;
488 }
489
490 return true;
491}
492
493bool WebAssemblyDAGToDAGISel::SelectAddrAddOperands(MVT OffsetType, SDValue N,
494 SDValue &Offset,
495 SDValue &Addr) {
496 assert(N.getNumOperands() == 2 && "Attempting to fold in a non-binary op");
497
498 // WebAssembly constant offsets are performed as unsigned with infinite
499 // precision, so we need to check for NoUnsignedWrap so that we don't fold an
500 // offset for an add that needs wrapping.
501 if (N.getOpcode() == ISD::ADD && !N.getNode()->getFlags().hasNoUnsignedWrap())
502 return false;
503
504 for (size_t i = 0; i < 2; ++i) {
505 SDValue Op = N.getOperand(i);
506 SDValue OtherOp = N.getOperand(i: i == 0 ? 1 : 0);
507
508 // Folds constants in an add into the offset.
509 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op)) {
510 Offset =
511 CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL: SDLoc(N), VT: OffsetType);
512 Addr = OtherOp;
513 return true;
514 }
515
516 // Fold target global addresses into the offset.
517 if (!TM.isPositionIndependent()) {
518 if (Op.getOpcode() == WebAssemblyISD::Wrapper)
519 Op = Op.getOperand(i: 0);
520
521 if (Op.getOpcode() == ISD::TargetGlobalAddress) {
522 Addr = OtherOp;
523 Offset = Op;
524 return true;
525 }
526 }
527 }
528 return false;
529}
530
531bool WebAssemblyDAGToDAGISel::SelectAddrOperands(MVT AddrType,
532 unsigned ConstOpc, SDValue N,
533 SDValue &Offset,
534 SDValue &Addr) {
535 SDLoc DL(N);
536
537 // Fold target global addresses into the offset.
538 if (!TM.isPositionIndependent()) {
539 SDValue Op(N);
540 if (Op.getOpcode() == WebAssemblyISD::Wrapper)
541 Op = Op.getOperand(i: 0);
542
543 if (Op.getOpcode() == ISD::TargetGlobalAddress) {
544 Offset = Op;
545 Addr = SDValue(
546 CurDAG->getMachineNode(Opcode: ConstOpc, dl: DL, VT: AddrType,
547 Op1: CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType)),
548 0);
549 return true;
550 }
551 }
552
553 // Fold anything inside an add into the offset.
554 if (N.getOpcode() == ISD::ADD &&
555 SelectAddrAddOperands(OffsetType: AddrType, N, Offset, Addr))
556 return true;
557
558 // Likewise, treat an 'or' node as an 'add' if the or'ed bits are known to be
559 // zero and fold them into the offset too.
560 if (N.getOpcode() == ISD::OR) {
561 bool OrIsAdd;
562 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
563 OrIsAdd =
564 CurDAG->MaskedValueIsZero(Op: N->getOperand(Num: 0), Mask: CN->getAPIntValue());
565 } else {
566 KnownBits Known0 = CurDAG->computeKnownBits(Op: N->getOperand(Num: 0), Depth: 0);
567 KnownBits Known1 = CurDAG->computeKnownBits(Op: N->getOperand(Num: 1), Depth: 0);
568 OrIsAdd = (~Known0.Zero & ~Known1.Zero) == 0;
569 }
570
571 if (OrIsAdd && SelectAddrAddOperands(OffsetType: AddrType, N, Offset, Addr))
572 return true;
573 }
574
575 // Fold constant addresses into the offset.
576 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
577 Offset = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: AddrType);
578 Addr = SDValue(
579 CurDAG->getMachineNode(Opcode: ConstOpc, dl: DL, VT: AddrType,
580 Op1: CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType)),
581 0);
582 return true;
583 }
584
585 // Else it's a plain old load/store with no offset.
586 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType);
587 Addr = N;
588 return true;
589}
590
591bool WebAssemblyDAGToDAGISel::SelectAddrOperands32(SDValue Op, SDValue &Offset,
592 SDValue &Addr) {
593 return SelectAddrOperands(AddrType: MVT::i32, ConstOpc: WebAssembly::CONST_I32, N: Op, Offset, Addr);
594}
595
596bool WebAssemblyDAGToDAGISel::SelectAddrOperands64(SDValue Op, SDValue &Offset,
597 SDValue &Addr) {
598 return SelectAddrOperands(AddrType: MVT::i64, ConstOpc: WebAssembly::CONST_I64, N: Op, Offset, Addr);
599}
600
601static MemSDNode *findMemSDNode(SDNode *N) {
602 while (N) {
603 if (auto *MemNode = dyn_cast<MemSDNode>(Val: N))
604 return MemNode;
605 switch (N->getOpcode()) {
606 case ISD::ZERO_EXTEND:
607 case ISD::SIGN_EXTEND:
608 case ISD::ANY_EXTEND:
609 case ISD::SIGN_EXTEND_INREG:
610 case ISD::AssertZext:
611 case ISD::AssertSext:
612 case ISD::TRUNCATE:
613 case ISD::BITCAST:
614 case ISD::AND:
615 N = N->getOperand(Num: 0).getNode();
616 break;
617 default:
618 return nullptr;
619 }
620 }
621 return nullptr;
622}
623
624bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands(SDNode *Op, SDValue N,
625 SDValue &Offset,
626 SDValue &Addr,
627 SDValue &Order,
628 bool Is64) {
629 auto *MemNode = findMemSDNode(N: Op);
630 if (!MemNode)
631 return false;
632
633 bool Match = Is64 ? SelectAddrOperands64(Op: N, Offset, Addr)
634 : SelectAddrOperands32(Op: N, Offset, Addr);
635 if (!Match)
636 return false;
637
638 auto Ordering = MemNode->getMergedOrdering();
639 unsigned OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
640 if (Subtarget->hasRelaxedAtomics())
641 OrderVal = getWebAssemblyMemoryOrder(Ordering);
642 Order = CurDAG->getTargetConstant(Val: OrderVal, DL: SDLoc(Op), VT: MVT::i32);
643 return true;
644}
645
646bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands32(SDNode *Op, SDValue N,
647 SDValue &Offset,
648 SDValue &Addr,
649 SDValue &Order) {
650 return SelectAtomicAddrOperands(Op, N, Offset, Addr, Order, /*Is64=*/false);
651}
652
653bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands64(SDNode *Op, SDValue N,
654 SDValue &Offset,
655 SDValue &Addr,
656 SDValue &Order) {
657 return SelectAtomicAddrOperands(Op, N, Offset, Addr, Order, /*Is64=*/true);
658}
659
660/// This pass converts a legalized DAG into a WebAssembly-specific DAG, ready
661/// for instruction scheduling.
662WebAssemblyISelDAGToDAGPass::WebAssemblyISelDAGToDAGPass(
663 WebAssemblyTargetMachine &TM, CodeGenOptLevel OptLevel)
664 : SelectionDAGISelPass(
665 std::make_unique<WebAssemblyDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
666
667FunctionPass *
668llvm::createWebAssemblyISelDagLegacyPass(WebAssemblyTargetMachine &TM,
669 CodeGenOptLevel OptLevel) {
670 return new WebAssemblyDAGToDAGISelLegacy(TM, OptLevel);
671}
672