1//- WebAssemblyISelDAGToDAG.cpp - A dag to dag inst selector for WebAssembly -//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file defines an instruction selector for the WebAssembly target.
11///
12//===----------------------------------------------------------------------===//
13
14#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
15#include "WebAssembly.h"
16#include "WebAssemblyISelLowering.h"
17#include "WebAssemblyTargetMachine.h"
18#include "WebAssemblyUtilities.h"
19#include "llvm/BinaryFormat/Wasm.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/CodeGen/WasmEHInfo.h"
23#include "llvm/IR/DiagnosticInfo.h"
24#include "llvm/IR/Function.h" // To access function attributes.
25#include "llvm/IR/IntrinsicsWebAssembly.h"
26#include "llvm/MC/MCSymbolWasm.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Support/raw_ostream.h"
30
31using namespace llvm;
32
33#define DEBUG_TYPE "wasm-isel"
34#define PASS_NAME "WebAssembly Instruction Selection"
35
36//===--------------------------------------------------------------------===//
37/// WebAssembly-specific code to select WebAssembly machine instructions for
38/// SelectionDAG operations.
39///
40namespace {
41class WebAssemblyDAGToDAGISel final : public SelectionDAGISel {
42 /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
43 /// right decision when generating code for different targets.
44 const WebAssemblySubtarget *Subtarget;
45
46public:
47 WebAssemblyDAGToDAGISel() = delete;
48
49 WebAssemblyDAGToDAGISel(WebAssemblyTargetMachine &TM,
50 CodeGenOptLevel OptLevel)
51 : SelectionDAGISel(TM, OptLevel), Subtarget(nullptr) {}
52
53 bool runOnMachineFunction(MachineFunction &MF) override {
54 LLVM_DEBUG(dbgs() << "********** ISelDAGToDAG **********\n"
55 "********** Function: "
56 << MF.getName() << '\n');
57
58 Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
59
60 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
61 }
62
63 void PreprocessISelDAG() override;
64
65 void Select(SDNode *Node) override;
66
67 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
68 InlineAsm::ConstraintCode ConstraintID,
69 std::vector<SDValue> &OutOps) override;
70
71 bool SelectAddrOperands32(SDValue Op, SDValue &Offset, SDValue &Addr);
72 bool SelectAddrOperands64(SDValue Op, SDValue &Offset, SDValue &Addr);
73 bool SelectAtomicAddrOperands(SDNode *Op, SDValue N, SDValue &Offset,
74 SDValue &Addr, SDValue &Order, bool Is64);
75 bool SelectAtomicAddrOperands32(SDNode *Op, SDValue N, SDValue &Offset,
76 SDValue &Addr, SDValue &Order);
77 bool SelectAtomicAddrOperands64(SDNode *Op, SDValue N, SDValue &Offset,
78 SDValue &Addr, SDValue &Order);
79
80// Include the pieces autogenerated from the target description.
81#include "WebAssemblyGenDAGISel.inc"
82
83private:
84 // add select functions here...
85
86 bool SelectAddrOperands(MVT AddrType, unsigned ConstOpc, SDValue Op,
87 SDValue &Offset, SDValue &Addr);
88 bool SelectAddrAddOperands(MVT OffsetType, SDValue N, SDValue &Offset,
89 SDValue &Addr);
90};
91
92class WebAssemblyDAGToDAGISelLegacy : public SelectionDAGISelLegacy {
93public:
94 static char ID;
95 explicit WebAssemblyDAGToDAGISelLegacy(WebAssemblyTargetMachine &TM,
96 CodeGenOptLevel OptLevel)
97 : SelectionDAGISelLegacy(
98 ID, std::make_unique<WebAssemblyDAGToDAGISel>(args&: TM, args&: OptLevel)) {}
99};
100} // end anonymous namespace
101
102char WebAssemblyDAGToDAGISelLegacy::ID;
103
104INITIALIZE_PASS(WebAssemblyDAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false,
105 false)
106
107void WebAssemblyDAGToDAGISel::PreprocessISelDAG() {
108 // Stack objects that should be allocated to locals are hoisted to WebAssembly
109 // locals when they are first used. However for those without uses, we hoist
110 // them here. It would be nice if there were some hook to do this when they
111 // are added to the MachineFrameInfo, but that's not the case right now.
112 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
113 for (int Idx = 0; Idx < FrameInfo.getObjectIndexEnd(); Idx++)
114 WebAssemblyFrameLowering::getLocalForStackObject(MF&: *MF, FrameIndex: Idx);
115
116 SelectionDAGISel::PreprocessISelDAG();
117}
118
119static SDValue getTagSymNode(int Tag, SelectionDAG *DAG) {
120 assert(Tag == WebAssembly::CPP_EXCEPTION || Tag == WebAssembly::C_LONGJMP);
121 auto &MF = DAG->getMachineFunction();
122 const auto &TLI = DAG->getTargetLoweringInfo();
123 MVT PtrVT = TLI.getPointerTy(DL: DAG->getDataLayout());
124 const char *SymName = Tag == WebAssembly::CPP_EXCEPTION
125 ? MF.createExternalSymbolName(Name: "__cpp_exception")
126 : MF.createExternalSymbolName(Name: "__c_longjmp");
127 return DAG->getTargetExternalSymbol(Sym: SymName, VT: PtrVT);
128}
129
130static APInt encodeFunctionSignature(SelectionDAG *DAG, SDLoc &DL,
131 SmallVector<MVT, 4> &Returns,
132 SmallVector<MVT, 4> &Params) {
133 auto toWasmValType = [](MVT VT) {
134 if (VT == MVT::i32) {
135 return wasm::ValType::I32;
136 }
137 if (VT == MVT::i64) {
138 return wasm::ValType::I64;
139 }
140 if (VT == MVT::f32) {
141 return wasm::ValType::F32;
142 }
143 if (VT == MVT::f64) {
144 return wasm::ValType::F64;
145 }
146 if (VT == MVT::externref) {
147 return wasm::ValType::EXTERNREF;
148 }
149 if (VT == MVT::funcref) {
150 return wasm::ValType::FUNCREF;
151 }
152 if (VT == MVT::exnref) {
153 return wasm::ValType::EXNREF;
154 }
155 LLVM_DEBUG(errs() << "Unhandled type for llvm.wasm.ref.test.func: " << VT
156 << "\n");
157 llvm_unreachable("Unhandled type for llvm.wasm.ref.test.func");
158 };
159 auto NParams = Params.size();
160 auto NReturns = Returns.size();
161 auto BitWidth = (NParams + NReturns + 2) * 64;
162 auto Sig = APInt(BitWidth, 0);
163
164 // Annoying special case: if getSignificantBits() <= 64 then InstrEmitter will
165 // emit an Imm instead of a CImm. It simplifies WebAssemblyMCInstLower if we
166 // always emit a CImm. So xor NParams with 0x7ffffff to ensure
167 // getSignificantBits() > 64
168 Sig |= NReturns ^ 0x7ffffff;
169 for (auto &Return : Returns) {
170 auto V = toWasmValType(Return);
171 Sig <<= 64;
172 Sig |= (int64_t)V;
173 }
174 Sig <<= 64;
175 Sig |= NParams;
176 for (auto &Param : Params) {
177 auto V = toWasmValType(Param);
178 Sig <<= 64;
179 Sig |= (int64_t)V;
180 }
181 return Sig;
182}
183
184static unsigned getWebAssemblyMemoryOrder(AtomicOrdering Ordering) {
185 unsigned OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
186 switch (Ordering) {
187 case AtomicOrdering::Unordered:
188 case AtomicOrdering::Monotonic:
189 case AtomicOrdering::Acquire:
190 case AtomicOrdering::Release:
191 case AtomicOrdering::AcquireRelease:
192 OrderVal = wasm::WASM_MEM_ORDER_ACQ_REL;
193 break;
194 case AtomicOrdering::SequentiallyConsistent:
195 OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
196 break;
197 default:
198 llvm_unreachable("Invalid atomic ordering");
199 }
200 return OrderVal;
201}
202
203void WebAssemblyDAGToDAGISel::Select(SDNode *Node) {
204 // If we have a custom node, we already have selected!
205 if (Node->isMachineOpcode()) {
206 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
207 Node->setNodeId(-1);
208 return;
209 }
210
211 MVT PtrVT = TLI->getPointerTy(DL: CurDAG->getDataLayout());
212 auto GlobalGetIns = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
213 : WebAssembly::GLOBAL_GET_I32;
214
215 SDLoc DL(Node);
216 MachineFunction &MF = CurDAG->getMachineFunction();
217 switch (Node->getOpcode()) {
218 case ISD::ATOMIC_FENCE: {
219 if (!MF.getSubtarget<WebAssemblySubtarget>().hasAtomics())
220 break;
221
222 uint64_t SyncScopeID = Node->getConstantOperandVal(Num: 2);
223 MachineSDNode *Fence = nullptr;
224 switch (SyncScopeID) {
225 case SyncScope::SingleThread:
226 // We lower a single-thread fence to a pseudo compiler barrier instruction
227 // preventing instruction reordering. This will not be emitted in final
228 // binary.
229 Fence = CurDAG->getMachineNode(Opcode: WebAssembly::COMPILER_FENCE,
230 dl: DL, // debug loc
231 VT: MVT::Other, // outchain type
232 Op1: Node->getOperand(Num: 0) // inchain
233 );
234 break;
235 case SyncScope::System: {
236 unsigned Order = wasm::WASM_MEM_ORDER_SEQ_CST;
237 if (MF.getSubtarget<WebAssemblySubtarget>().hasRelaxedAtomics()) {
238 auto Ordering =
239 static_cast<AtomicOrdering>(Node->getConstantOperandVal(Num: 1));
240 Order = getWebAssemblyMemoryOrder(Ordering);
241 }
242 Fence = CurDAG->getMachineNode(
243 Opcode: WebAssembly::ATOMIC_FENCE,
244 dl: DL, // debug loc
245 VT: MVT::Other, // outchain type
246 Op1: CurDAG->getTargetConstant(Val: Order, DL, VT: MVT::i32), // order
247 Op2: Node->getOperand(Num: 0) // inchain
248 );
249 break;
250 }
251 default:
252 llvm_unreachable("Unknown scope!");
253 }
254
255 ReplaceNode(F: Node, T: Fence);
256 CurDAG->RemoveDeadNode(N: Node);
257 return;
258 }
259
260 case ISD::INTRINSIC_WO_CHAIN: {
261 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
262 switch (IntNo) {
263 case Intrinsic::wasm_tls_size: {
264 MachineSDNode *TLSSize = CurDAG->getMachineNode(
265 Opcode: GlobalGetIns, dl: DL, VT: PtrVT,
266 Op1: CurDAG->getTargetExternalSymbol(Sym: "__tls_size", VT: PtrVT));
267 ReplaceNode(F: Node, T: TLSSize);
268 return;
269 }
270
271 case Intrinsic::wasm_tls_align: {
272 MachineSDNode *TLSAlign = CurDAG->getMachineNode(
273 Opcode: GlobalGetIns, dl: DL, VT: PtrVT,
274 Op1: CurDAG->getTargetExternalSymbol(Sym: "__tls_align", VT: PtrVT));
275 ReplaceNode(F: Node, T: TLSAlign);
276 return;
277 }
278 case Intrinsic::wasm_ptr_to_funcref: {
279 // Convert a function pointer to a funcref by reading the corresponding
280 // entry from the __indirect_function_table.
281 MachineFunction &MF = CurDAG->getMachineFunction();
282 auto PtrVT = MVT::getIntegerVT(BitWidth: MF.getDataLayout().getPointerSizeInBits());
283 MCSymbol *Table = WebAssembly::getOrCreateFunctionTableSymbol(
284 Ctx&: MF.getContext(), Subtarget);
285 SDValue TableSym = CurDAG->getMCSymbol(Sym: Table, VT: PtrVT);
286 SDValue FuncPtr = Node->getOperand(Num: 1);
287 if (Subtarget->hasAddr64() && FuncPtr.getValueType() == MVT::i64) {
288 // table.get expects an i32 but on 64 bit platforms the function pointer
289 // is an i64. In that case, i32.wrap_i64 to convert.
290 FuncPtr = SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::I32_WRAP_I64, dl: DL,
291 VT: MVT::i32, Op1: FuncPtr),
292 0);
293 }
294 MachineSDNode *FuncRef = CurDAG->getMachineNode(
295 Opcode: WebAssembly::TABLE_GET_FUNCREF, dl: DL, VT: MVT::funcref, Op1: TableSym, Op2: FuncPtr);
296 ReplaceNode(F: Node, T: FuncRef);
297 return;
298 }
299 case Intrinsic::wasm_ref_test_func: {
300 // First emit the TABLE_GET instruction to convert function pointer ==>
301 // funcref
302 MachineFunction &MF = CurDAG->getMachineFunction();
303 auto PtrVT = MVT::getIntegerVT(BitWidth: MF.getDataLayout().getPointerSizeInBits());
304 MCSymbol *Table = WebAssembly::getOrCreateFunctionTableSymbol(
305 Ctx&: MF.getContext(), Subtarget);
306 SDValue TableSym = CurDAG->getMCSymbol(Sym: Table, VT: PtrVT);
307 SDValue FuncPtr = Node->getOperand(Num: 1);
308 if (Subtarget->hasAddr64() && FuncPtr.getValueType() == MVT::i64) {
309 // table.get expects an i32 but on 64 bit platforms the function pointer
310 // is an i64. In that case, i32.wrap_i64 to convert.
311 FuncPtr = SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::I32_WRAP_I64, dl: DL,
312 VT: MVT::i32, Op1: FuncPtr),
313 0);
314 }
315 SDValue FuncRef =
316 SDValue(CurDAG->getMachineNode(Opcode: WebAssembly::TABLE_GET_FUNCREF, dl: DL,
317 VT: MVT::funcref, Op1: TableSym, Op2: FuncPtr),
318 0);
319
320 // Encode the signature information into the type index placeholder.
321 // This gets decoded and converted into the actual type signature in
322 // WebAssemblyMCInstLower.cpp.
323 SmallVector<MVT, 4> Params;
324 SmallVector<MVT, 4> Returns;
325
326 bool IsParam = false;
327 // Operand 0 is the return register, Operand 1 is the function pointer.
328 // The remaining operands encode the type of the function we are testing
329 // for.
330 for (unsigned I = 2, E = Node->getNumOperands(); I < E; ++I) {
331 MVT VT = Node->getOperand(Num: I).getValueType().getSimpleVT();
332 if (VT == MVT::Untyped) {
333 IsParam = true;
334 continue;
335 }
336 if (IsParam) {
337 Params.push_back(Elt: VT);
338 } else {
339 Returns.push_back(Elt: VT);
340 }
341 }
342 auto Sig = encodeFunctionSignature(DAG: CurDAG, DL, Returns, Params);
343
344 auto SigOp = CurDAG->getTargetConstant(
345 Val: Sig, DL, VT: EVT::getIntegerVT(Context&: *CurDAG->getContext(), BitWidth: Sig.getBitWidth()));
346 MachineSDNode *RefTestNode = CurDAG->getMachineNode(
347 Opcode: WebAssembly::REF_TEST_FUNCREF, dl: DL, VT: MVT::i32, Ops: {SigOp, FuncRef});
348 ReplaceNode(F: Node, T: RefTestNode);
349 return;
350 }
351 }
352 break;
353 }
354
355 case ISD::INTRINSIC_W_CHAIN: {
356 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
357 const auto &TLI = CurDAG->getTargetLoweringInfo();
358 MVT PtrVT = TLI.getPointerTy(DL: CurDAG->getDataLayout());
359 switch (IntNo) {
360 case Intrinsic::wasm_tls_base: {
361 MachineSDNode *TLSBase = llvm::WebAssembly::getTLSBase(
362 DAG&: *CurDAG, DL, Subtarget, Chain: Node->getOperand(Num: 0));
363 ReplaceNode(F: Node, T: TLSBase);
364 return;
365 }
366
367 case Intrinsic::wasm_catch: {
368 int Tag = Node->getConstantOperandVal(Num: 2);
369 SDValue SymNode = getTagSymNode(Tag, DAG: CurDAG);
370 unsigned CatchOpcode = WebAssembly::WasmUseLegacyEH
371 ? WebAssembly::CATCH_LEGACY
372 : WebAssembly::CATCH;
373 MachineSDNode *Catch =
374 CurDAG->getMachineNode(Opcode: CatchOpcode, dl: DL,
375 ResultTys: {
376 PtrVT, // exception pointer
377 MVT::Other // outchain type
378 },
379 Ops: {
380 SymNode, // exception symbol
381 Node->getOperand(Num: 0) // inchain
382 });
383 ReplaceNode(F: Node, T: Catch);
384 return;
385 }
386 }
387 break;
388 }
389
390 case ISD::INTRINSIC_VOID: {
391 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
392 switch (IntNo) {
393 case Intrinsic::wasm_throw: {
394 int Tag = Node->getConstantOperandVal(Num: 2);
395 SDValue SymNode = getTagSymNode(Tag, DAG: CurDAG);
396 MachineSDNode *Throw =
397 CurDAG->getMachineNode(Opcode: WebAssembly::THROW, dl: DL,
398 VT: MVT::Other, // outchain type
399 Ops: {
400 SymNode, // exception symbol
401 Node->getOperand(Num: 3), // thrown value
402 Node->getOperand(Num: 0) // inchain
403 });
404 ReplaceNode(F: Node, T: Throw);
405 return;
406 }
407 case Intrinsic::wasm_rethrow: {
408 // RETHROW's BB argument will be populated in LateEHPrepare. Just use a
409 // '0' as a placeholder for now.
410 MachineSDNode *Rethrow = CurDAG->getMachineNode(
411 Opcode: WebAssembly::RETHROW, dl: DL,
412 VT: MVT::Other, // outchain type
413 Ops: {
414 CurDAG->getConstant(Val: 0, DL, VT: MVT::i32), // placeholder
415 Node->getOperand(Num: 0) // inchain
416 });
417 ReplaceNode(F: Node, T: Rethrow);
418 return;
419 }
420 }
421 break;
422 }
423
424 case WebAssemblyISD::CALL:
425 case WebAssemblyISD::RET_CALL: {
426 // CALL has both variable operands and variable results, but ISel only
427 // supports one or the other. Split calls into two nodes glued together, one
428 // for the operands and one for the results. These two nodes will be
429 // recombined in a custom inserter hook into a single MachineInstr.
430 SmallVector<SDValue, 16> Ops;
431 for (size_t i = 1; i < Node->getNumOperands(); ++i) {
432 SDValue Op = Node->getOperand(Num: i);
433 // Remove the wrapper when the call target is a function, an external
434 // symbol (which will be lowered to a library function), or an alias of
435 // a function. If the target is not a function/external symbol, we
436 // shouldn't remove the wrapper, because we cannot call it directly and
437 // instead we want it to be loaded with a CONST instruction and called
438 // with a call_indirect later.
439 if (i == 1 && Op->getOpcode() == WebAssemblyISD::Wrapper) {
440 SDValue NewOp = Op->getOperand(Num: 0);
441 if (auto *GlobalOp = dyn_cast<GlobalAddressSDNode>(Val: NewOp.getNode())) {
442 if (isa<Function>(
443 Val: GlobalOp->getGlobal()->stripPointerCastsAndAliases()))
444 Op = NewOp;
445 } else if (isa<ExternalSymbolSDNode>(Val: NewOp.getNode())) {
446 Op = NewOp;
447 }
448 }
449 Ops.push_back(Elt: Op);
450 }
451
452 // Add the chain last
453 Ops.push_back(Elt: Node->getOperand(Num: 0));
454 MachineSDNode *CallParams =
455 CurDAG->getMachineNode(Opcode: WebAssembly::CALL_PARAMS, dl: DL, VT: MVT::Glue, Ops);
456
457 unsigned Results = Node->getOpcode() == WebAssemblyISD::CALL
458 ? WebAssembly::CALL_RESULTS
459 : WebAssembly::RET_CALL_RESULTS;
460
461 SDValue Link(CallParams, 0);
462 MachineSDNode *CallResults =
463 CurDAG->getMachineNode(Opcode: Results, dl: DL, VTs: Node->getVTList(), Ops: Link);
464 ReplaceNode(F: Node, T: CallResults);
465 return;
466 }
467
468 default:
469 break;
470 }
471
472 // Select the default instruction.
473 SelectCode(N: Node);
474}
475
476bool WebAssemblyDAGToDAGISel::SelectInlineAsmMemoryOperand(
477 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
478 std::vector<SDValue> &OutOps) {
479 switch (ConstraintID) {
480 case InlineAsm::ConstraintCode::m:
481 // We just support simple memory operands that just have a single address
482 // operand and need no special handling.
483 OutOps.push_back(x: Op);
484 return false;
485 default:
486 break;
487 }
488
489 return true;
490}
491
492bool WebAssemblyDAGToDAGISel::SelectAddrAddOperands(MVT OffsetType, SDValue N,
493 SDValue &Offset,
494 SDValue &Addr) {
495 assert(N.getNumOperands() == 2 && "Attempting to fold in a non-binary op");
496
497 // WebAssembly constant offsets are performed as unsigned with infinite
498 // precision, so we need to check for NoUnsignedWrap so that we don't fold an
499 // offset for an add that needs wrapping.
500 if (N.getOpcode() == ISD::ADD && !N.getNode()->getFlags().hasNoUnsignedWrap())
501 return false;
502
503 for (size_t i = 0; i < 2; ++i) {
504 SDValue Op = N.getOperand(i);
505 SDValue OtherOp = N.getOperand(i: i == 0 ? 1 : 0);
506
507 // Folds constants in an add into the offset.
508 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: Op)) {
509 Offset =
510 CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL: SDLoc(N), VT: OffsetType);
511 Addr = OtherOp;
512 return true;
513 }
514
515 // Fold target global addresses into the offset.
516 if (!TM.isPositionIndependent()) {
517 if (Op.getOpcode() == WebAssemblyISD::Wrapper)
518 Op = Op.getOperand(i: 0);
519
520 if (Op.getOpcode() == ISD::TargetGlobalAddress) {
521 Addr = OtherOp;
522 Offset = Op;
523 return true;
524 }
525 }
526 }
527 return false;
528}
529
530bool WebAssemblyDAGToDAGISel::SelectAddrOperands(MVT AddrType,
531 unsigned ConstOpc, SDValue N,
532 SDValue &Offset,
533 SDValue &Addr) {
534 SDLoc DL(N);
535
536 // Fold target global addresses into the offset.
537 if (!TM.isPositionIndependent()) {
538 SDValue Op(N);
539 if (Op.getOpcode() == WebAssemblyISD::Wrapper)
540 Op = Op.getOperand(i: 0);
541
542 if (Op.getOpcode() == ISD::TargetGlobalAddress) {
543 Offset = Op;
544 Addr = SDValue(
545 CurDAG->getMachineNode(Opcode: ConstOpc, dl: DL, VT: AddrType,
546 Op1: CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType)),
547 0);
548 return true;
549 }
550 }
551
552 // Fold anything inside an add into the offset.
553 if (N.getOpcode() == ISD::ADD &&
554 SelectAddrAddOperands(OffsetType: AddrType, N, Offset, Addr))
555 return true;
556
557 // Likewise, treat an 'or' node as an 'add' if the or'ed bits are known to be
558 // zero and fold them into the offset too.
559 if (N.getOpcode() == ISD::OR) {
560 bool OrIsAdd;
561 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
562 OrIsAdd =
563 CurDAG->MaskedValueIsZero(Op: N->getOperand(Num: 0), Mask: CN->getAPIntValue());
564 } else {
565 KnownBits Known0 = CurDAG->computeKnownBits(Op: N->getOperand(Num: 0), Depth: 0);
566 KnownBits Known1 = CurDAG->computeKnownBits(Op: N->getOperand(Num: 1), Depth: 0);
567 OrIsAdd = (~Known0.Zero & ~Known1.Zero) == 0;
568 }
569
570 if (OrIsAdd && SelectAddrAddOperands(OffsetType: AddrType, N, Offset, Addr))
571 return true;
572 }
573
574 // Fold constant addresses into the offset.
575 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
576 Offset = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: AddrType);
577 Addr = SDValue(
578 CurDAG->getMachineNode(Opcode: ConstOpc, dl: DL, VT: AddrType,
579 Op1: CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType)),
580 0);
581 return true;
582 }
583
584 // Else it's a plain old load/store with no offset.
585 Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: AddrType);
586 Addr = N;
587 return true;
588}
589
590bool WebAssemblyDAGToDAGISel::SelectAddrOperands32(SDValue Op, SDValue &Offset,
591 SDValue &Addr) {
592 return SelectAddrOperands(AddrType: MVT::i32, ConstOpc: WebAssembly::CONST_I32, N: Op, Offset, Addr);
593}
594
595bool WebAssemblyDAGToDAGISel::SelectAddrOperands64(SDValue Op, SDValue &Offset,
596 SDValue &Addr) {
597 return SelectAddrOperands(AddrType: MVT::i64, ConstOpc: WebAssembly::CONST_I64, N: Op, Offset, Addr);
598}
599
600static MemSDNode *findMemSDNode(SDNode *N) {
601 while (N) {
602 if (auto *MemNode = dyn_cast<MemSDNode>(Val: N))
603 return MemNode;
604 switch (N->getOpcode()) {
605 case ISD::ZERO_EXTEND:
606 case ISD::SIGN_EXTEND:
607 case ISD::ANY_EXTEND:
608 case ISD::SIGN_EXTEND_INREG:
609 case ISD::AssertZext:
610 case ISD::AssertSext:
611 case ISD::TRUNCATE:
612 case ISD::BITCAST:
613 case ISD::AND:
614 N = N->getOperand(Num: 0).getNode();
615 break;
616 default:
617 return nullptr;
618 }
619 }
620 return nullptr;
621}
622
623bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands(SDNode *Op, SDValue N,
624 SDValue &Offset,
625 SDValue &Addr,
626 SDValue &Order,
627 bool Is64) {
628 auto *MemNode = findMemSDNode(N: Op);
629 if (!MemNode)
630 return false;
631
632 bool Match = Is64 ? SelectAddrOperands64(Op: N, Offset, Addr)
633 : SelectAddrOperands32(Op: N, Offset, Addr);
634 if (!Match)
635 return false;
636
637 auto Ordering = MemNode->getMergedOrdering();
638 unsigned OrderVal = wasm::WASM_MEM_ORDER_SEQ_CST;
639 if (Subtarget->hasRelaxedAtomics())
640 OrderVal = getWebAssemblyMemoryOrder(Ordering);
641 Order = CurDAG->getTargetConstant(Val: OrderVal, DL: SDLoc(Op), VT: MVT::i32);
642 return true;
643}
644
645bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands32(SDNode *Op, SDValue N,
646 SDValue &Offset,
647 SDValue &Addr,
648 SDValue &Order) {
649 return SelectAtomicAddrOperands(Op, N, Offset, Addr, Order, /*Is64=*/false);
650}
651
652bool WebAssemblyDAGToDAGISel::SelectAtomicAddrOperands64(SDNode *Op, SDValue N,
653 SDValue &Offset,
654 SDValue &Addr,
655 SDValue &Order) {
656 return SelectAtomicAddrOperands(Op, N, Offset, Addr, Order, /*Is64=*/true);
657}
658
659/// This pass converts a legalized DAG into a WebAssembly-specific DAG, ready
660/// for instruction scheduling.
661FunctionPass *llvm::createWebAssemblyISelDag(WebAssemblyTargetMachine &TM,
662 CodeGenOptLevel OptLevel) {
663 return new WebAssemblyDAGToDAGISelLegacy(TM, OptLevel);
664}
665