1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64TargetMachine.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/SDPatternMatch.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/IR/Function.h" // To access function attributes.
23#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
26#include "llvm/Support/AArch64MemoryHints.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Support/MathExtras.h"
31#include "llvm/Support/raw_ostream.h"
32
33using namespace llvm;
34using namespace llvm::SDPatternMatch;
35
36#define DEBUG_TYPE "aarch64-isel"
37#define PASS_NAME "AArch64 Instruction Selection"
38
39// https://github.com/llvm/llvm-project/issues/114425
40#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
41#pragma inline_depth(0)
42#endif
43
44//===--------------------------------------------------------------------===//
45/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49
50class AArch64DAGToDAGISel : public SelectionDAGISel {
51
52 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
53 /// make the right decision when generating code for different targets.
54 const AArch64Subtarget *Subtarget;
55
56public:
57 AArch64DAGToDAGISel() = delete;
58
59 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
60 CodeGenOptLevel OptLevel)
61 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
62
63 bool runOnMachineFunction(MachineFunction &MF) override {
64 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
65 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
66 }
67
68 void Select(SDNode *Node) override;
69 void PreprocessISelDAG() override;
70
71 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
72 /// inline asm expressions.
73 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
74 InlineAsm::ConstraintCode ConstraintID,
75 std::vector<SDValue> &OutOps) override;
76
77 template <signed Low, signed High, signed Scale>
78 bool SelectRDVLImm(SDValue N, SDValue &Imm);
79
80 template <signed Low, signed High>
81 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
82
83 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
84 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
85 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
86 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
87 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
88 return SelectShiftedRegister(N, AllowROR: false, Reg, Shift);
89 }
90 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
91 return SelectShiftedRegister(N, AllowROR: true, Reg, Shift);
92 }
93 template <unsigned ShiftWidth>
94 bool SelectShiftMask(SDValue N, SDValue &ShAmt);
95
96 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
97 return SelectAddrModeIndexed7S(N, Size: 1, Base, OffImm);
98 }
99 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
100 return SelectAddrModeIndexed7S(N, Size: 2, Base, OffImm);
101 }
102 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
103 return SelectAddrModeIndexed7S(N, Size: 4, Base, OffImm);
104 }
105 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
106 return SelectAddrModeIndexed7S(N, Size: 8, Base, OffImm);
107 }
108 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
109 return SelectAddrModeIndexed7S(N, Size: 16, Base, OffImm);
110 }
111 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
112 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 9, Size: 16, Base, OffImm);
113 }
114 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
115 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: false, BW: 6, Size: 16, Base, OffImm);
116 }
117 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
118 return SelectAddrModeIndexed(N, Size: 1, Base, OffImm);
119 }
120 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
121 return SelectAddrModeIndexed(N, Size: 2, Base, OffImm);
122 }
123 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
124 return SelectAddrModeIndexed(N, Size: 4, Base, OffImm);
125 }
126 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
127 return SelectAddrModeIndexed(N, Size: 8, Base, OffImm);
128 }
129 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
130 return SelectAddrModeIndexed(N, Size: 16, Base, OffImm);
131 }
132 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
133 return SelectAddrModeUnscaled(N, Size: 1, Base, OffImm);
134 }
135 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
136 return SelectAddrModeUnscaled(N, Size: 2, Base, OffImm);
137 }
138 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
139 return SelectAddrModeUnscaled(N, Size: 4, Base, OffImm);
140 }
141 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
142 return SelectAddrModeUnscaled(N, Size: 8, Base, OffImm);
143 }
144 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
145 return SelectAddrModeUnscaled(N, Size: 16, Base, OffImm);
146 }
147 template <unsigned Size, unsigned Max>
148 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
149 // Test if there is an appropriate addressing mode and check if the
150 // immediate fits.
151 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
152 if (Found) {
153 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: OffImm)) {
154 int64_t C = CI->getSExtValue();
155 if (C <= Max)
156 return true;
157 }
158 }
159
160 // Otherwise, base only, materialize address in register.
161 Base = N;
162 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
163 return true;
164 }
165
166 template<int Width>
167 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
168 SDValue &SignExtend, SDValue &DoShift) {
169 return SelectAddrModeWRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
170 }
171
172 template<int Width>
173 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
174 SDValue &SignExtend, SDValue &DoShift) {
175 return SelectAddrModeXRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
176 }
177
178 bool SelectExtractHigh(SDValue N, SDValue &Res) {
179 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
180 N = N->getOperand(Num: 0);
181 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
182 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
183 return false;
184 EVT VT = N->getValueType(ResNo: 0);
185 EVT LVT = N->getOperand(Num: 0).getValueType();
186 unsigned Index = N->getConstantOperandVal(Num: 1);
187 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
188 Index != VT.getVectorNumElements())
189 return false;
190 Res = N->getOperand(Num: 0);
191 return true;
192 }
193
194 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
195 if (N.getOpcode() != AArch64ISD::VLSHR)
196 return false;
197 SDValue Op = N->getOperand(Num: 0);
198 EVT VT = Op.getValueType();
199 unsigned ShtAmt = N->getConstantOperandVal(Num: 1);
200 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
201 return false;
202
203 APInt Imm;
204 if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::MOVIshift)
205 Imm = APInt(VT.getScalarSizeInBits(),
206 Op.getOperand(i: 1).getConstantOperandVal(i: 0)
207 << Op.getOperand(i: 1).getConstantOperandVal(i: 1));
208 else if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::DUP &&
209 isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getOperand(i: 0)))
210 Imm = APInt(VT.getScalarSizeInBits(),
211 Op.getOperand(i: 1).getConstantOperandVal(i: 0));
212 else
213 return false;
214
215 if (Imm != 1ULL << (ShtAmt - 1))
216 return false;
217
218 Res1 = Op.getOperand(i: 0);
219 Res2 = CurDAG->getTargetConstant(Val: ShtAmt, DL: SDLoc(N), VT: MVT::i32);
220 return true;
221 }
222
223 bool SelectDupZeroOrUndef(SDValue N) {
224 switch(N->getOpcode()) {
225 case ISD::UNDEF:
226 case ISD::POISON:
227 return true;
228 case AArch64ISD::DUP:
229 case ISD::SPLAT_VECTOR: {
230 auto Opnd0 = N->getOperand(Num: 0);
231 if (isNullConstant(V: Opnd0))
232 return true;
233 if (isNullFPConstant(V: Opnd0))
234 return true;
235 break;
236 }
237 default:
238 break;
239 }
240
241 return false;
242 }
243
244 bool SelectAny(SDValue) { return true; }
245
246 bool SelectDupZero(SDValue N) {
247 switch(N->getOpcode()) {
248 case AArch64ISD::DUP:
249 case ISD::SPLAT_VECTOR: {
250 auto Opnd0 = N->getOperand(Num: 0);
251 if (isNullConstant(V: Opnd0))
252 return true;
253 if (isNullFPConstant(V: Opnd0))
254 return true;
255 break;
256 }
257 }
258
259 return false;
260 }
261
262 template <MVT::SimpleValueType VT, bool Negate>
263 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
264 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
265 }
266
267 template <MVT::SimpleValueType VT, bool Negate>
268 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
269 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
270 }
271
272 template <MVT::SimpleValueType VT>
273 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
274 return SelectSVECpyDupImm(N, VT, Imm, Shift);
275 }
276
277 template <MVT::SimpleValueType VT, bool Invert = false>
278 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
279 return SelectSVELogicalImm(N, VT, Imm, Invert);
280 }
281
282 template <MVT::SimpleValueType VT>
283 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
284 return SelectSVEArithImm(N, VT, Imm);
285 }
286
287 template <unsigned Low, unsigned High, bool AllowSaturation = false>
288 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
289 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
290 }
291
292 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
293 if (N->getOpcode() != ISD::SPLAT_VECTOR)
294 return false;
295
296 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
297 return SelectSVEShiftImm(N: N->getOperand(Num: 0), /* Low */ 1,
298 /* High */ EltVT.getFixedSizeInBits(),
299 /* AllowSaturation */ true, Imm);
300 }
301
302 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
303 template<signed Min, signed Max, signed Scale, bool Shift>
304 bool SelectCntImm(SDValue N, SDValue &Imm) {
305 if (!isa<ConstantSDNode>(Val: N))
306 return false;
307
308 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
309 if (Shift)
310 MulImm = 1LL << MulImm;
311
312 if ((MulImm % std::abs(x: Scale)) != 0)
313 return false;
314
315 MulImm /= Scale;
316 if ((MulImm >= Min) && (MulImm <= Max)) {
317 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
318 return true;
319 }
320
321 return false;
322 }
323
324 template <signed Max, signed Scale>
325 bool SelectEXTImm(SDValue N, SDValue &Imm) {
326 if (!isa<ConstantSDNode>(Val: N))
327 return false;
328
329 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
330
331 if (MulImm >= 0 && MulImm <= Max) {
332 MulImm *= Scale;
333 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
334 return true;
335 }
336
337 return false;
338 }
339
340 template <unsigned BaseReg, unsigned Max>
341 bool ImmToReg(SDValue N, SDValue &Imm) {
342 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: N)) {
343 uint64_t C = CI->getZExtValue();
344
345 if (C > Max)
346 return false;
347
348 Imm = CurDAG->getRegister(Reg: BaseReg + C, VT: MVT::Other);
349 return true;
350 }
351 return false;
352 }
353
354 /// Form sequences of consecutive 64/128-bit registers for use in NEON
355 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
356 /// between 1 and 4 elements. If it contains a single element that is returned
357 /// unchanged; otherwise a REG_SEQUENCE value is returned.
358 SDValue createDTuple(ArrayRef<SDValue> Vecs);
359 SDValue createQTuple(ArrayRef<SDValue> Vecs);
360 // Form a sequence of SVE registers for instructions using list of vectors,
361 // e.g. structured loads and stores (ldN, stN).
362 SDValue createZTuple(ArrayRef<SDValue> Vecs);
363
364 // Similar to above, except the register must start at a multiple of the
365 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
366 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
367
368 /// Generic helper for the createDTuple/createQTuple
369 /// functions. Those should almost always be called instead.
370 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
371 const unsigned SubRegs[]);
372
373 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
374
375 bool tryIndexedLoad(SDNode *N);
376
377 void SelectPtrauthAuth(SDNode *N);
378 void SelectPtrauthResign(SDNode *N);
379 void SelectPtrauthResignWithPC(SDNode *N);
380
381 bool trySelectStackSlotTagP(SDNode *N);
382 void SelectTagP(SDNode *N);
383
384 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
385 unsigned SubRegIdx);
386 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
387 unsigned SubRegIdx);
388 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
389 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
390 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
391 unsigned Opc_rr, unsigned Opc_ri,
392 bool IsIntr = false);
393 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
394 unsigned Scale, unsigned Opc_ri,
395 unsigned Opc_rr);
396 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
397 bool IsZmMulti, unsigned Opcode,
398 bool HasPred = false);
399 void SelectPExtPair(SDNode *N, unsigned Opc);
400 void SelectWhilePair(SDNode *N, unsigned Opc);
401 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
402 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
403 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
404 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
405 bool IsTupleInput, unsigned Opc);
406 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
407
408 template <unsigned MaxIdx, unsigned Scale>
409 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
410 unsigned Op);
411 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
412 unsigned Op, unsigned MaxIdx, unsigned Scale,
413 unsigned BaseReg = 0);
414 /// SVE Reg+Imm addressing mode.
415 template <int64_t Min, int64_t Max>
416 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
417 SDValue &OffImm);
418 /// SVE Reg+Reg address mode.
419 template <unsigned Scale>
420 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
421 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
422 }
423
424 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
425 unsigned Opc, uint32_t MaxImm);
426 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
427
428 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
429 unsigned NumInVecs);
430
431 template <unsigned MaxIdx, unsigned Scale>
432 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
433 return SelectSMETileSlice(N, MaxSize: MaxIdx, Vector, Offset, Scale);
434 }
435
436 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
437 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
438 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
439 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
440 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
441 unsigned Opc_rr, unsigned Opc_ri);
442 std::tuple<unsigned, SDValue, SDValue>
443 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
444 const SDValue &OldBase, const SDValue &OldOffset,
445 unsigned Scale);
446
447 bool tryBitfieldExtractOp(SDNode *N);
448 bool tryBitfieldInsertOp(SDNode *N);
449 bool tryBitfieldInsertInZeroOp(SDNode *N);
450
451 bool tryReadRegister(SDNode *N);
452 bool tryWriteRegister(SDNode *N);
453
454 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
455 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
456
457 bool trySelectXAR(SDNode *N);
458
459 bool tryFoldCselToFMaxMin(SDNode *N);
460
461// Include the pieces autogenerated from the target description.
462#include "AArch64GenDAGISel.inc"
463
464private:
465 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
466 SDValue &Shift);
467 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
468 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
469 SDValue &OffImm) {
470 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 7, Size, Base, OffImm);
471 }
472 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
473 unsigned Size, SDValue &Base,
474 SDValue &OffImm);
475 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
476 SDValue &OffImm);
477 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
478 SDValue &OffImm);
479 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
480 SDValue &Offset, SDValue &SignExtend,
481 SDValue &DoShift);
482 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
483 SDValue &Offset, SDValue &SignExtend,
484 SDValue &DoShift);
485 bool isWorthNegatingImm(SDValue V) const;
486 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
487 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
488 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
489 SDValue &Offset, SDValue &SignExtend);
490
491 template<unsigned RegWidth>
492 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
493 return SelectCVTFixedPosOperand(N, FixedPos, Width: RegWidth);
494 }
495 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
496
497 template <unsigned RegWidth>
498 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
499 return SelectCVTFixedPointVec(N, FixedPos, Width: RegWidth);
500 }
501 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
502
503 template<unsigned RegWidth>
504 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
505 return SelectCVTFixedPosRecipOperand(N, FixedPos, Width: RegWidth);
506 }
507
508 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
509 unsigned Width);
510
511 template <unsigned FloatWidth>
512 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
513 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, Width: FloatWidth);
514 }
515
516 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
517 unsigned Width);
518
519 bool SelectCMP_SWAP(SDNode *N);
520
521 AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
522 bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const;
523
524 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
525 bool Negate);
526 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
527 SDValue &Shift, bool Negate);
528 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
529 bool Negate);
530 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
531 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
532
533 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
534 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
535 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
536 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
537 bool SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm);
538 bool SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm);
539
540 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
541 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
542 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
543 bool AllowSaturation, SDValue &Imm);
544
545 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
546 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
547 SDValue &Offset);
548 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
549 SDValue &Offset, unsigned Scale = 1);
550
551 bool SelectAllActivePredicate(SDValue N);
552 bool SelectAnyPredicate(SDValue N);
553
554 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
555
556 template <bool MatchCBB>
557 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
558};
559
560class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
561public:
562 static char ID;
563 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
564 CodeGenOptLevel OptLevel)
565 : SelectionDAGISelLegacy(
566 ID, std::make_unique<AArch64DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
567};
568} // end anonymous namespace
569
570char AArch64DAGToDAGISelLegacy::ID = 0;
571
572INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
573
574AArch64DAGToDAGISelPass::AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
575 : SelectionDAGISelPass(
576 std::make_unique<AArch64DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
577
578/// addBitcastHints - This method adds bitcast hints to the operands of a node
579/// to help instruction selector determine which operands are in Neon registers.
580static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N) {
581 SDLoc DL(&N);
582 auto getFloatVT = [&](EVT VT) {
583 EVT ScalarVT = VT.getScalarType();
584 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
585 return VT.changeElementType(Context&: *(DAG.getContext()),
586 EltVT: ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
587 };
588 SmallVector<SDValue, 2> NewOps;
589 NewOps.reserve(N: N.getNumOperands());
590
591 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
592 auto bitcasted = DAG.getBitcast(VT: getFloatVT(N.getOperand(Num: I).getValueType()),
593 V: N.getOperand(Num: I));
594 NewOps.push_back(Elt: bitcasted);
595 }
596 EVT OrigVT = N.getValueType(ResNo: 0);
597 SDValue OpNode = DAG.getNode(Opcode: N.getOpcode(), DL, VT: getFloatVT(OrigVT), Ops: NewOps);
598 return DAG.getBitcast(VT: OrigVT, V: OpNode);
599}
600
601/// isIntImmediate - This method tests to see if the node is a constant
602/// operand. If so Imm will receive the 64-bit value.
603static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
604 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(Val: N)) {
605 Imm = C->getZExtValue();
606 return true;
607 }
608 return false;
609}
610
611// isIntImmediate - This method tests to see if a constant operand.
612// If so Imm will receive the value.
613static bool isIntImmediate(SDValue N, uint64_t &Imm) {
614 return isIntImmediate(N: N.getNode(), Imm);
615}
616
617// isOpcWithIntImmediate - This method tests to see if the node is a specific
618// opcode and that it has a immediate integer right operand.
619// If so Imm will receive the 32 bit value.
620static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
621 uint64_t &Imm) {
622 return N->getOpcode() == Opc &&
623 isIntImmediate(N: N->getOperand(Num: 1).getNode(), Imm);
624}
625
626// isIntImmediateEq - This method tests to see if N is a constant operand that
627// is equivalent to 'ImmExpected'.
628#ifndef NDEBUG
629static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
630 uint64_t Imm;
631 if (!isIntImmediate(N.getNode(), Imm))
632 return false;
633 return Imm == ImmExpected;
634}
635#endif
636
637static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
638 assert(RegWidth == 32 || RegWidth == 64);
639 if (RegWidth == 32)
640 return APInt(RegWidth,
641 uint32_t(AArch64_AM::decodeAdvSIMDModImmType11(Imm)));
642 return APInt(RegWidth, AArch64_AM::decodeAdvSIMDModImmType12(Imm));
643}
644
645// Decodes the raw integer splat value from a NEON splat operation.
646static std::optional<APInt> DecodeNEONSplat(SDValue N,
647 const AArch64Subtarget *Subtarget) {
648 assert(N.getValueType().isInteger() && "Only integers are supported");
649 if (N->getOpcode() == AArch64ISD::NVCAST ||
650 (N->getOpcode() == ISD::BITCAST && Subtarget->isLittleEndian()))
651 N = N->getOperand(Num: 0);
652 unsigned SplatWidth = N.getScalarValueSizeInBits();
653 if (N.getOpcode() == AArch64ISD::FMOV)
654 return DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth: SplatWidth);
655 if (N->getOpcode() == AArch64ISD::MOVI)
656 return APInt(SplatWidth, N.getConstantOperandVal(i: 0));
657 if (N->getOpcode() == AArch64ISD::MOVIshift)
658 return APInt(SplatWidth, N.getConstantOperandVal(i: 0)
659 << N.getConstantOperandVal(i: 1));
660 if (N->getOpcode() == AArch64ISD::MVNIshift)
661 return ~APInt(SplatWidth, N.getConstantOperandVal(i: 0)
662 << N.getConstantOperandVal(i: 1));
663 if (N->getOpcode() == AArch64ISD::MOVIedit)
664 return APInt(SplatWidth, AArch64_AM::decodeAdvSIMDModImmType10(
665 Imm: N.getConstantOperandVal(i: 0)));
666 if (N->getOpcode() == AArch64ISD::DUP)
667 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
668 return Const->getAPIntValue().trunc(width: SplatWidth);
669 APInt SplatVal;
670 if (ISD::isConstantSplatVector(V: N, SplatValue&: SplatVal))
671 return SplatVal.trunc(width: SplatWidth);
672 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
673 // in AArch64ISelLowering.
674 return std::nullopt;
675}
676
677// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
678// matching the element size of N.
679static std::optional<APInt>
680GetNEONSplatValue(SDValue N, const AArch64Subtarget *Subtarget) {
681 unsigned SplatWidth = N.getScalarValueSizeInBits();
682 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N, Subtarget)) {
683 if (SplatVal->getBitWidth() <= SplatWidth)
684 return APInt::getSplat(NewLen: SplatWidth, V: *SplatVal);
685 if (SplatVal->isSplat(SplatSizeInBits: SplatWidth))
686 return SplatVal->trunc(width: SplatWidth);
687 }
688 return std::nullopt;
689}
690
691bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
692 SDValue &Imm) {
693 std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget);
694 if (!ImmVal)
695 return false;
696 uint64_t Encoding;
697 if (!AArch64_AM::isSVELogicalImm(SizeInBits: N.getScalarValueSizeInBits(),
698 ImmVal: ImmVal->getZExtValue(), Encoding))
699 return false;
700
701 Imm = CurDAG->getTargetConstant(Val: Encoding, DL: SDLoc(N), VT: MVT::i64);
702 return true;
703}
704
705bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
706 SDValue &Shift) {
707 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
708 return SelectSVEAddSubImm(DL: SDLoc(N), Value: *ImmVal,
709 VT: N.getValueType().getScalarType().getSimpleVT(),
710 Imm, Shift,
711 /*Negate=*/false);
712 return false;
713}
714
715bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
716 SDValue &Imm) {
717 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
718 return SelectSVESignedArithImm(DL: SDLoc(N), Value: *ImmVal, Imm);
719 return false;
720}
721
722bool AArch64DAGToDAGISel::SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm) {
723 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
724 if (!ImmAPIntVal)
725 return false;
726
727 int64_t ImmVal = ImmAPIntVal->getSExtValue();
728 if (ImmVal < -128 || ImmVal > 127)
729 return false;
730
731 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
732 return true;
733}
734
735bool AArch64DAGToDAGISel::SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm) {
736 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
737 if (!ImmAPIntVal)
738 return false;
739
740 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
741 if (ImmVal > 255)
742 return false;
743
744 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
745 return true;
746}
747
748bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
749 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
750 std::vector<SDValue> &OutOps) {
751 switch(ConstraintID) {
752 default:
753 llvm_unreachable("Unexpected asm memory constraint");
754 case InlineAsm::ConstraintCode::m:
755 case InlineAsm::ConstraintCode::o:
756 case InlineAsm::ConstraintCode::Q:
757 // We need to make sure that this one operand does not end up in XZR, thus
758 // require the address to be in a pointer register.
759 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
760 const TargetRegisterClass *TRC =
761 TII->getInlineAsmMemoryOperandRegClass(C: ConstraintID);
762 SDLoc dl(Op);
763 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL: dl, VT: MVT::i64);
764 SDValue NewOp =
765 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
766 dl, VT: Op.getValueType(),
767 Op1: Op, Op2: RC), 0);
768 OutOps.push_back(x: NewOp);
769 return false;
770 }
771 return true;
772}
773
774// Returns true if every use of N is a variable shift (SHL/SRL/SRA/ROTR) that
775// uses N as its shift-amount operand. When that holds, folding N into the
776// shift amount (e.g. generating a NEG or MVN) is safe even with multiple
777// uses, because each shift only reads the low bits of the amount.
778static bool allUsesAreShiftAmounts(SDValue N) {
779 return all_of(Range: N.getNode()->users(), P: [&](SDNode *User) {
780 switch (User->getOpcode()) {
781 case ISD::SHL:
782 case ISD::SRL:
783 case ISD::SRA:
784 case ISD::ROTR:
785 return User->getOperand(Num: 1).getNode() == N.getNode();
786 case ISD::ANY_EXTEND:
787 case ISD::ZERO_EXTEND:
788 case ISD::SIGN_EXTEND:
789 // A 32-bit shift amount may be widened to i64 before it reaches the
790 // shift; look through the extend to the ultimate users.
791 return allUsesAreShiftAmounts(N: SDValue(User, 0));
792 default:
793 return false;
794 }
795 });
796}
797
798template <unsigned ShiftWidth>
799bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
800 // AArch64 shift instructions only use the low log2(ShiftWidth) bits of the
801 // shift amount. The matcher may run on a node wider or narrower than the
802 // shift result (an i32 amount widened to i64, or an i64 modulus reduced to
803 // an i32 subtract), so the redundancy threshold and modulus come from the
804 // ShiftWidth template parameter, while the NEG/NOT opcode follows the node's
805 // own type. Return false if nothing was combined so other patterns (e.g.
806 // zext/sext GPR32 -> SUBREG_TO_REG) can match.
807 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
808 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
809 uint64_t Mask = N.getConstantOperandVal(i: 1);
810 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
811 if ((unsigned)llvm::countr_one(Value: Mask) >= Log2_32(Value: ShiftWidth)) {
812 ShAmt = N.getOperand(i: 0);
813 return true;
814 }
815 }
816 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
817 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
818 // shift can use X directly; the original ADD/SUB stays for any other users.
819 if ((N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) &&
820 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
821 uint64_t Imm;
822 if (isIntImmediate(N: N.getOperand(i: 1).getNode(), Imm) &&
823 (Imm % ShiftWidth == 0)) {
824 ShAmt = N.getOperand(i: 0);
825 return true;
826 }
827 }
828
829 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
830 // to generate a NEG instead of a SUB from a constant.
831 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
832 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
833 uint64_t Imm;
834 if (isIntImmediate(N: N.getOperand(i: 0).getNode(), Imm) && Imm != 0 &&
835 (Imm % ShiftWidth == 0)) {
836 SDLoc DL(N);
837 EVT VT = N.getValueType();
838 bool Is32 = VT == MVT::i32;
839 unsigned NegOpc = Is32 ? AArch64::SUBWrr : AArch64::SUBXrr;
840 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
841 SDValue Zero =
842 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT);
843 MachineSDNode *Neg =
844 CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero, Op2: N.getOperand(i: 1));
845 ShAmt = SDValue(Neg, 0);
846 return true;
847 }
848 }
849
850 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
851 // to generate a NOT (MVN) instead of a SUB from a constant.
852 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
853 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
854 uint64_t Imm;
855 if (isIntImmediate(N: N.getOperand(i: 0).getNode(), Imm) &&
856 (Imm % ShiftWidth == ShiftWidth - 1)) {
857 SDLoc DL(N);
858 EVT VT = N.getValueType();
859 bool Is32 = VT == MVT::i32;
860 unsigned NotOpc = Is32 ? AArch64::ORNWrr : AArch64::ORNXrr;
861 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
862 SDValue Zero =
863 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT);
864 MachineSDNode *Not =
865 CurDAG->getMachineNode(Opcode: NotOpc, dl: DL, VT, Op1: Zero, Op2: N.getOperand(i: 1));
866 ShAmt = SDValue(Not, 0);
867 return true;
868 }
869 }
870
871 return false;
872}
873
874/// SelectArithImmed - Select an immediate value that can be represented as
875/// a 12-bit value shifted left by either 0 or 12. If so, return true with
876/// Val set to the 12-bit value and Shift set to the shifter operand.
877bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
878 SDValue &Shift) {
879 // This function is called from the addsub_shifted_imm ComplexPattern,
880 // which lists [imm] as the list of opcode it's interested in, however
881 // we still need to check whether the operand is actually an immediate
882 // here because the ComplexPattern opcode list is only used in
883 // root-level opcode matching.
884 if (!isa<ConstantSDNode>(Val: N.getNode()))
885 return false;
886
887 uint64_t Immed = N.getNode()->getAsZExtVal();
888
889 if (!AArch64_AM::isLegalArithImmed(C: Immed))
890 return false;
891
892 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(C: Immed);
893 Immed >>= ShiftAmt;
894
895 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
896 SDLoc dl(N);
897 Val = CurDAG->getTargetConstant(Val: Immed, DL: dl, VT: MVT::i32);
898 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: dl, VT: MVT::i32);
899 return true;
900}
901
902/// SelectNegArithImmed - As above, but negates the value before trying to
903/// select it.
904bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
905 SDValue &Shift) {
906 // This function is called from the addsub_shifted_imm ComplexPattern,
907 // which lists [imm] as the list of opcode it's interested in, however
908 // we still need to check whether the operand is actually an immediate
909 // here because the ComplexPattern opcode list is only used in
910 // root-level opcode matching.
911 if (!isa<ConstantSDNode>(Val: N.getNode()))
912 return false;
913
914 // The immediate operand must be a 24-bit zero-extended immediate.
915 uint64_t Immed = N.getNode()->getAsZExtVal();
916
917 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
918 // have the opposite effect on the C flag, so this pattern mustn't match under
919 // those circumstances.
920 if (Immed == 0)
921 return false;
922
923 if (N.getValueType() == MVT::i32)
924 Immed = ~((uint32_t)Immed) + 1;
925 else
926 Immed = ~Immed + 1ULL;
927 if (Immed & 0xFFFFFFFFFF000000ULL)
928 return false;
929
930 Immed &= 0xFFFFFFULL;
931 return SelectArithImmed(N: CurDAG->getConstant(Val: Immed, DL: SDLoc(N), VT: MVT::i32), Val,
932 Shift);
933}
934
935/// getShiftTypeForNode - Translate a shift node to the corresponding
936/// ShiftType value.
937static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
938 switch (N.getOpcode()) {
939 default:
940 return AArch64_AM::InvalidShiftExtend;
941 case ISD::SHL:
942 return AArch64_AM::LSL;
943 case ISD::SRL:
944 return AArch64_AM::LSR;
945 case ISD::SRA:
946 return AArch64_AM::ASR;
947 case ISD::ROTR:
948 return AArch64_AM::ROR;
949 }
950}
951
952static bool isMemOpOrPrefetch(SDNode *N) {
953 return isa<MemSDNode>(Val: *N) || N->getOpcode() == AArch64ISD::PREFETCH;
954}
955
956/// Determine whether it is worth it to fold SHL into the addressing
957/// mode.
958static bool isWorthFoldingSHL(SDValue V) {
959 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
960 // It is worth folding logical shift of up to three places.
961 auto *CSD = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
962 if (!CSD)
963 return false;
964 unsigned ShiftVal = CSD->getZExtValue();
965 if (ShiftVal > 3)
966 return false;
967
968 // Check if this particular node is reused in any non-memory related
969 // operation. If yes, do not try to fold this node into the address
970 // computation, since the computation will be kept.
971 const SDNode *Node = V.getNode();
972 for (SDNode *UI : Node->users())
973 if (!isMemOpOrPrefetch(N: UI))
974 for (SDNode *UII : UI->users())
975 if (!isMemOpOrPrefetch(N: UII))
976 return false;
977 return true;
978}
979
980/// Determine whether it is worth to fold V into an extended register addressing
981/// mode.
982bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
983 // Trivial if we are optimizing for code size or if there is only
984 // one use of the value.
985 if (CurDAG->shouldOptForSize() || V.hasOneUse())
986 return true;
987
988 // If a subtarget has a slow shift, folding a shift into multiple loads
989 // costs additional micro-ops.
990 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
991 return false;
992
993 // Check whether we're going to emit the address arithmetic anyway because
994 // it's used by a non-address operation.
995 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
996 return true;
997 if (V.getOpcode() == ISD::ADD) {
998 const SDValue LHS = V.getOperand(i: 0);
999 const SDValue RHS = V.getOperand(i: 1);
1000 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: LHS))
1001 return true;
1002 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: RHS))
1003 return true;
1004 }
1005
1006 // It hurts otherwise, since the value will be reused.
1007 return false;
1008}
1009
1010/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
1011/// to select more shifted register
1012bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
1013 SDValue &Shift) {
1014 EVT VT = N.getValueType();
1015 if (VT != MVT::i32 && VT != MVT::i64)
1016 return false;
1017
1018 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
1019 return false;
1020 SDValue LHS = N.getOperand(i: 0);
1021 if (!LHS->hasOneUse())
1022 return false;
1023
1024 unsigned LHSOpcode = LHS->getOpcode();
1025 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
1026 return false;
1027
1028 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1029 if (!ShiftAmtNode)
1030 return false;
1031
1032 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
1033 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1034 if (!RHSC)
1035 return false;
1036
1037 APInt AndMask = RHSC->getAPIntValue();
1038 unsigned LowZBits, MaskLen;
1039 if (!AndMask.isShiftedMask(MaskIdx&: LowZBits, MaskLen))
1040 return false;
1041
1042 unsigned BitWidth = N.getValueSizeInBits();
1043 SDLoc DL(LHS);
1044 uint64_t NewShiftC;
1045 unsigned NewShiftOp;
1046 if (LHSOpcode == ISD::SHL) {
1047 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
1048 if (LowZBits <= ShiftAmtC)
1049 return false;
1050
1051 NewShiftC = LowZBits - ShiftAmtC;
1052 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1053 } else {
1054 if (LowZBits == 0)
1055 return false;
1056
1057 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
1058 NewShiftC = LowZBits + ShiftAmtC;
1059 if (NewShiftC >= BitWidth)
1060 return false;
1061
1062 // SRA need all high bits
1063 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
1064 return false;
1065
1066 if (LHSOpcode == ISD::SRL)
1067 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1068 else
1069 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
1070 }
1071
1072 assert(NewShiftC < BitWidth && "Invalid shift amount");
1073 SDValue NewShiftAmt = CurDAG->getTargetConstant(Val: NewShiftC, DL, VT);
1074 // If the mask doesn't extend to the MSB, extract only the masked field.
1075 unsigned MSB = BitWidth - 1;
1076 if (LowZBits + MaskLen < BitWidth)
1077 MSB = std::min<uint64_t>(a: NewShiftC + MaskLen, b: BitWidth) - 1;
1078 SDValue MSBC = CurDAG->getTargetConstant(Val: MSB, DL, VT);
1079 Reg = SDValue(CurDAG->getMachineNode(Opcode: NewShiftOp, dl: DL, VT, Op1: LHS->getOperand(Num: 0),
1080 Op2: NewShiftAmt, Op3: MSBC),
1081 0);
1082 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LowZBits);
1083 Shift = CurDAG->getTargetConstant(Val: ShVal, DL, VT: MVT::i32);
1084 return true;
1085}
1086
1087/// getExtendTypeForNode - Translate an extend node to the corresponding
1088/// ExtendType value.
1089static AArch64_AM::ShiftExtendType
1090getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
1091 if (N.getOpcode() == ISD::SIGN_EXTEND ||
1092 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1093 EVT SrcVT;
1094 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
1095 SrcVT = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
1096 else
1097 SrcVT = N.getOperand(i: 0).getValueType();
1098
1099 if (!IsLoadStore && SrcVT == MVT::i8)
1100 return AArch64_AM::SXTB;
1101 else if (!IsLoadStore && SrcVT == MVT::i16)
1102 return AArch64_AM::SXTH;
1103 else if (SrcVT == MVT::i32)
1104 return AArch64_AM::SXTW;
1105 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1106
1107 return AArch64_AM::InvalidShiftExtend;
1108 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
1109 N.getOpcode() == ISD::ANY_EXTEND) {
1110 EVT SrcVT = N.getOperand(i: 0).getValueType();
1111 if (!IsLoadStore && SrcVT == MVT::i8)
1112 return AArch64_AM::UXTB;
1113 else if (!IsLoadStore && SrcVT == MVT::i16)
1114 return AArch64_AM::UXTH;
1115 else if (SrcVT == MVT::i32)
1116 return AArch64_AM::UXTW;
1117 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1118
1119 return AArch64_AM::InvalidShiftExtend;
1120 } else if (N.getOpcode() == ISD::AND) {
1121 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1122 if (!CSD)
1123 return AArch64_AM::InvalidShiftExtend;
1124 uint64_t AndMask = CSD->getZExtValue();
1125
1126 switch (AndMask) {
1127 default:
1128 return AArch64_AM::InvalidShiftExtend;
1129 case 0xFF:
1130 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
1131 case 0xFFFF:
1132 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
1133 case 0xFFFFFFFF:
1134 return AArch64_AM::UXTW;
1135 }
1136 }
1137
1138 return AArch64_AM::InvalidShiftExtend;
1139}
1140
1141/// Determine whether constant -V is cheaper to materialise than V.
1142bool AArch64DAGToDAGISel::isWorthNegatingImm(SDValue V) const {
1143 assert(isa<ConstantSDNode>(V) && "invalid node");
1144
1145 EVT VT = V.getValueType();
1146 assert((VT == MVT::i32 || VT == MVT::i64) && "invalid type");
1147
1148 // It's only worth negating the constant if it doesn't have other uses.
1149 if (!V.hasOneUse())
1150 return false;
1151
1152 uint64_t Imm = cast<ConstantSDNode>(Val&: V)->getZExtValue();
1153 unsigned BitSize = VT.getSizeInBits();
1154 SmallVector<AArch64_IMM::ImmInsnModel, 4> OrigCost, NewCost;
1155 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: OrigCost);
1156 AArch64_IMM::expandMOVImm(Imm: -Imm, BitSize, Insn&: NewCost);
1157 return NewCost.size() < OrigCost.size();
1158}
1159
1160/// Determine whether it is worth to fold V into an extended register of an
1161/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
1162/// instruction, and the shift should be treated as worth folding even if has
1163/// multiple uses.
1164bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1165 // Trivial if we are optimizing for code size or if there is only
1166 // one use of the value.
1167 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1168 return true;
1169
1170 // If a subtarget has a fastpath LSL we can fold a logical shift into
1171 // the add/sub and save a cycle.
1172 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1173 V.getConstantOperandVal(i: 1) <= 4 &&
1174 getExtendTypeForNode(N: V.getOperand(i: 0)) == AArch64_AM::InvalidShiftExtend)
1175 return true;
1176
1177 // It hurts otherwise, since the value will be reused.
1178 return false;
1179}
1180
1181/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1182/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1183/// instructions allow the shifted register to be rotated, but the arithmetic
1184/// instructions do not. The AllowROR parameter specifies whether ROR is
1185/// supported.
1186bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1187 SDValue &Reg, SDValue &Shift) {
1188 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1189 return true;
1190
1191 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
1192 if (ShType == AArch64_AM::InvalidShiftExtend)
1193 return false;
1194 if (!AllowROR && ShType == AArch64_AM::ROR)
1195 return false;
1196
1197 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1198 unsigned BitSize = N.getValueSizeInBits();
1199 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1200 unsigned ShVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
1201
1202 Reg = N.getOperand(i: 0);
1203 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: SDLoc(N), VT: MVT::i32);
1204 return isWorthFoldingALU(V: N, LSL: true);
1205 }
1206
1207 return false;
1208}
1209
1210/// Instructions that accept extend modifiers like UXTW expect the register
1211/// being extended to be a GPR32, but the incoming DAG might be acting on a
1212/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1213/// this is the case.
1214static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
1215 if (N.getValueType() == MVT::i32)
1216 return N;
1217
1218 SDLoc dl(N);
1219 return CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i32, Operand: N);
1220}
1221
1222// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1223template<signed Low, signed High, signed Scale>
1224bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1225 if (!isa<ConstantSDNode>(Val: N))
1226 return false;
1227
1228 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
1229 if ((MulImm % std::abs(x: Scale)) == 0) {
1230 int64_t RDVLImm = MulImm / Scale;
1231 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1232 Imm = CurDAG->getSignedTargetConstant(Val: RDVLImm, DL: SDLoc(N), VT: MVT::i32);
1233 return true;
1234 }
1235 }
1236
1237 return false;
1238}
1239
1240// Returns a suitable RDSVL multiplier from a left shift.
1241template <signed Low, signed High>
1242bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1243 if (!isa<ConstantSDNode>(Val: N))
1244 return false;
1245
1246 int64_t MulImm = 1LL << cast<ConstantSDNode>(Val&: N)->getSExtValue();
1247 if (MulImm >= Low && MulImm <= High) {
1248 Imm = CurDAG->getSignedTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
1249 return true;
1250 }
1251
1252 return false;
1253}
1254
1255/// SelectArithExtendedRegister - Select a "extended register" operand. This
1256/// operand folds in an extend followed by an optional left shift.
1257bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1258 SDValue &Shift) {
1259 unsigned ShiftVal = 0;
1260 AArch64_AM::ShiftExtendType Ext;
1261
1262 if (N.getOpcode() == ISD::SHL) {
1263 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1264 if (!CSD)
1265 return false;
1266 ShiftVal = CSD->getZExtValue();
1267 if (ShiftVal > 4)
1268 return false;
1269
1270 Ext = getExtendTypeForNode(N: N.getOperand(i: 0));
1271 if (Ext == AArch64_AM::InvalidShiftExtend)
1272 return false;
1273
1274 Reg = N.getOperand(i: 0).getOperand(i: 0);
1275 } else {
1276 Ext = getExtendTypeForNode(N);
1277 if (Ext == AArch64_AM::InvalidShiftExtend)
1278 return false;
1279
1280 // Don't match sext of vector extracts. These can use SMOV, but if we match
1281 // this as an extended register, we'll always fold the extend into an ALU op
1282 // user of the extend (which results in a UMOV).
1283 if (AArch64_AM::isSignExtendShiftType(Type: Ext)) {
1284 SDValue Op = N.getOperand(i: 0);
1285 if (Op->getOpcode() == ISD::ANY_EXTEND)
1286 Op = Op->getOperand(Num: 0);
1287 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1288 Op.getOperand(i: 0).getValueType().isFixedLengthVector())
1289 return false;
1290 }
1291
1292 Reg = N.getOperand(i: 0);
1293
1294 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1295 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1296 auto isDef32 = [](SDValue N) {
1297 unsigned Opc = N.getOpcode();
1298 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1299 Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
1300 Opc != ISD::AssertZext && Opc != ISD::AssertAlign &&
1301 Opc != ISD::FREEZE;
1302 };
1303 if (Ext == AArch64_AM::UXTW && Reg->getValueType(ResNo: 0).getSizeInBits() == 32 &&
1304 isDef32(Reg))
1305 return false;
1306 }
1307
1308 // Don't match if the sext can be folded with an asr to form an SBFX.
1309 if (Ext == AArch64_AM::SXTW && Reg.getOpcode() == ISD::SRA &&
1310 Reg.getValueType() == MVT::i32 &&
1311 isa<ConstantSDNode>(Val: Reg.getOperand(i: 1)) && Reg.hasOneUse())
1312 return false;
1313
1314 // AArch64 mandates that the RHS of the operation must use the smallest
1315 // register class that could contain the size being extended from. Thus,
1316 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1317 // there might not be an actual 32-bit value in the program. We can
1318 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1319 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1320 Reg = narrowIfNeeded(CurDAG, N: Reg);
1321 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1322 VT: MVT::i32);
1323 return isWorthFoldingALU(V: N);
1324}
1325
1326/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1327/// operand is referred by the instructions have SP operand
1328bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1329 SDValue &Shift) {
1330 unsigned ShiftVal = 0;
1331 AArch64_AM::ShiftExtendType Ext;
1332
1333 if (N.getOpcode() != ISD::SHL)
1334 return false;
1335
1336 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1337 if (!CSD)
1338 return false;
1339 ShiftVal = CSD->getZExtValue();
1340 if (ShiftVal > 4)
1341 return false;
1342
1343 Ext = AArch64_AM::UXTX;
1344 Reg = N.getOperand(i: 0);
1345 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1346 VT: MVT::i32);
1347 return isWorthFoldingALU(V: N);
1348}
1349
1350/// If there's a use of this ADDlow that's not itself a load/store then we'll
1351/// need to create a real ADD instruction from it anyway and there's no point in
1352/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1353/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1354/// leads to duplicated ADRP instructions.
1355static bool isWorthFoldingADDlow(SDValue N) {
1356 for (auto *User : N->users()) {
1357 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1358 User->getOpcode() != ISD::ATOMIC_LOAD &&
1359 User->getOpcode() != ISD::ATOMIC_STORE)
1360 return false;
1361
1362 // ldar and stlr have much more restrictive addressing modes (just a
1363 // register).
1364 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
1365 return false;
1366 }
1367
1368 return true;
1369}
1370
1371/// Check if the immediate offset is valid as a scaled immediate.
1372static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1373 unsigned Size) {
1374 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1375 Offset < (Range << Log2_32(Value: Size)))
1376 return true;
1377 return false;
1378}
1379
1380/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1381/// immediate" address. The "Size" argument is the size in bytes of the memory
1382/// reference, which determines the scale.
1383bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1384 unsigned BW, unsigned Size,
1385 SDValue &Base,
1386 SDValue &OffImm) {
1387 SDLoc dl(N);
1388 const DataLayout &DL = CurDAG->getDataLayout();
1389 const TargetLowering *TLI = getTargetLowering();
1390 if (N.getOpcode() == ISD::FrameIndex) {
1391 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1392 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1393 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1394 return true;
1395 }
1396
1397 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1398 // selected here doesn't support labels/immediates, only base+offset.
1399 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1400 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1401 if (IsSignedImm) {
1402 int64_t RHSC = RHS->getSExtValue();
1403 unsigned Scale = Log2_32(Value: Size);
1404 int64_t Range = 0x1LL << (BW - 1);
1405
1406 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1407 RHSC < (Range << Scale)) {
1408 Base = N.getOperand(i: 0);
1409 if (Base.getOpcode() == ISD::FrameIndex) {
1410 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1411 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1412 }
1413 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1414 return true;
1415 }
1416 } else {
1417 // unsigned Immediate
1418 uint64_t RHSC = RHS->getZExtValue();
1419 unsigned Scale = Log2_32(Value: Size);
1420 uint64_t Range = 0x1ULL << BW;
1421
1422 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1423 Base = N.getOperand(i: 0);
1424 if (Base.getOpcode() == ISD::FrameIndex) {
1425 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1426 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1427 }
1428 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1429 return true;
1430 }
1431 }
1432 }
1433 }
1434 // Base only. The address will be materialized into a register before
1435 // the memory is accessed.
1436 // add x0, Xbase, #offset
1437 // stp x1, x2, [x0]
1438 Base = N;
1439 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1440 return true;
1441}
1442
1443/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1444/// immediate" address. The "Size" argument is the size in bytes of the memory
1445/// reference, which determines the scale.
1446bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1447 SDValue &Base, SDValue &OffImm) {
1448 SDLoc dl(N);
1449 const DataLayout &DL = CurDAG->getDataLayout();
1450 const TargetLowering *TLI = getTargetLowering();
1451 if (N.getOpcode() == ISD::FrameIndex) {
1452 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1453 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1454 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1455 return true;
1456 }
1457
1458 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1459 GlobalAddressSDNode *GAN =
1460 dyn_cast<GlobalAddressSDNode>(Val: N.getOperand(i: 1).getNode());
1461 Base = N.getOperand(i: 0);
1462 OffImm = N.getOperand(i: 1);
1463 if (!GAN)
1464 return true;
1465
1466 if (GAN->getOffset() % Size == 0 &&
1467 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1468 return true;
1469 }
1470
1471 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1472 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1473 int64_t RHSC = (int64_t)RHS->getZExtValue();
1474 unsigned Scale = Log2_32(Value: Size);
1475 if (isValidAsScaledImmediate(Offset: RHSC, Range: 0x1000, Size)) {
1476 Base = N.getOperand(i: 0);
1477 if (Base.getOpcode() == ISD::FrameIndex) {
1478 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1479 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1480 }
1481 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1482 return true;
1483 }
1484 }
1485 }
1486
1487 // Before falling back to our general case, check if the unscaled
1488 // instructions can handle this. If so, that's preferable.
1489 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1490 return false;
1491
1492 // Base only. The address will be materialized into a register before
1493 // the memory is accessed.
1494 // add x0, Xbase, #offset
1495 // ldr x0, [x0]
1496 Base = N;
1497 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1498 return true;
1499}
1500
1501/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1502/// immediate" address. This should only match when there is an offset that
1503/// is not valid for a scaled immediate addressing mode. The "Size" argument
1504/// is the size in bytes of the memory reference, which is needed here to know
1505/// what is valid for a scaled immediate.
1506bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1507 SDValue &Base,
1508 SDValue &OffImm) {
1509 if (!CurDAG->isBaseWithConstantOffset(Op: N))
1510 return false;
1511 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1512 int64_t RHSC = RHS->getSExtValue();
1513 if (RHSC >= -256 && RHSC < 256) {
1514 Base = N.getOperand(i: 0);
1515 if (Base.getOpcode() == ISD::FrameIndex) {
1516 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1517 const TargetLowering *TLI = getTargetLowering();
1518 Base = CurDAG->getTargetFrameIndex(
1519 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1520 }
1521 OffImm = CurDAG->getTargetConstant(Val: RHSC, DL: SDLoc(N), VT: MVT::i64);
1522 return true;
1523 }
1524 }
1525 return false;
1526}
1527
1528static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
1529 SDLoc dl(N);
1530 SDValue ImpDef = SDValue(
1531 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl, VT: MVT::i64), 0);
1532 return CurDAG->getTargetInsertSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i64, Operand: ImpDef,
1533 Subreg: N);
1534}
1535
1536/// Check if the given SHL node (\p N), can be used to form an
1537/// extended register for an addressing mode.
1538bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1539 bool WantExtend, SDValue &Offset,
1540 SDValue &SignExtend) {
1541 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1542 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1543 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1544 return false;
1545
1546 SDLoc dl(N);
1547 if (WantExtend) {
1548 AArch64_AM::ShiftExtendType Ext =
1549 getExtendTypeForNode(N: N.getOperand(i: 0), IsLoadStore: true);
1550 if (Ext == AArch64_AM::InvalidShiftExtend)
1551 return false;
1552
1553 Offset = narrowIfNeeded(CurDAG, N: N.getOperand(i: 0).getOperand(i: 0));
1554 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1555 VT: MVT::i32);
1556 } else {
1557 Offset = N.getOperand(i: 0);
1558 SignExtend = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1559 }
1560
1561 unsigned LegalShiftVal = Log2_32(Value: Size);
1562 unsigned ShiftVal = CSD->getZExtValue();
1563
1564 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1565 return false;
1566
1567 return isWorthFoldingAddr(V: N, Size);
1568}
1569
1570bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1571 SDValue &Base, SDValue &Offset,
1572 SDValue &SignExtend,
1573 SDValue &DoShift) {
1574 if (N.getOpcode() != ISD::ADD)
1575 return false;
1576 SDValue LHS = N.getOperand(i: 0);
1577 SDValue RHS = N.getOperand(i: 1);
1578 SDLoc dl(N);
1579
1580 // We don't want to match immediate adds here, because they are better lowered
1581 // to the register-immediate addressing modes.
1582 if (isa<ConstantSDNode>(Val: LHS) || isa<ConstantSDNode>(Val: RHS))
1583 return false;
1584
1585 // Check if this particular node is reused in any non-memory related
1586 // operation. If yes, do not try to fold this node into the address
1587 // computation, since the computation will be kept.
1588 const SDNode *Node = N.getNode();
1589 for (SDNode *UI : Node->users()) {
1590 if (!isMemOpOrPrefetch(N: UI))
1591 return false;
1592 }
1593
1594 // Remember if it is worth folding N when it produces extended register.
1595 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1596
1597 // Try to match a shifted extend on the RHS.
1598 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1599 SelectExtendedSHL(N: RHS, Size, WantExtend: true, Offset, SignExtend)) {
1600 Base = LHS;
1601 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1602 return true;
1603 }
1604
1605 // Try to match a shifted extend on the LHS.
1606 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1607 SelectExtendedSHL(N: LHS, Size, WantExtend: true, Offset, SignExtend)) {
1608 Base = RHS;
1609 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1610 return true;
1611 }
1612
1613 // There was no shift, whatever else we find.
1614 DoShift = CurDAG->getTargetConstant(Val: false, DL: dl, VT: MVT::i32);
1615
1616 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1617 // Try to match an unshifted extend on the LHS.
1618 if (IsExtendedRegisterWorthFolding &&
1619 (Ext = getExtendTypeForNode(N: LHS, IsLoadStore: true)) !=
1620 AArch64_AM::InvalidShiftExtend) {
1621 Base = RHS;
1622 Offset = narrowIfNeeded(CurDAG, N: LHS.getOperand(i: 0));
1623 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1624 VT: MVT::i32);
1625 if (isWorthFoldingAddr(V: LHS, Size))
1626 return true;
1627 }
1628
1629 // Try to match an unshifted extend on the RHS.
1630 if (IsExtendedRegisterWorthFolding &&
1631 (Ext = getExtendTypeForNode(N: RHS, IsLoadStore: true)) !=
1632 AArch64_AM::InvalidShiftExtend) {
1633 Base = LHS;
1634 Offset = narrowIfNeeded(CurDAG, N: RHS.getOperand(i: 0));
1635 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1636 VT: MVT::i32);
1637 if (isWorthFoldingAddr(V: RHS, Size))
1638 return true;
1639 }
1640
1641 return false;
1642}
1643
1644// Check if the given immediate is preferred by ADD. If an immediate can be
1645// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1646// encoded by one MOVZ, return true.
1647static bool isPreferredADD(int64_t ImmOff) {
1648 // Constant in [0x0, 0xfff] can be encoded in ADD.
1649 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1650 return true;
1651 // Check if it can be encoded in an "ADD LSL #12".
1652 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1653 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1654 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1655 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1656 return false;
1657}
1658
1659bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1660 SDValue &Base, SDValue &Offset,
1661 SDValue &SignExtend,
1662 SDValue &DoShift) {
1663 if (N.getOpcode() != ISD::ADD)
1664 return false;
1665 SDValue LHS = N.getOperand(i: 0);
1666 SDValue RHS = N.getOperand(i: 1);
1667 SDLoc DL(N);
1668
1669 // Check if this particular node is reused in any non-memory related
1670 // operation. If yes, do not try to fold this node into the address
1671 // computation, since the computation will be kept.
1672 const SDNode *Node = N.getNode();
1673 for (SDNode *UI : Node->users()) {
1674 if (!isMemOpOrPrefetch(N: UI))
1675 return false;
1676 }
1677
1678 // Watch out if RHS is a wide immediate, it can not be selected into
1679 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1680 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1681 // instructions like:
1682 // MOV X0, WideImmediate
1683 // ADD X1, BaseReg, X0
1684 // LDR X2, [X1, 0]
1685 // For such situation, using [BaseReg, XReg] addressing mode can save one
1686 // ADD/SUB:
1687 // MOV X0, WideImmediate
1688 // LDR X2, [BaseReg, X0]
1689 if (isa<ConstantSDNode>(Val: RHS)) {
1690 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1691 // Skip the immediate can be selected by load/store addressing mode.
1692 // Also skip the immediate can be encoded by a single ADD (SUB is also
1693 // checked by using -ImmOff).
1694 if (isValidAsScaledImmediate(Offset: ImmOff, Range: 0x1000, Size) ||
1695 isPreferredADD(ImmOff) || isPreferredADD(ImmOff: -ImmOff))
1696 return false;
1697
1698 SDValue Ops[] = { RHS };
1699 SDNode *MOVI =
1700 CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
1701 SDValue MOVIV = SDValue(MOVI, 0);
1702 // This ADD of two X register will be selected into [Reg+Reg] mode.
1703 N = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: LHS, N2: MOVIV);
1704 }
1705
1706 // Remember if it is worth folding N when it produces extended register.
1707 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1708
1709 // Try to match a shifted extend on the RHS.
1710 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1711 SelectExtendedSHL(N: RHS, Size, WantExtend: false, Offset, SignExtend)) {
1712 Base = LHS;
1713 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1714 return true;
1715 }
1716
1717 // Try to match a shifted extend on the LHS.
1718 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1719 SelectExtendedSHL(N: LHS, Size, WantExtend: false, Offset, SignExtend)) {
1720 Base = RHS;
1721 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1722 return true;
1723 }
1724
1725 // Match any non-shifted, non-extend, non-immediate add expression.
1726 Base = LHS;
1727 Offset = RHS;
1728 SignExtend = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1729 DoShift = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1730 // Reg1 + Reg2 is free: no check needed.
1731 return true;
1732}
1733
1734SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1735 static const unsigned RegClassIDs[] = {
1736 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1737 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1738 AArch64::dsub2, AArch64::dsub3};
1739
1740 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1741}
1742
1743SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1744 static const unsigned RegClassIDs[] = {
1745 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1746 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1747 AArch64::qsub2, AArch64::qsub3};
1748
1749 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1750}
1751
1752SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1753 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1754 AArch64::ZPR3RegClassID,
1755 AArch64::ZPR4RegClassID};
1756 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1757 AArch64::zsub2, AArch64::zsub3};
1758
1759 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1760}
1761
1762SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1763 assert(Regs.size() == 2 || Regs.size() == 4);
1764
1765 // The createTuple interface requires 3 RegClassIDs for each possible
1766 // tuple type even though we only have them for ZPR2 and ZPR4.
1767 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1768 AArch64::ZPR4Mul4RegClassID};
1769 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1770 AArch64::zsub2, AArch64::zsub3};
1771 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1772}
1773
1774SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1775 const unsigned RegClassIDs[],
1776 const unsigned SubRegs[]) {
1777 // There's no special register-class for a vector-list of 1 element: it's just
1778 // a vector.
1779 if (Regs.size() == 1)
1780 return Regs[0];
1781
1782 assert(Regs.size() >= 2 && Regs.size() <= 4);
1783
1784 SDLoc DL(Regs[0]);
1785
1786 SmallVector<SDValue, 4> Ops;
1787
1788 // First operand of REG_SEQUENCE is the desired RegClass.
1789 Ops.push_back(
1790 Elt: CurDAG->getTargetConstant(Val: RegClassIDs[Regs.size() - 2], DL, VT: MVT::i32));
1791
1792 // Then we get pairs of source & subregister-position for the components.
1793 for (unsigned i = 0; i < Regs.size(); ++i) {
1794 Ops.push_back(Elt: Regs[i]);
1795 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: SubRegs[i], DL, VT: MVT::i32));
1796 }
1797
1798 SDNode *N =
1799 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped, Ops);
1800 return SDValue(N, 0);
1801}
1802
1803void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1804 bool isExt) {
1805 SDLoc dl(N);
1806 EVT VT = N->getValueType(ResNo: 0);
1807
1808 unsigned ExtOff = isExt;
1809
1810 // Form a REG_SEQUENCE to force register allocation.
1811 unsigned Vec0Off = ExtOff + 1;
1812 SmallVector<SDValue, 4> Regs(N->ops().slice(N: Vec0Off, M: NumVecs));
1813 SDValue RegSeq = createQTuple(Regs);
1814
1815 SmallVector<SDValue, 6> Ops;
1816 if (isExt)
1817 Ops.push_back(Elt: N->getOperand(Num: 1));
1818 Ops.push_back(Elt: RegSeq);
1819 Ops.push_back(Elt: N->getOperand(Num: NumVecs + ExtOff + 1));
1820 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT, Ops));
1821}
1822
1823static std::tuple<SDValue, SDValue>
1824extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1825 SDLoc DL(Disc);
1826 SDValue AddrDisc;
1827 SDValue ConstDisc;
1828
1829 // If this is a blend, remember the constant and address discriminators.
1830 // Otherwise, it's either a constant discriminator, or a non-blended
1831 // address discriminator.
1832 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1833 Disc->getConstantOperandVal(Num: 0) == Intrinsic::ptrauth_blend) {
1834 AddrDisc = Disc->getOperand(Num: 1);
1835 ConstDisc = Disc->getOperand(Num: 2);
1836 } else {
1837 ConstDisc = Disc;
1838 }
1839
1840 // If the constant discriminator (either the blend RHS, or the entire
1841 // discriminator value) isn't a 16-bit constant, bail out, and let the
1842 // discriminator be computed separately.
1843 auto *ConstDiscN = dyn_cast<ConstantSDNode>(Val&: ConstDisc);
1844 if (!ConstDiscN || !isUInt<16>(x: ConstDiscN->getZExtValue()))
1845 return std::make_tuple(args: DAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), args&: Disc);
1846
1847 // If there's no address discriminator, use XZR directly.
1848 if (!AddrDisc)
1849 AddrDisc = DAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64);
1850
1851 return std::make_tuple(
1852 args: DAG->getTargetConstant(Val: ConstDiscN->getZExtValue(), DL, VT: MVT::i64),
1853 args&: AddrDisc);
1854}
1855
1856void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1857 SDLoc DL(N);
1858 // IntrinsicID is operand #0
1859 SDValue Val = N->getOperand(Num: 1);
1860 SDValue AUTKey = N->getOperand(Num: 2);
1861 SDValue AUTDisc = N->getOperand(Num: 3);
1862
1863 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1864 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1865
1866 SDValue AUTAddrDisc, AUTConstDisc;
1867 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1868 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1869
1870 if (!Subtarget->isX16X17Safer()) {
1871 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1872 // Copy deactivation symbol if present.
1873 if (N->getNumOperands() > 4)
1874 Ops.push_back(x: N->getOperand(Num: 4));
1875
1876 SDNode *AUT =
1877 CurDAG->getMachineNode(Opcode: AArch64::AUTxMxN, dl: DL, VT1: MVT::i64, VT2: MVT::i64, Ops);
1878 ReplaceNode(F: N, T: AUT);
1879 } else {
1880 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1881 Reg: AArch64::X16, N: Val, Glue: SDValue());
1882 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(R: 1)};
1883
1884 SDNode *AUT = CurDAG->getMachineNode(Opcode: AArch64::AUTx16x17, dl: DL, VT: MVT::i64, Ops);
1885 ReplaceNode(F: N, T: AUT);
1886 }
1887}
1888
1889void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1890 SDLoc DL(N);
1891 // IntrinsicID is operand #0, if W_CHAIN it is #1
1892 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1893 SDValue Val = N->getOperand(Num: OffsetBase + 1);
1894 SDValue AUTKey = N->getOperand(Num: OffsetBase + 2);
1895 SDValue AUTDisc = N->getOperand(Num: OffsetBase + 3);
1896 SDValue PACKey = N->getOperand(Num: OffsetBase + 4);
1897 SDValue PACDisc = N->getOperand(Num: OffsetBase + 5);
1898 uint32_t IntNum = N->getConstantOperandVal(Num: OffsetBase + 0);
1899 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
1900
1901 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1902 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1903
1904 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1905 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1906
1907 SDValue AUTAddrDisc, AUTConstDisc;
1908 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1909 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1910
1911 SDValue PACAddrDisc, PACConstDisc;
1912 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1913 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1914
1915 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1916 Reg: AArch64::X16, N: Val, Glue: SDValue());
1917
1918 if (HasLoad) {
1919 SDValue Addend = N->getOperand(Num: OffsetBase + 6);
1920 SDValue IncomingChain = N->getOperand(Num: 0);
1921 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1922 PACKey, PACConstDisc, PACAddrDisc,
1923 Addend, IncomingChain, X16Copy.getValue(R: 1)};
1924
1925 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTRELLOADPAC, dl: DL,
1926 VT1: MVT::i64, VT2: MVT::Other, Ops);
1927 ReplaceNode(F: N, T: AUTRELLOADPAC);
1928 } else {
1929 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1930 PACConstDisc, PACAddrDisc, X16Copy.getValue(R: 1)};
1931
1932 SDNode *AUTPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTPAC, dl: DL, VT: MVT::i64, Ops);
1933 ReplaceNode(F: N, T: AUTPAC);
1934 }
1935}
1936
1937void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
1938 SDLoc DL(N);
1939 SDValue Val = N->getOperand(Num: 1);
1940 SDValue AUTKey = N->getOperand(Num: 2);
1941 SDValue AUTDisc = N->getOperand(Num: 3);
1942 SDValue AUTPC = N->getOperand(Num: 4);
1943 SDValue PACKey = N->getOperand(Num: 5);
1944 SDValue PACDisc = N->getOperand(Num: 6);
1945
1946 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1947 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1948
1949 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1950 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1951
1952 SDValue PACAddrDisc, PACConstDisc;
1953 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1954 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1955
1956 SDValue X17Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1957 Reg: AArch64::X17, N: Val, Glue: SDValue());
1958 SDValue X16Copy = CurDAG->getCopyToReg(
1959 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X16, N: AUTDisc, Glue: X17Copy.getValue(R: 1));
1960 SDValue X15Copy = CurDAG->getCopyToReg(
1961 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X15, N: AUTPC, Glue: X16Copy.getValue(R: 1));
1962
1963 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
1964 X15Copy.getValue(R: 1)};
1965 SDNode *AUTPCPAC =
1966 CurDAG->getMachineNode(Opcode: AArch64::AUTPCPAC, dl: DL, VT: MVT::i64, Ops);
1967 ReplaceNode(F: N, T: AUTPCPAC);
1968}
1969
1970bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1971 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1972 if (LD->isUnindexed())
1973 return false;
1974 EVT VT = LD->getMemoryVT();
1975 EVT DstVT = N->getValueType(ResNo: 0);
1976 ISD::MemIndexedMode AM = LD->getAddressingMode();
1977 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1978 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(Val: LD->getOffset());
1979 int OffsetVal = (int)OffsetOp->getZExtValue();
1980
1981 // We're not doing validity checking here. That was done when checking
1982 // if we should mark the load as indexed or not. We're just selecting
1983 // the right instruction.
1984 unsigned Opcode = 0;
1985
1986 ISD::LoadExtType ExtType = LD->getExtensionType();
1987 bool InsertTo64 = false;
1988 bool UseLd1 =
1989 (VT.is64BitVector() || VT.is128BitVector()) &&
1990 (!Subtarget->isLittleEndian() || (Subtarget->requiresStrictAlign() &&
1991 LD->getAlign() < VT.getStoreSize()));
1992 if (VT == MVT::i64)
1993 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1994 else if (VT == MVT::i32) {
1995 if (ExtType == ISD::NON_EXTLOAD)
1996 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1997 else if (ExtType == ISD::SEXTLOAD)
1998 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1999 else {
2000 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2001 InsertTo64 = true;
2002 // The result of the load is only i32. It's the subreg_to_reg that makes
2003 // it into an i64.
2004 DstVT = MVT::i32;
2005 }
2006 } else if (VT == MVT::i16) {
2007 if (ExtType == ISD::SEXTLOAD) {
2008 if (DstVT == MVT::i64)
2009 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
2010 else
2011 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
2012 } else {
2013 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
2014 InsertTo64 = DstVT == MVT::i64;
2015 // The result of the load is only i32. It's the subreg_to_reg that makes
2016 // it into an i64.
2017 DstVT = MVT::i32;
2018 }
2019 } else if (VT == MVT::i8) {
2020 if (ExtType == ISD::SEXTLOAD) {
2021 if (DstVT == MVT::i64)
2022 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
2023 else
2024 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
2025 } else {
2026 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
2027 InsertTo64 = DstVT == MVT::i64;
2028 // The result of the load is only i32. It's the subreg_to_reg that makes
2029 // it into an i64.
2030 DstVT = MVT::i32;
2031 }
2032 } else if (VT == MVT::f16) {
2033 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2034 } else if (VT == MVT::bf16) {
2035 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2036 } else if (VT == MVT::f32) {
2037 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
2038 } else if (VT == MVT::f64 || (VT.is64BitVector() && !UseLd1)) {
2039 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
2040 } else if (VT.is128BitVector() && !UseLd1) {
2041 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
2042 } else if (VT.is64BitVector() && UseLd1) {
2043 if (IsPre || OffsetVal != 8)
2044 return false;
2045 switch (VT.getScalarSizeInBits()) {
2046 case 8:
2047 Opcode = AArch64::LD1Onev8b_POST;
2048 break;
2049 case 16:
2050 Opcode = AArch64::LD1Onev4h_POST;
2051 break;
2052 case 32:
2053 Opcode = AArch64::LD1Onev2s_POST;
2054 break;
2055 case 64:
2056 Opcode = AArch64::LD1Onev1d_POST;
2057 break;
2058 default:
2059 llvm_unreachable("Expected vector element to be a power of 2");
2060 }
2061 } else if (VT.is128BitVector() && UseLd1) {
2062 if (IsPre || OffsetVal != 16)
2063 return false;
2064 switch (VT.getScalarSizeInBits()) {
2065 case 8:
2066 Opcode = AArch64::LD1Onev16b_POST;
2067 break;
2068 case 16:
2069 Opcode = AArch64::LD1Onev8h_POST;
2070 break;
2071 case 32:
2072 Opcode = AArch64::LD1Onev4s_POST;
2073 break;
2074 case 64:
2075 Opcode = AArch64::LD1Onev2d_POST;
2076 break;
2077 default:
2078 llvm_unreachable("Expected vector element to be a power of 2");
2079 }
2080 } else
2081 return false;
2082 SDValue Chain = LD->getChain();
2083 SDValue Base = LD->getBasePtr();
2084 SDLoc dl(N);
2085 // LD1 encodes an immediate offset by using XZR as the offset register.
2086 SDValue Offset = UseLd1 ? CurDAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64)
2087 : CurDAG->getTargetConstant(Val: OffsetVal, DL: dl, VT: MVT::i64);
2088 SDValue Ops[] = { Base, Offset, Chain };
2089 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, VT1: MVT::i64, VT2: DstVT,
2090 VT3: MVT::Other, Ops);
2091
2092 // Transfer memoperands.
2093 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
2094 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Res), NewMemRefs: {MemOp});
2095
2096 // Either way, we're replacing the node, so tell the caller that.
2097 SDValue LoadedVal = SDValue(Res, 1);
2098 if (InsertTo64) {
2099 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL: dl, VT: MVT::i32);
2100 LoadedVal = SDValue(CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl,
2101 VT: MVT::i64, Op1: LoadedVal, Op2: SubReg),
2102 0);
2103 }
2104
2105 ReplaceUses(F: SDValue(N, 0), T: LoadedVal);
2106 ReplaceUses(F: SDValue(N, 1), T: SDValue(Res, 0));
2107 ReplaceUses(F: SDValue(N, 2), T: SDValue(Res, 2));
2108 CurDAG->RemoveDeadNode(N);
2109 return true;
2110}
2111
2112void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
2113 unsigned SubRegIdx) {
2114 SDLoc dl(N);
2115 EVT VT = N->getValueType(ResNo: 0);
2116 SDValue Chain = N->getOperand(Num: 0);
2117
2118 SDValue Ops[] = {N->getOperand(Num: 2), // Mem operand;
2119 Chain};
2120
2121 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2122
2123 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2124 SDValue SuperReg = SDValue(Ld, 0);
2125 for (unsigned i = 0; i < NumVecs; ++i)
2126 ReplaceUses(F: SDValue(N, i),
2127 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2128
2129 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2130
2131 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
2132 // because it's too simple to have needed special treatment during lowering.
2133 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
2134 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2135 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2136 }
2137
2138 CurDAG->RemoveDeadNode(N);
2139}
2140
2141void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
2142 unsigned Opc, unsigned SubRegIdx) {
2143 SDLoc dl(N);
2144 EVT VT = N->getValueType(ResNo: 0);
2145 SDValue Chain = N->getOperand(Num: 0);
2146
2147 SDValue Ops[] = {N->getOperand(Num: 1), // Mem operand
2148 N->getOperand(Num: 2), // Incremental
2149 Chain};
2150
2151 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2152 MVT::Untyped, MVT::Other};
2153
2154 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2155
2156 // Update uses of write back register
2157 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2158
2159 // Update uses of vector list
2160 SDValue SuperReg = SDValue(Ld, 1);
2161 if (NumVecs == 1)
2162 ReplaceUses(F: SDValue(N, 0), T: SuperReg);
2163 else
2164 for (unsigned i = 0; i < NumVecs; ++i)
2165 ReplaceUses(F: SDValue(N, i),
2166 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2167
2168 // Transfer memoperands.
2169 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2170 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2171
2172 // Update the chain
2173 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2174 CurDAG->RemoveDeadNode(N);
2175}
2176
2177/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2178/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2179/// new Base and an SDValue representing the new offset.
2180std::tuple<unsigned, SDValue, SDValue>
2181AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2182 unsigned Opc_ri,
2183 const SDValue &OldBase,
2184 const SDValue &OldOffset,
2185 unsigned Scale) {
2186 SDValue NewBase = OldBase;
2187 SDValue NewOffset = OldOffset;
2188 // Detect a possible Reg+Imm addressing mode.
2189 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2190 Root: N, N: OldBase, Base&: NewBase, OffImm&: NewOffset);
2191
2192 // Detect a possible reg+reg addressing mode, but only if we haven't already
2193 // detected a Reg+Imm one.
2194 const bool IsRegReg =
2195 !IsRegImm && SelectSVERegRegAddrMode(N: OldBase, Scale, Base&: NewBase, Offset&: NewOffset);
2196
2197 // Select the instruction.
2198 return std::make_tuple(args&: IsRegReg ? Opc_rr : Opc_ri, args&: NewBase, args&: NewOffset);
2199}
2200
2201enum class SelectTypeKind {
2202 Int1 = 0,
2203 Int = 1,
2204 FP = 2,
2205 AnyType = 3,
2206};
2207
2208/// This function selects an opcode from a list of opcodes, which is
2209/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2210/// element types, in this order.
2211template <SelectTypeKind Kind>
2212static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2213 // Only match scalable vector VTs
2214 if (!VT.isScalableVector())
2215 return 0;
2216
2217 EVT EltVT = VT.getVectorElementType();
2218 unsigned Key = VT.getVectorMinNumElements();
2219 switch (Kind) {
2220 case SelectTypeKind::AnyType:
2221 break;
2222 case SelectTypeKind::Int:
2223 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2224 EltVT != MVT::i64)
2225 return 0;
2226 break;
2227 case SelectTypeKind::Int1:
2228 if (EltVT != MVT::i1)
2229 return 0;
2230 break;
2231 case SelectTypeKind::FP:
2232 if (EltVT == MVT::bf16)
2233 Key = 16;
2234 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2235 EltVT != MVT::f64)
2236 return 0;
2237 break;
2238 }
2239
2240 unsigned Offset;
2241 switch (Key) {
2242 case 16: // 8-bit or bf16
2243 Offset = 0;
2244 break;
2245 case 8: // 16-bit
2246 Offset = 1;
2247 break;
2248 case 4: // 32-bit
2249 Offset = 2;
2250 break;
2251 case 2: // 64-bit
2252 Offset = 3;
2253 break;
2254 default:
2255 return 0;
2256 }
2257
2258 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2259}
2260
2261// This function is almost identical to SelectWhilePair, but has an
2262// extra check on the range of the immediate operand.
2263// TODO: Merge these two functions together at some point?
2264void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2265 // Immediate can be either 0 or 1.
2266 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
2267 if (Imm->getZExtValue() > 1)
2268 return;
2269
2270 SDLoc DL(N);
2271 EVT VT = N->getValueType(ResNo: 0);
2272 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2273 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2274 SDValue SuperReg = SDValue(WhilePair, 0);
2275
2276 for (unsigned I = 0; I < 2; ++I)
2277 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2278 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2279
2280 CurDAG->RemoveDeadNode(N);
2281}
2282
2283void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2284 SDLoc DL(N);
2285 EVT VT = N->getValueType(ResNo: 0);
2286
2287 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2288
2289 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2290 SDValue SuperReg = SDValue(WhilePair, 0);
2291
2292 for (unsigned I = 0; I < 2; ++I)
2293 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2294 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2295
2296 CurDAG->RemoveDeadNode(N);
2297}
2298
2299void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2300 unsigned Opcode) {
2301 EVT VT = N->getValueType(ResNo: 0);
2302 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2303 SDValue Ops = createZTuple(Regs);
2304 SDLoc DL(N);
2305 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Op1: Ops);
2306 SDValue SuperReg = SDValue(Intrinsic, 0);
2307 for (unsigned i = 0; i < NumVecs; ++i)
2308 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2309 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2310
2311 CurDAG->RemoveDeadNode(N);
2312}
2313
2314void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2315 unsigned Opcode) {
2316 SDLoc DL(N);
2317 EVT VT = N->getValueType(ResNo: 0);
2318 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2319 Ops.push_back(/*Chain*/ Elt: N->getOperand(Num: 0));
2320
2321 SDNode *Instruction =
2322 CurDAG->getMachineNode(Opcode, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2323 SDValue SuperReg = SDValue(Instruction, 0);
2324
2325 for (unsigned i = 0; i < NumVecs; ++i)
2326 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2327 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2328
2329 // Copy chain
2330 unsigned ChainIdx = NumVecs;
2331 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Instruction, 1));
2332 CurDAG->RemoveDeadNode(N);
2333}
2334
2335void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2336 unsigned NumVecs,
2337 bool IsZmMulti,
2338 unsigned Opcode,
2339 bool HasPred) {
2340 assert(Opcode != 0 && "Unexpected opcode");
2341
2342 SDLoc DL(N);
2343 EVT VT = N->getValueType(ResNo: 0);
2344 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2345 SmallVector<SDValue, 4> Ops;
2346
2347 auto GetMultiVecOperand = [&]() {
2348 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2349 OpsIter += NumVecs;
2350 return createZMulTuple(Regs);
2351 };
2352
2353 if (HasPred)
2354 Ops.push_back(Elt: *OpsIter++);
2355
2356 Ops.push_back(Elt: GetMultiVecOperand());
2357 if (IsZmMulti)
2358 Ops.push_back(Elt: GetMultiVecOperand());
2359 else
2360 Ops.push_back(Elt: *OpsIter++);
2361
2362 // Append any remaining operands.
2363 Ops.append(in_start: OpsIter, in_end: N->op_end());
2364 SDNode *Intrinsic;
2365 Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Ops);
2366 SDValue SuperReg = SDValue(Intrinsic, 0);
2367 for (unsigned i = 0; i < NumVecs; ++i)
2368 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2369 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2370
2371 CurDAG->RemoveDeadNode(N);
2372}
2373
2374void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2375 unsigned Scale, unsigned Opc_ri,
2376 unsigned Opc_rr, bool IsIntr) {
2377 assert(Scale < 5 && "Invalid scaling value.");
2378 SDLoc DL(N);
2379 EVT VT = N->getValueType(ResNo: 0);
2380 SDValue Chain = N->getOperand(Num: 0);
2381
2382 // Optimize addressing mode.
2383 SDValue Base, Offset;
2384 unsigned Opc;
2385 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2386 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: IsIntr ? 3 : 2),
2387 OldOffset: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), Scale);
2388
2389 SDValue Ops[] = {N->getOperand(Num: IsIntr ? 2 : 1), // Predicate
2390 Base, // Memory operand
2391 Offset, Chain};
2392
2393 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2394
2395 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2396 SDValue SuperReg = SDValue(Load, 0);
2397 for (unsigned i = 0; i < NumVecs; ++i)
2398 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2399 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2400
2401 // Copy chain
2402 unsigned ChainIdx = NumVecs;
2403 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2404 CurDAG->RemoveDeadNode(N);
2405}
2406
2407void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2408 unsigned NumVecs,
2409 unsigned Scale,
2410 unsigned Opc_ri,
2411 unsigned Opc_rr) {
2412 assert(Scale < 4 && "Invalid scaling value.");
2413 SDLoc DL(N);
2414 EVT VT = N->getValueType(ResNo: 0);
2415 SDValue Chain = N->getOperand(Num: 0);
2416
2417 SDValue PNg = N->getOperand(Num: 2);
2418 SDValue Base = N->getOperand(Num: 3);
2419 SDValue Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
2420 unsigned Opc;
2421 std::tie(args&: Opc, args&: Base, args&: Offset) =
2422 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, OldBase: Base, OldOffset: Offset, Scale);
2423
2424 SDValue Ops[] = {PNg, // Predicate-as-counter
2425 Base, // Memory operand
2426 Offset, Chain};
2427
2428 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2429
2430 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2431 SDValue SuperReg = SDValue(Load, 0);
2432 for (unsigned i = 0; i < NumVecs; ++i)
2433 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2434 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2435
2436 // Copy chain
2437 unsigned ChainIdx = NumVecs;
2438 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2439 CurDAG->RemoveDeadNode(N);
2440}
2441
2442void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2443 unsigned Opcode) {
2444 if (N->getValueType(ResNo: 0) != MVT::nxv4f32)
2445 return;
2446 SelectUnaryMultiIntrinsic(N, NumOutVecs: NumVecs, IsTupleInput: true, Opc: Opcode);
2447}
2448
2449void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2450 unsigned NumOutVecs,
2451 unsigned Opc,
2452 uint32_t MaxImm) {
2453 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 4)))
2454 if (Imm->getZExtValue() > MaxImm)
2455 return;
2456
2457 SDValue ZtValue;
2458 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2459 return;
2460
2461 SDValue Chain = Node->getOperand(Num: 0);
2462 SDValue Ops[] = {ZtValue, Node->getOperand(Num: 3), Node->getOperand(Num: 4), Chain};
2463 SDLoc DL(Node);
2464 EVT VT = Node->getValueType(ResNo: 0);
2465
2466 SDNode *Instruction =
2467 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2468 SDValue SuperReg = SDValue(Instruction, 0);
2469
2470 for (unsigned I = 0; I < NumOutVecs; ++I)
2471 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2472 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2473
2474 // Copy chain
2475 unsigned ChainIdx = NumOutVecs;
2476 ReplaceUses(F: SDValue(Node, ChainIdx), T: SDValue(Instruction, 1));
2477 CurDAG->RemoveDeadNode(N: Node);
2478}
2479
2480void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2481 unsigned NumIndexVecs) {
2482 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2483 "unexpected number of index vectors");
2484
2485 constexpr unsigned FirstIndexOp = 3;
2486 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2487 auto *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
2488 if (!Imm || Imm->getZExtValue() > 1)
2489 return;
2490
2491 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2492 // ACLE form provides three index vectors, so the lane selects which adjacent
2493 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2494 unsigned Lane = Imm->getZExtValue();
2495 unsigned IndexOp = FirstIndexOp;
2496 if (NumIndexVecs == 3)
2497 IndexOp += Lane;
2498
2499 SDValue TableTuple = createZTuple(Regs: {Node->getOperand(Num: 1), Node->getOperand(Num: 2)});
2500 SDValue IndexTuple =
2501 createZTuple(Regs: {Node->getOperand(Num: IndexOp), Node->getOperand(Num: IndexOp + 1)});
2502 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(Num: ImmOp)};
2503
2504 SDLoc DL(Node);
2505 EVT VT = Node->getValueType(ResNo: 0);
2506 SDNode *Instruction =
2507 CurDAG->getMachineNode(Opcode: AArch64::LUTI6_4Z2Z2ZI, dl: DL, VT: MVT::Untyped, Ops);
2508 SDValue SuperReg = SDValue(Instruction, 0);
2509
2510 for (unsigned I = 0; I < 4; ++I)
2511 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2512 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2513
2514 CurDAG->RemoveDeadNode(N: Node);
2515}
2516
2517void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2518 unsigned NumOutVecs,
2519 unsigned Opc,
2520 unsigned NumInVecs) {
2521 assert((NumInVecs == 2 || NumInVecs == 3) &&
2522 "unexpected number of input vectors");
2523
2524 SDValue ZtValue;
2525 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2526 return;
2527
2528 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 3, M: NumInVecs));
2529 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2530 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(Num: 0)};
2531
2532 SDLoc DL(Node);
2533 EVT VT = Node->getValueType(ResNo: 0);
2534
2535 SDNode *Instruction =
2536 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2537 SDValue SuperReg = SDValue(Instruction, 0);
2538
2539 for (unsigned I = 0; I < NumOutVecs; ++I)
2540 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2541 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2542
2543 ReplaceUses(F: SDValue(Node, NumOutVecs), T: SDValue(Instruction, 1));
2544 CurDAG->RemoveDeadNode(N: Node);
2545}
2546
2547void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2548 unsigned Op) {
2549 SDLoc DL(N);
2550 EVT VT = N->getValueType(ResNo: 0);
2551
2552 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2553 SDValue Zd = createZMulTuple(Regs);
2554 SDValue Zn = N->getOperand(Num: 1 + NumVecs);
2555 SDValue Zm = N->getOperand(Num: 2 + NumVecs);
2556
2557 SDValue Ops[] = {Zd, Zn, Zm};
2558
2559 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT: MVT::Untyped, Ops);
2560 SDValue SuperReg = SDValue(Intrinsic, 0);
2561 for (unsigned i = 0; i < NumVecs; ++i)
2562 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2563 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2564
2565 CurDAG->RemoveDeadNode(N);
2566}
2567
2568bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2569 switch (BaseReg) {
2570 default:
2571 return false;
2572 case AArch64::ZA:
2573 case AArch64::ZAB0:
2574 if (TileNum == 0)
2575 break;
2576 return false;
2577 case AArch64::ZAH0:
2578 if (TileNum <= 1)
2579 break;
2580 return false;
2581 case AArch64::ZAS0:
2582 if (TileNum <= 3)
2583 break;
2584 return false;
2585 case AArch64::ZAD0:
2586 if (TileNum <= 7)
2587 break;
2588 return false;
2589 }
2590
2591 BaseReg += TileNum;
2592 return true;
2593}
2594
2595template <unsigned MaxIdx, unsigned Scale>
2596void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2597 unsigned BaseReg, unsigned Op) {
2598 unsigned TileNum = 0;
2599 if (BaseReg != AArch64::ZA)
2600 TileNum = N->getConstantOperandVal(Num: 2);
2601
2602 if (!SelectSMETile(BaseReg, TileNum))
2603 return;
2604
2605 SDValue SliceBase, Base, Offset;
2606 if (BaseReg == AArch64::ZA)
2607 SliceBase = N->getOperand(Num: 2);
2608 else
2609 SliceBase = N->getOperand(Num: 3);
2610
2611 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2612 return;
2613
2614 SDLoc DL(N);
2615 SDValue SubReg = CurDAG->getRegister(Reg: BaseReg, VT: MVT::Other);
2616 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(Num: 0)};
2617 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2618
2619 EVT VT = N->getValueType(ResNo: 0);
2620 for (unsigned I = 0; I < NumVecs; ++I)
2621 ReplaceUses(F: SDValue(N, I),
2622 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2623 Operand: SDValue(Mov, 0)));
2624 // Copy chain
2625 unsigned ChainIdx = NumVecs;
2626 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2627 CurDAG->RemoveDeadNode(N);
2628}
2629
2630void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2631 unsigned Op, unsigned MaxIdx,
2632 unsigned Scale, unsigned BaseReg) {
2633 // Slice can be in different positions
2634 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2635 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2636 SDValue SliceBase = N->getOperand(Num: 2);
2637 if (BaseReg != AArch64::ZA)
2638 SliceBase = N->getOperand(Num: 3);
2639
2640 SDValue Base, Offset;
2641 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2642 return;
2643 // The correct Za tile number is computed in Machine Instruction
2644 // See EmitZAInstr
2645 // DAG cannot select Za tile as an output register with ZReg
2646 SDLoc DL(N);
2647 SmallVector<SDValue, 6> Ops;
2648 if (BaseReg != AArch64::ZA )
2649 Ops.push_back(Elt: N->getOperand(Num: 2));
2650 Ops.push_back(Elt: Base);
2651 Ops.push_back(Elt: Offset);
2652 Ops.push_back(Elt: N->getOperand(Num: 0)); //Chain
2653 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2654
2655 EVT VT = N->getValueType(ResNo: 0);
2656 for (unsigned I = 0; I < NumVecs; ++I)
2657 ReplaceUses(F: SDValue(N, I),
2658 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2659 Operand: SDValue(Mov, 0)));
2660
2661 // Copy chain
2662 unsigned ChainIdx = NumVecs;
2663 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2664 CurDAG->RemoveDeadNode(N);
2665}
2666
2667void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2668 unsigned NumOutVecs,
2669 bool IsTupleInput,
2670 unsigned Opc) {
2671 SDLoc DL(N);
2672 EVT VT = N->getValueType(ResNo: 0);
2673 unsigned NumInVecs = N->getNumOperands() - 1;
2674
2675 SmallVector<SDValue, 6> Ops;
2676 if (IsTupleInput) {
2677 assert((NumInVecs == 2 || NumInVecs == 4) &&
2678 "Don't know how to handle multi-register input!");
2679 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumInVecs));
2680 Ops.push_back(Elt: createZMulTuple(Regs));
2681 } else {
2682 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2683 for (unsigned I = 0; I < NumInVecs; I++)
2684 Ops.push_back(Elt: N->getOperand(Num: 1 + I));
2685 }
2686
2687 SDNode *Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2688 SDValue SuperReg = SDValue(Res, 0);
2689
2690 for (unsigned I = 0; I < NumOutVecs; I++)
2691 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2692 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2693 CurDAG->RemoveDeadNode(N);
2694}
2695
2696void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2697 unsigned Opc) {
2698 SDLoc dl(N);
2699 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2700
2701 // Form a REG_SEQUENCE to force register allocation.
2702 bool Is128Bit = VT.getSizeInBits() == 128;
2703 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2704 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2705
2706 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), N->getOperand(Num: 0)};
2707 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2708
2709 // Transfer memoperands.
2710 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2711 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2712
2713 ReplaceNode(F: N, T: St);
2714}
2715
2716void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2717 unsigned Scale, unsigned Opc_rr,
2718 unsigned Opc_ri) {
2719 SDLoc dl(N);
2720
2721 // Form a REG_SEQUENCE to force register allocation.
2722 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2723 SDValue RegSeq = createZTuple(Regs);
2724
2725 // Optimize addressing mode.
2726 unsigned Opc;
2727 SDValue Offset, Base;
2728 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2729 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: NumVecs + 3),
2730 OldOffset: CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64), Scale);
2731
2732 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), // predicate
2733 Base, // address
2734 Offset, // offset
2735 N->getOperand(Num: 0)}; // chain
2736 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2737
2738 // Transfer memoperands.
2739 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2740 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2741
2742 ReplaceNode(F: N, T: St);
2743}
2744
2745void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2746 unsigned Opc) {
2747 SDLoc dl(N);
2748 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2749 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2750 MVT::Other}; // Type for the Chain
2751
2752 // Form a REG_SEQUENCE to force register allocation.
2753 bool Is128Bit = VT.getSizeInBits() == 128;
2754 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2755 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2756
2757 SDValue Ops[] = {RegSeq,
2758 N->getOperand(Num: NumVecs + 1), // base register
2759 N->getOperand(Num: NumVecs + 2), // Incremental
2760 N->getOperand(Num: 0)}; // Chain
2761 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2762
2763 // Transfer memoperands.
2764 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2765 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2766
2767 ReplaceNode(F: N, T: St);
2768}
2769
2770namespace {
2771/// WidenVector - Given a value in the V64 register class, produce the
2772/// equivalent value in the V128 register class.
2773class WidenVector {
2774 SelectionDAG &DAG;
2775
2776public:
2777 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2778
2779 SDValue operator()(SDValue V64Reg) {
2780 EVT VT = V64Reg.getValueType();
2781 unsigned NarrowSize = VT.getVectorNumElements();
2782 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2783 MVT WideTy = MVT::getVectorVT(VT: EltTy, NumElements: 2 * NarrowSize);
2784 SDLoc DL(V64Reg);
2785
2786 SDValue Undef =
2787 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: WideTy), 0);
2788 return DAG.getTargetInsertSubreg(SRIdx: AArch64::dsub, DL, VT: WideTy, Operand: Undef, Subreg: V64Reg);
2789 }
2790};
2791} // namespace
2792
2793/// NarrowVector - Given a value in the V128 register class, produce the
2794/// equivalent value in the V64 register class.
2795static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
2796 EVT VT = V128Reg.getValueType();
2797 unsigned WideSize = VT.getVectorNumElements();
2798 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2799 MVT NarrowTy = MVT::getVectorVT(VT: EltTy, NumElements: WideSize / 2);
2800
2801 return DAG.getTargetExtractSubreg(SRIdx: AArch64::dsub, DL: SDLoc(V128Reg), VT: NarrowTy,
2802 Operand: V128Reg);
2803}
2804
2805void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2806 unsigned Opc) {
2807 SDLoc dl(N);
2808 EVT VT = N->getValueType(ResNo: 0);
2809 bool Narrow = VT.getSizeInBits() == 64;
2810
2811 // Form a REG_SEQUENCE to force register allocation.
2812 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2813
2814 if (Narrow)
2815 transform(Range&: Regs, d_first: Regs.begin(),
2816 F: WidenVector(*CurDAG));
2817
2818 SDValue RegSeq = createQTuple(Regs);
2819
2820 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2821
2822 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2823
2824 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2825 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2826 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2827 SDValue SuperReg = SDValue(Ld, 0);
2828
2829 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2830 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2831 AArch64::qsub2, AArch64::qsub3 };
2832 for (unsigned i = 0; i < NumVecs; ++i) {
2833 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT, Operand: SuperReg);
2834 if (Narrow)
2835 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2836 ReplaceUses(F: SDValue(N, i), T: NV);
2837 }
2838
2839 // Transfer memoperands.
2840 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2841 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2842
2843 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2844 CurDAG->RemoveDeadNode(N);
2845}
2846
2847void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2848 unsigned Opc) {
2849 SDLoc dl(N);
2850 EVT VT = N->getValueType(ResNo: 0);
2851 bool Narrow = VT.getSizeInBits() == 64;
2852
2853 // Form a REG_SEQUENCE to force register allocation.
2854 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2855
2856 if (Narrow)
2857 transform(Range&: Regs, d_first: Regs.begin(),
2858 F: WidenVector(*CurDAG));
2859
2860 SDValue RegSeq = createQTuple(Regs);
2861
2862 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2863 RegSeq->getValueType(ResNo: 0), MVT::Other};
2864
2865 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2866
2867 SDValue Ops[] = {RegSeq,
2868 CurDAG->getTargetConstant(Val: LaneNo, DL: dl,
2869 VT: MVT::i64), // Lane Number
2870 N->getOperand(Num: NumVecs + 2), // Base register
2871 N->getOperand(Num: NumVecs + 3), // Incremental
2872 N->getOperand(Num: 0)};
2873 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2874
2875 // Update uses of the write back register
2876 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2877
2878 // Update uses of the vector list
2879 SDValue SuperReg = SDValue(Ld, 1);
2880 if (NumVecs == 1) {
2881 ReplaceUses(F: SDValue(N, 0),
2882 T: Narrow ? NarrowVector(V128Reg: SuperReg, DAG&: *CurDAG) : SuperReg);
2883 } else {
2884 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2885 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2886 AArch64::qsub2, AArch64::qsub3 };
2887 for (unsigned i = 0; i < NumVecs; ++i) {
2888 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT,
2889 Operand: SuperReg);
2890 if (Narrow)
2891 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2892 ReplaceUses(F: SDValue(N, i), T: NV);
2893 }
2894 }
2895
2896 // Transfer memoperands.
2897 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2898 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2899
2900 // Update the Chain
2901 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2902 CurDAG->RemoveDeadNode(N);
2903}
2904
2905void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
2906 unsigned Opc) {
2907 SDLoc dl(N);
2908 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2909 bool Narrow = VT.getSizeInBits() == 64;
2910
2911 // Form a REG_SEQUENCE to force register allocation.
2912 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2913
2914 if (Narrow)
2915 transform(Range&: Regs, d_first: Regs.begin(),
2916 F: WidenVector(*CurDAG));
2917
2918 SDValue RegSeq = createQTuple(Regs);
2919
2920 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2921
2922 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2923 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2924 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
2925
2926 // Transfer memoperands.
2927 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2928 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2929
2930 ReplaceNode(F: N, T: St);
2931}
2932
2933void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
2934 unsigned Opc) {
2935 SDLoc dl(N);
2936 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2937 bool Narrow = VT.getSizeInBits() == 64;
2938
2939 // Form a REG_SEQUENCE to force register allocation.
2940 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2941
2942 if (Narrow)
2943 transform(Range&: Regs, d_first: Regs.begin(),
2944 F: WidenVector(*CurDAG));
2945
2946 SDValue RegSeq = createQTuple(Regs);
2947
2948 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2949 MVT::Other};
2950
2951 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2952
2953 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2954 N->getOperand(Num: NumVecs + 2), // Base Register
2955 N->getOperand(Num: NumVecs + 3), // Incremental
2956 N->getOperand(Num: 0)};
2957 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2958
2959 // Transfer memoperands.
2960 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2961 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2962
2963 ReplaceNode(F: N, T: St);
2964}
2965
2966static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
2967 unsigned &Opc, SDValue &Opd0,
2968 unsigned &LSB, unsigned &MSB,
2969 unsigned NumberOfIgnoredLowBits,
2970 bool BiggerPattern) {
2971 assert(N->getOpcode() == ISD::AND &&
2972 "N must be a AND operation to call this function");
2973
2974 EVT VT = N->getValueType(ResNo: 0);
2975
2976 // Here we can test the type of VT and return false when the type does not
2977 // match, but since it is done prior to that call in the current context
2978 // we turned that into an assert to avoid redundant code.
2979 assert((VT == MVT::i32 || VT == MVT::i64) &&
2980 "Type checking must have been done before calling this function");
2981
2982 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2983 // changed the AND node to a 32-bit mask operation. We'll have to
2984 // undo that as part of the transform here if we want to catch all
2985 // the opportunities.
2986 // Currently the NumberOfIgnoredLowBits argument helps to recover
2987 // from these situations when matching bigger pattern (bitfield insert).
2988
2989 // For unsigned extracts, check for a shift right and mask
2990 uint64_t AndImm = 0;
2991 if (!isOpcWithIntImmediate(N, Opc: ISD::AND, Imm&: AndImm))
2992 return false;
2993
2994 const SDNode *Op0 = N->getOperand(Num: 0).getNode();
2995
2996 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2997 // simplified. Try to undo that
2998 AndImm |= maskTrailingOnes<uint64_t>(N: NumberOfIgnoredLowBits);
2999
3000 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
3001 if (AndImm & (AndImm + 1))
3002 return false;
3003
3004 bool ClampMSB = false;
3005 uint64_t SrlImm = 0;
3006 // Handle the SRL + ANY_EXTEND case.
3007 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
3008 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3009 // Extend the incoming operand of the SRL to 64-bit.
3010 Opd0 = Widen(CurDAG, N: Op0->getOperand(Num: 0).getOperand(i: 0));
3011 // Make sure to clamp the MSB so that we preserve the semantics of the
3012 // original operations.
3013 ClampMSB = true;
3014 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
3015 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL,
3016 Imm&: SrlImm)) {
3017 // If the shift result was truncated, we can still combine them.
3018 Opd0 = Op0->getOperand(Num: 0).getOperand(i: 0);
3019
3020 // Use the type of SRL node.
3021 VT = Opd0->getValueType(ResNo: 0);
3022 } else if (isOpcWithIntImmediate(N: Op0, Opc: ISD::SRL, Imm&: SrlImm)) {
3023 Opd0 = Op0->getOperand(Num: 0);
3024 ClampMSB = (VT == MVT::i32);
3025 } else if (BiggerPattern) {
3026 // Let's pretend a 0 shift right has been performed.
3027 // The resulting code will be at least as good as the original one
3028 // plus it may expose more opportunities for bitfield insert pattern.
3029 // FIXME: Currently we limit this to the bigger pattern, because
3030 // some optimizations expect AND and not UBFM.
3031 Opd0 = N->getOperand(Num: 0);
3032 } else
3033 return false;
3034
3035 // Bail out on large immediates. This happens when no proper
3036 // combining/constant folding was performed.
3037 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
3038 LLVM_DEBUG(
3039 (dbgs() << N
3040 << ": Found large shift immediate, this should not happen\n"));
3041 return false;
3042 }
3043
3044 LSB = SrlImm;
3045 MSB = SrlImm +
3046 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(Value: AndImm)
3047 : llvm::countr_one<uint64_t>(Value: AndImm)) -
3048 1;
3049 if (ClampMSB)
3050 // Since we're moving the extend before the right shift operation, we need
3051 // to clamp the MSB to make sure we don't shift in undefined bits instead of
3052 // the zeros which would get shifted in with the original right shift
3053 // operation.
3054 MSB = MSB > 31 ? 31 : MSB;
3055
3056 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3057 return true;
3058}
3059
3060static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
3061 SDValue &Opd0, unsigned &Immr,
3062 unsigned &Imms) {
3063 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
3064
3065 EVT VT = N->getValueType(ResNo: 0);
3066 unsigned BitWidth = VT.getSizeInBits();
3067 assert((VT == MVT::i32 || VT == MVT::i64) &&
3068 "Type checking must have been done before calling this function");
3069
3070 SDValue Op = N->getOperand(Num: 0);
3071 if (Op->getOpcode() == ISD::TRUNCATE) {
3072 Op = Op->getOperand(Num: 0);
3073 VT = Op->getValueType(ResNo: 0);
3074 BitWidth = VT.getSizeInBits();
3075 }
3076
3077 uint64_t ShiftImm;
3078 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRL, Imm&: ShiftImm) &&
3079 !isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
3080 return false;
3081
3082 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
3083 if (ShiftImm + Width > BitWidth)
3084 return false;
3085
3086 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
3087 Opd0 = Op.getOperand(i: 0);
3088 Immr = ShiftImm;
3089 Imms = ShiftImm + Width - 1;
3090 return true;
3091}
3092
3093static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
3094 SDValue &Opd0, unsigned &LSB,
3095 unsigned &MSB) {
3096 // We are looking for the following pattern which basically extracts several
3097 // continuous bits from the source value and places it from the LSB of the
3098 // destination value, all other bits of the destination value or set to zero:
3099 //
3100 // Value2 = AND Value, MaskImm
3101 // SRL Value2, ShiftImm
3102 //
3103 // with MaskImm >> ShiftImm to search for the bit width.
3104 //
3105 // This gets selected into a single UBFM:
3106 //
3107 // UBFM Value, ShiftImm, Log2_64(MaskImm)
3108 //
3109
3110 if (N->getOpcode() != ISD::SRL)
3111 return false;
3112
3113 uint64_t AndMask = 0;
3114 if (!isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm&: AndMask))
3115 return false;
3116
3117 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3118
3119 uint64_t SrlImm = 0;
3120 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3121 return false;
3122
3123 // Check whether we really have several bits extract here.
3124 if (!isMask_64(Value: AndMask >> SrlImm))
3125 return false;
3126
3127 Opc = N->getValueType(ResNo: 0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3128 LSB = SrlImm;
3129 MSB = llvm::Log2_64(Value: AndMask);
3130 return true;
3131}
3132
3133static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
3134 unsigned &Immr, unsigned &Imms,
3135 bool BiggerPattern) {
3136 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
3137 "N must be a SHR/SRA operation to call this function");
3138
3139 EVT VT = N->getValueType(ResNo: 0);
3140
3141 // Here we can test the type of VT and return false when the type does not
3142 // match, but since it is done prior to that call in the current context
3143 // we turned that into an assert to avoid redundant code.
3144 assert((VT == MVT::i32 || VT == MVT::i64) &&
3145 "Type checking must have been done before calling this function");
3146
3147 // Check for AND + SRL doing several bits extract.
3148 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB&: Immr, MSB&: Imms))
3149 return true;
3150
3151 // We're looking for a shift of a shift.
3152 uint64_t ShlImm = 0;
3153 uint64_t TruncBits = 0;
3154 if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3155 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3156 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
3157 N->getOperand(Num: 0).getNode()->getOpcode() == ISD::TRUNCATE) {
3158 // We are looking for a shift of truncate. Truncate from i64 to i32 could
3159 // be considered as setting high 32 bits as zero. Our strategy here is to
3160 // always generate 64bit UBFM. This consistency will help the CSE pass
3161 // later find more redundancy.
3162 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3163 TruncBits = Opd0->getValueType(ResNo: 0).getSizeInBits() - VT.getSizeInBits();
3164 VT = Opd0.getValueType();
3165 assert(VT == MVT::i64 && "the promoted type should be i64");
3166 } else if (BiggerPattern) {
3167 // Let's pretend a 0 shift left has been performed.
3168 // FIXME: Currently we limit this to the bigger pattern case,
3169 // because some optimizations expect AND and not UBFM
3170 Opd0 = N->getOperand(Num: 0);
3171 } else
3172 return false;
3173
3174 // Missing combines/constant folding may have left us with strange
3175 // constants.
3176 if (ShlImm >= VT.getSizeInBits()) {
3177 LLVM_DEBUG(
3178 (dbgs() << N
3179 << ": Found large shift immediate, this should not happen\n"));
3180 return false;
3181 }
3182
3183 uint64_t SrlImm = 0;
3184 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3185 return false;
3186
3187 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
3188 "bad amount in shift node!");
3189 int immr = SrlImm - ShlImm;
3190 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
3191 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
3192 // SRA requires a signed extraction
3193 if (VT == MVT::i32)
3194 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
3195 else
3196 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
3197 return true;
3198}
3199
3200static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3201 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3202 unsigned NumberOfIgnoredLowBits = 0,
3203 bool BiggerPattern = false) {
3204 if (N->getValueType(ResNo: 0) != MVT::i32 && N->getValueType(ResNo: 0) != MVT::i64)
3205 return false;
3206
3207 switch (N->getOpcode()) {
3208 default:
3209 if (!N->isMachineOpcode())
3210 return false;
3211 break;
3212 case ISD::AND:
3213 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB&: Immr, MSB&: Imms,
3214 NumberOfIgnoredLowBits, BiggerPattern);
3215 case ISD::SRL:
3216 case ISD::SRA:
3217 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3218
3219 case ISD::SIGN_EXTEND_INREG:
3220 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3221 }
3222
3223 unsigned NOpc = N->getMachineOpcode();
3224 switch (NOpc) {
3225 default:
3226 return false;
3227 case AArch64::SBFMWri:
3228 case AArch64::UBFMWri:
3229 case AArch64::SBFMXri:
3230 case AArch64::UBFMXri:
3231 Opc = NOpc;
3232 Opd0 = N->getOperand(Num: 0);
3233 Immr = N->getConstantOperandVal(Num: 1);
3234 Imms = N->getConstantOperandVal(Num: 2);
3235 return true;
3236 }
3237 // Unreachable
3238 return false;
3239}
3240
3241bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3242 unsigned Opc, Immr, Imms;
3243 SDValue Opd0;
3244 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3245 return false;
3246
3247 EVT VT = N->getValueType(ResNo: 0);
3248 SDLoc dl(N);
3249
3250 // If the bit extract operation is 64bit but the original type is 32bit, we
3251 // need to add one EXTRACT_SUBREG.
3252 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3253 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT: MVT::i64),
3254 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT: MVT::i64)};
3255
3256 SDNode *BFM = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i64, Ops: Ops64);
3257 SDValue Inner = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl,
3258 VT: MVT::i32, Operand: SDValue(BFM, 0));
3259 ReplaceNode(F: N, T: Inner.getNode());
3260 return true;
3261 }
3262
3263 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3264 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3265 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3266 return true;
3267}
3268
3269/// Does DstMask form a complementary pair with the mask provided by
3270/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3271/// this asks whether DstMask zeroes precisely those bits that will be set by
3272/// the other half.
3273static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3274 unsigned NumberOfIgnoredHighBits, EVT VT) {
3275 assert((VT == MVT::i32 || VT == MVT::i64) &&
3276 "i32 or i64 mask type expected!");
3277 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3278
3279 // Enable implicitTrunc as we're intentionally ignoring high bits.
3280 APInt SignificantDstMask =
3281 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3282 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(width: BitWidth);
3283
3284 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3285 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3286}
3287
3288// Look for bits that will be useful for later uses.
3289// A bit is consider useless as soon as it is dropped and never used
3290// before it as been dropped.
3291// E.g., looking for useful bit of x
3292// 1. y = x & 0x7
3293// 2. z = y >> 2
3294// After #1, x useful bits are 0x7, then the useful bits of x, live through
3295// y.
3296// After #2, the useful bits of x are 0x4.
3297// However, if x is used on an unpredictable instruction, then all its bits
3298// are useful.
3299// E.g.
3300// 1. y = x & 0x7
3301// 2. z = y >> 2
3302// 3. str x, [@x]
3303static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3304
3305static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
3306 unsigned Depth) {
3307 uint64_t Imm =
3308 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3309 Imm = AArch64_AM::decodeLogicalImmediate(val: Imm, regSize: UsefulBits.getBitWidth());
3310 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3311 getUsefulBits(Op, UsefulBits, Depth: Depth + 1);
3312}
3313
3314static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
3315 uint64_t Imm, uint64_t MSB,
3316 unsigned Depth) {
3317 // inherit the bitwidth value
3318 APInt OpUsefulBits(UsefulBits);
3319 OpUsefulBits = 1;
3320
3321 if (MSB >= Imm) {
3322 OpUsefulBits <<= MSB - Imm + 1;
3323 --OpUsefulBits;
3324 // The interesting part will be in the lower part of the result
3325 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3326 // The interesting part was starting at Imm in the argument
3327 OpUsefulBits <<= Imm;
3328 } else {
3329 OpUsefulBits <<= MSB + 1;
3330 --OpUsefulBits;
3331 // The interesting part will be shifted in the result
3332 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3333 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3334 // The interesting part was at zero in the argument
3335 OpUsefulBits.lshrInPlace(ShiftAmt: OpUsefulBits.getBitWidth() - Imm);
3336 }
3337
3338 UsefulBits &= OpUsefulBits;
3339}
3340
3341static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3342 unsigned Depth) {
3343 uint64_t Imm =
3344 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3345 uint64_t MSB =
3346 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3347
3348 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3349}
3350
3351static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
3352 unsigned Depth) {
3353 uint64_t ShiftTypeAndValue =
3354 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3355 APInt Mask(UsefulBits);
3356 Mask.clearAllBits();
3357 Mask.flipAllBits();
3358
3359 if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSL) {
3360 // Shift Left
3361 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3362 Mask <<= ShiftAmt;
3363 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3364 Mask.lshrInPlace(ShiftAmt);
3365 } else if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSR) {
3366 // Shift Right
3367 // We do not handle AArch64_AM::ASR, because the sign will change the
3368 // number of useful bits
3369 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3370 Mask.lshrInPlace(ShiftAmt);
3371 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3372 Mask <<= ShiftAmt;
3373 } else
3374 return;
3375
3376 UsefulBits &= Mask;
3377}
3378
3379static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3380 unsigned Depth) {
3381 uint64_t Imm =
3382 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3383 uint64_t MSB =
3384 cast<const ConstantSDNode>(Val: Op.getOperand(i: 3).getNode())->getZExtValue();
3385
3386 APInt OpUsefulBits(UsefulBits);
3387 OpUsefulBits = 1;
3388
3389 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3390 ResultUsefulBits.flipAllBits();
3391 APInt Mask(UsefulBits.getBitWidth(), 0);
3392
3393 getUsefulBits(Op, UsefulBits&: ResultUsefulBits, Depth: Depth + 1);
3394
3395 if (MSB >= Imm) {
3396 // The instruction is a BFXIL.
3397 uint64_t Width = MSB - Imm + 1;
3398 uint64_t LSB = Imm;
3399
3400 OpUsefulBits <<= Width;
3401 --OpUsefulBits;
3402
3403 if (Op.getOperand(i: 1) == Orig) {
3404 // Copy the low bits from the result to bits starting from LSB.
3405 Mask = ResultUsefulBits & OpUsefulBits;
3406 Mask <<= LSB;
3407 }
3408
3409 if (Op.getOperand(i: 0) == Orig)
3410 // Bits starting from LSB in the input contribute to the result.
3411 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3412 } else {
3413 // The instruction is a BFI.
3414 uint64_t Width = MSB + 1;
3415 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3416
3417 OpUsefulBits <<= Width;
3418 --OpUsefulBits;
3419 OpUsefulBits <<= LSB;
3420
3421 if (Op.getOperand(i: 1) == Orig) {
3422 // Copy the bits from the result to the zero bits.
3423 Mask = ResultUsefulBits & OpUsefulBits;
3424 Mask.lshrInPlace(ShiftAmt: LSB);
3425 }
3426
3427 if (Op.getOperand(i: 0) == Orig)
3428 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3429 }
3430
3431 UsefulBits &= Mask;
3432}
3433
3434static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3435 SDValue Orig, unsigned Depth) {
3436
3437 // Users of this node should have already been instruction selected
3438 // FIXME: Can we turn that into an assert?
3439 if (!UserNode->isMachineOpcode())
3440 return;
3441
3442 switch (UserNode->getMachineOpcode()) {
3443 default:
3444 return;
3445 case AArch64::ANDSWri:
3446 case AArch64::ANDSXri:
3447 case AArch64::ANDWri:
3448 case AArch64::ANDXri:
3449 // We increment Depth only when we call the getUsefulBits
3450 return getUsefulBitsFromAndWithImmediate(Op: SDValue(UserNode, 0), UsefulBits,
3451 Depth);
3452 case AArch64::UBFMWri:
3453 case AArch64::UBFMXri:
3454 return getUsefulBitsFromUBFM(Op: SDValue(UserNode, 0), UsefulBits, Depth);
3455
3456 case AArch64::ORRWrs:
3457 case AArch64::ORRXrs:
3458 if (UserNode->getOperand(Num: 0) != Orig && UserNode->getOperand(Num: 1) == Orig)
3459 getUsefulBitsFromOrWithShiftedReg(Op: SDValue(UserNode, 0), UsefulBits,
3460 Depth);
3461 return;
3462 case AArch64::BFMWri:
3463 case AArch64::BFMXri:
3464 return getUsefulBitsFromBFM(Op: SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3465
3466 case AArch64::STRBBui:
3467 case AArch64::STURBBi:
3468 if (UserNode->getOperand(Num: 0) != Orig)
3469 return;
3470 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3471 return;
3472
3473 case AArch64::STRHHui:
3474 case AArch64::STURHHi:
3475 if (UserNode->getOperand(Num: 0) != Orig)
3476 return;
3477 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3478 return;
3479 }
3480}
3481
3482static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3483 if (Depth >= SelectionDAG::MaxRecursionDepth)
3484 return;
3485 // Initialize UsefulBits
3486 if (!Depth) {
3487 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3488 // At the beginning, assume every produced bits is useful
3489 UsefulBits = APInt(Bitwidth, 0);
3490 UsefulBits.flipAllBits();
3491 }
3492 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3493
3494 for (SDNode *Node : Op.getNode()->users()) {
3495 // A use cannot produce useful bits
3496 APInt UsefulBitsForUse = APInt(UsefulBits);
3497 getUsefulBitsForUse(UserNode: Node, UsefulBits&: UsefulBitsForUse, Orig: Op, Depth);
3498 UsersUsefulBits |= UsefulBitsForUse;
3499 }
3500 // UsefulBits contains the produced bits that are meaningful for the
3501 // current definition, thus a user cannot make a bit meaningful at
3502 // this point
3503 UsefulBits &= UsersUsefulBits;
3504}
3505
3506/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3507/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3508/// 0, return Op unchanged.
3509static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3510 if (ShlAmount == 0)
3511 return Op;
3512
3513 EVT VT = Op.getValueType();
3514 SDLoc dl(Op);
3515 unsigned BitWidth = VT.getSizeInBits();
3516 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3517
3518 SDNode *ShiftNode;
3519 if (ShlAmount > 0) {
3520 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3521 ShiftNode = CurDAG->getMachineNode(
3522 Opcode: UBFMOpc, dl, VT, Op1: Op,
3523 Op2: CurDAG->getTargetConstant(Val: BitWidth - ShlAmount, DL: dl, VT),
3524 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1 - ShlAmount, DL: dl, VT));
3525 } else {
3526 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3527 assert(ShlAmount < 0 && "expected right shift");
3528 int ShrAmount = -ShlAmount;
3529 ShiftNode = CurDAG->getMachineNode(
3530 Opcode: UBFMOpc, dl, VT, Op1: Op, Op2: CurDAG->getTargetConstant(Val: ShrAmount, DL: dl, VT),
3531 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL: dl, VT));
3532 }
3533
3534 return SDValue(ShiftNode, 0);
3535}
3536
3537// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3538static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3539 bool BiggerPattern,
3540 const uint64_t NonZeroBits,
3541 SDValue &Src, int &DstLSB,
3542 int &Width);
3543
3544// For bit-field-positioning pattern "shl VAL, N)".
3545static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3546 bool BiggerPattern,
3547 const uint64_t NonZeroBits,
3548 SDValue &Src, int &DstLSB,
3549 int &Width);
3550
3551/// Does this tree qualify as an attempt to move a bitfield into position,
3552/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3553static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
3554 bool BiggerPattern, SDValue &Src,
3555 int &DstLSB, int &Width) {
3556 EVT VT = Op.getValueType();
3557 unsigned BitWidth = VT.getSizeInBits();
3558 (void)BitWidth;
3559 assert(BitWidth == 32 || BitWidth == 64);
3560
3561 KnownBits Known = CurDAG->computeKnownBits(Op);
3562
3563 // Non-zero in the sense that they're not provably zero, which is the key
3564 // point if we want to use this value
3565 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3566 if (!isShiftedMask_64(Value: NonZeroBits))
3567 return false;
3568
3569 switch (Op.getOpcode()) {
3570 default:
3571 break;
3572 case ISD::AND:
3573 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3574 NonZeroBits, Src, DstLSB, Width);
3575 case ISD::SHL:
3576 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3577 NonZeroBits, Src, DstLSB, Width);
3578 }
3579
3580 return false;
3581}
3582
3583static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3584 bool BiggerPattern,
3585 const uint64_t NonZeroBits,
3586 SDValue &Src, int &DstLSB,
3587 int &Width) {
3588 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3589
3590 EVT VT = Op.getValueType();
3591 assert((VT == MVT::i32 || VT == MVT::i64) &&
3592 "Caller guarantees VT is one of i32 or i64");
3593 (void)VT;
3594
3595 uint64_t AndImm;
3596 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::AND, Imm&: AndImm))
3597 return false;
3598
3599 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3600 // 1) (AndImm & (1 << POS) == 0)
3601 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3602 //
3603 // 1) and 2) don't agree so something must be wrong (e.g., in
3604 // 'SelectionDAG::computeKnownBits')
3605 assert((~AndImm & NonZeroBits) == 0 &&
3606 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3607
3608 SDValue AndOp0 = Op.getOperand(i: 0);
3609
3610 uint64_t ShlImm;
3611 SDValue ShlOp0;
3612 if (isOpcWithIntImmediate(N: AndOp0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3613 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3614 ShlOp0 = AndOp0.getOperand(i: 0);
3615 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3616 isOpcWithIntImmediate(N: AndOp0.getOperand(i: 0).getNode(), Opc: ISD::SHL,
3617 Imm&: ShlImm)) {
3618 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3619
3620 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3621 SDValue ShlVal = AndOp0.getOperand(i: 0);
3622
3623 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3624 // expect VT to be MVT::i32.
3625 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3626
3627 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3628 ShlOp0 = Widen(CurDAG, N: ShlVal.getOperand(i: 0));
3629 } else
3630 return false;
3631
3632 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3633 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3634 // AndOp0+AND.
3635 if (!BiggerPattern && !AndOp0.hasOneUse())
3636 return false;
3637
3638 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3639 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3640
3641 // Bail out on large Width. This happens when no proper combining / constant
3642 // folding was performed.
3643 if (Width >= (int)VT.getSizeInBits()) {
3644 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3645 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3646 // "val".
3647 // If VT is i32, what Width >= 32 means:
3648 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3649 // demands at least 'Width' bits (after dag-combiner). This together with
3650 // `any_extend` Op (undefined higher bits) indicates missed combination
3651 // when lowering the 'and' IR instruction to an machine IR instruction.
3652 LLVM_DEBUG(
3653 dbgs()
3654 << "Found large Width in bit-field-positioning -- this indicates no "
3655 "proper combining / constant folding was performed\n");
3656 return false;
3657 }
3658
3659 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3660 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3661 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3662 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3663 // which case it is not profitable to insert an extra shift.
3664 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3665 return false;
3666
3667 Src = getLeftShift(CurDAG, Op: ShlOp0, ShlAmount: ShlImm - DstLSB);
3668 return true;
3669}
3670
3671// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3672// UBFIZ.
3673static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op,
3674 SDValue &Src, int &DstLSB,
3675 int &Width) {
3676 // Caller should have verified that N is a left shift with constant shift
3677 // amount; asserts that.
3678 assert(Op.getOpcode() == ISD::SHL &&
3679 "Op.getNode() should be a SHL node to call this function");
3680 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3681 "Op.getNode() should shift ShlImm to call this function");
3682
3683 uint64_t AndImm = 0;
3684 SDValue Op0 = Op.getOperand(i: 0);
3685 if (!isOpcWithIntImmediate(N: Op0.getNode(), Opc: ISD::AND, Imm&: AndImm))
3686 return false;
3687
3688 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3689 if (isMask_64(Value: ShiftedAndImm)) {
3690 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3691 // should end with Mask, and could be prefixed with random bits if those
3692 // bits are shifted out.
3693 //
3694 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3695 // the AND result corresponding to those bits are shifted out, so it's fine
3696 // to not extract them.
3697 Width = llvm::countr_one(Value: ShiftedAndImm);
3698 DstLSB = ShlImm;
3699 Src = Op0.getOperand(i: 0);
3700 return true;
3701 }
3702 return false;
3703}
3704
3705static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3706 bool BiggerPattern,
3707 const uint64_t NonZeroBits,
3708 SDValue &Src, int &DstLSB,
3709 int &Width) {
3710 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3711
3712 EVT VT = Op.getValueType();
3713 assert((VT == MVT::i32 || VT == MVT::i64) &&
3714 "Caller guarantees that type is i32 or i64");
3715 (void)VT;
3716
3717 uint64_t ShlImm;
3718 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SHL, Imm&: ShlImm))
3719 return false;
3720
3721 if (!BiggerPattern && !Op.hasOneUse())
3722 return false;
3723
3724 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3725 return true;
3726
3727 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3728 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3729
3730 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3731 return false;
3732
3733 Src = getLeftShift(CurDAG, Op: Op.getOperand(i: 0), ShlAmount: ShlImm - DstLSB);
3734 return true;
3735}
3736
3737static bool isShiftedMask(uint64_t Mask, EVT VT) {
3738 assert(VT == MVT::i32 || VT == MVT::i64);
3739 if (VT == MVT::i32)
3740 return isShiftedMask_32(Value: Mask);
3741 return isShiftedMask_64(Value: Mask);
3742}
3743
3744// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3745// inserted only sets known zero bits.
3746static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
3747 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3748
3749 EVT VT = N->getValueType(ResNo: 0);
3750 if (VT != MVT::i32 && VT != MVT::i64)
3751 return false;
3752
3753 unsigned BitWidth = VT.getSizeInBits();
3754
3755 uint64_t OrImm;
3756 if (!isOpcWithIntImmediate(N, Opc: ISD::OR, Imm&: OrImm))
3757 return false;
3758
3759 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3760 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3761 // performance neutral.
3762 if (AArch64_AM::isLogicalImmediate(imm: OrImm, regSize: BitWidth))
3763 return false;
3764
3765 uint64_t MaskImm;
3766 SDValue And = N->getOperand(Num: 0);
3767 // Must be a single use AND with an immediate operand.
3768 if (!And.hasOneUse() ||
3769 !isOpcWithIntImmediate(N: And.getNode(), Opc: ISD::AND, Imm&: MaskImm))
3770 return false;
3771
3772 // Compute the Known Zero for the AND as this allows us to catch more general
3773 // cases than just looking for AND with imm.
3774 KnownBits Known = CurDAG->computeKnownBits(Op: And);
3775
3776 // Non-zero in the sense that they're not provably zero, which is the key
3777 // point if we want to use this value.
3778 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3779
3780 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3781 if (!isShiftedMask(Mask: Known.Zero.getZExtValue(), VT))
3782 return false;
3783
3784 // The bits being inserted must only set those bits that are known to be zero.
3785 if ((OrImm & NotKnownZero) != 0) {
3786 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3787 // currently handle this case.
3788 return false;
3789 }
3790
3791 // BFI/BFXIL dst, src, #lsb, #width.
3792 int LSB = llvm::countr_one(Value: NotKnownZero);
3793 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3794
3795 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3796 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3797 unsigned ImmS = Width - 1;
3798
3799 // If we're creating a BFI instruction avoid cases where we need more
3800 // instructions to materialize the BFI constant as compared to the original
3801 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3802 // should be no worse in this case.
3803 bool IsBFI = LSB != 0;
3804 uint64_t BFIImm = OrImm >> LSB;
3805 if (IsBFI && !AArch64_AM::isLogicalImmediate(imm: BFIImm, regSize: BitWidth)) {
3806 // We have a BFI instruction and we know the constant can't be materialized
3807 // with a ORR-immediate with the zero register.
3808 unsigned OrChunks = 0, BFIChunks = 0;
3809 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3810 if (((OrImm >> Shift) & 0xFFFF) != 0)
3811 ++OrChunks;
3812 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3813 ++BFIChunks;
3814 }
3815 if (BFIChunks > OrChunks)
3816 return false;
3817 }
3818
3819 // Materialize the constant to be inserted.
3820 SDLoc DL(N);
3821 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3822 SDNode *MOVI = CurDAG->getMachineNode(
3823 Opcode: MOVIOpc, dl: DL, VT, Op1: CurDAG->getTargetConstant(Val: BFIImm, DL, VT));
3824
3825 // Create the BFI/BFXIL instruction.
3826 SDValue Ops[] = {And.getOperand(i: 0), SDValue(MOVI, 0),
3827 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3828 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3829 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3830 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3831 return true;
3832}
3833
3834static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG,
3835 SDValue &ShiftedOperand,
3836 uint64_t &EncodedShiftImm) {
3837 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3838 if (!Dst.hasOneUse())
3839 return false;
3840
3841 EVT VT = Dst.getValueType();
3842 assert((VT == MVT::i32 || VT == MVT::i64) &&
3843 "Caller should guarantee that VT is one of i32 or i64");
3844 const unsigned SizeInBits = VT.getSizeInBits();
3845
3846 SDLoc DL(Dst.getNode());
3847 uint64_t AndImm, ShlImm;
3848 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::AND, Imm&: AndImm) &&
3849 isShiftedMask_64(Value: AndImm)) {
3850 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3851 SDValue DstOp0 = Dst.getOperand(i: 0);
3852 if (!DstOp0.hasOneUse())
3853 return false;
3854
3855 // An example to illustrate the transformation
3856 // From:
3857 // lsr x8, x1, #1
3858 // and x8, x8, #0x3f80
3859 // bfxil x8, x1, #0, #7
3860 // To:
3861 // and x8, x23, #0x7f
3862 // ubfx x9, x23, #8, #7
3863 // orr x23, x8, x9, lsl #7
3864 //
3865 // The number of instructions remains the same, but ORR is faster than BFXIL
3866 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3867 // the dependency chain is improved after the transformation.
3868 uint64_t SrlImm;
3869 if (isOpcWithIntImmediate(N: DstOp0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3870 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(Val: AndImm);
3871 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3872 unsigned MaskWidth =
3873 llvm::countr_one(Value: AndImm >> NumTrailingZeroInShiftedMask);
3874 unsigned UBFMOpc =
3875 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3876 SDNode *UBFMNode = CurDAG->getMachineNode(
3877 Opcode: UBFMOpc, dl: DL, VT, Op1: DstOp0.getOperand(i: 0),
3878 Op2: CurDAG->getTargetConstant(Val: SrlImm + NumTrailingZeroInShiftedMask, DL,
3879 VT),
3880 Op3: CurDAG->getTargetConstant(
3881 Val: SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3882 ShiftedOperand = SDValue(UBFMNode, 0);
3883 EncodedShiftImm = AArch64_AM::getShifterImm(
3884 ST: AArch64_AM::LSL, Imm: NumTrailingZeroInShiftedMask);
3885 return true;
3886 }
3887 }
3888 return false;
3889 }
3890
3891 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3892 ShiftedOperand = Dst.getOperand(i: 0);
3893 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm);
3894 return true;
3895 }
3896
3897 uint64_t SrlImm;
3898 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3899 ShiftedOperand = Dst.getOperand(i: 0);
3900 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm);
3901 return true;
3902 }
3903 return false;
3904}
3905
3906// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3907// the operands and select it to AArch64::ORR with shifted registers if
3908// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3909static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3910 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3911 const bool BiggerPattern) {
3912 EVT VT = N->getValueType(ResNo: 0);
3913 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3914 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3915 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3916 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3917 assert((VT == MVT::i32 || VT == MVT::i64) &&
3918 "Expect result type to be i32 or i64 since N is combinable to BFM");
3919 SDLoc DL(N);
3920
3921 // Bail out if BFM simplifies away one node in BFM Dst.
3922 if (OrOpd1 != Dst)
3923 return false;
3924
3925 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3926 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3927 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3928 if (BiggerPattern) {
3929 uint64_t SrcAndImm;
3930 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::AND, Imm&: SrcAndImm) &&
3931 isMask_64(Value: SrcAndImm) && OrOpd0.getOperand(i: 0) == Src) {
3932 // OrOpd0 = AND Src, #Mask
3933 // So BFM simplifies away one AND node from Src and doesn't simplify away
3934 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3935 // one node (from Rd), ORR is better since it has higher throughput and
3936 // smaller latency than BFM on many AArch64 processors (and for the rest
3937 // ORR is at least as good as BFM).
3938 SDValue ShiftedOperand;
3939 uint64_t EncodedShiftImm;
3940 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3941 EncodedShiftImm)) {
3942 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3943 CurDAG->getTargetConstant(Val: EncodedShiftImm, DL, VT)};
3944 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3945 return true;
3946 }
3947 }
3948 return false;
3949 }
3950
3951 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3952
3953 uint64_t ShlImm;
3954 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3955 if (OrOpd0.getOperand(i: 0) == Src && OrOpd0.hasOneUse()) {
3956 SDValue Ops[] = {
3957 Dst, Src,
3958 CurDAG->getTargetConstant(
3959 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3960 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3961 return true;
3962 }
3963
3964 // Select the following pattern to left-shifted operand rather than BFI.
3965 // %val1 = op ..
3966 // %val2 = shl %val1, #imm
3967 // %res = or %val1, %val2
3968 //
3969 // If N is selected to be BFI, we know that
3970 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3971 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3972 //
3973 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3974 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3975 SDValue Ops[] = {
3976 OrOpd1, OrOpd1,
3977 CurDAG->getTargetConstant(
3978 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3979 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3980 return true;
3981 }
3982 }
3983
3984 uint64_t SrlImm;
3985 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3986 // Select the following pattern to right-shifted operand rather than BFXIL.
3987 // %val1 = op ..
3988 // %val2 = lshr %val1, #imm
3989 // %res = or %val1, %val2
3990 //
3991 // If N is selected to be BFXIL, we know that
3992 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3993 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
3994 //
3995 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
3996 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3997 SDValue Ops[] = {
3998 OrOpd1, OrOpd1,
3999 CurDAG->getTargetConstant(
4000 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm), DL, VT)};
4001 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
4002 return true;
4003 }
4004 }
4005
4006 return false;
4007}
4008
4009static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
4010 SelectionDAG *CurDAG) {
4011 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
4012
4013 EVT VT = N->getValueType(ResNo: 0);
4014 if (VT != MVT::i32 && VT != MVT::i64)
4015 return false;
4016
4017 unsigned BitWidth = VT.getSizeInBits();
4018
4019 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
4020 // have the expected shape. Try to undo that.
4021
4022 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
4023 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
4024
4025 // Given a OR operation, check if we have the following pattern
4026 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
4027 // isBitfieldExtractOp)
4028 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
4029 // countTrailingZeros(mask2) == imm2 - imm + 1
4030 // f = d | c
4031 // if yes, replace the OR instruction with:
4032 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
4033
4034 // OR is commutative, check all combinations of operand order and values of
4035 // BiggerPattern, i.e.
4036 // Opd0, Opd1, BiggerPattern=false
4037 // Opd1, Opd0, BiggerPattern=false
4038 // Opd0, Opd1, BiggerPattern=true
4039 // Opd1, Opd0, BiggerPattern=true
4040 // Several of these combinations may match, so check with BiggerPattern=false
4041 // first since that will produce better results by matching more instructions
4042 // and/or inserting fewer extra instructions.
4043 for (int I = 0; I < 4; ++I) {
4044
4045 SDValue Dst, Src;
4046 unsigned ImmR, ImmS;
4047 bool BiggerPattern = I / 2;
4048 SDValue OrOpd0Val = N->getOperand(Num: I % 2);
4049 SDNode *OrOpd0 = OrOpd0Val.getNode();
4050 SDValue OrOpd1Val = N->getOperand(Num: (I + 1) % 2);
4051 SDNode *OrOpd1 = OrOpd1Val.getNode();
4052
4053 unsigned BFXOpc;
4054 int DstLSB, Width;
4055 if (isBitfieldExtractOp(CurDAG, N: OrOpd0, Opc&: BFXOpc, Opd0&: Src, Immr&: ImmR, Imms&: ImmS,
4056 NumberOfIgnoredLowBits, BiggerPattern)) {
4057 // Check that the returned opcode is compatible with the pattern,
4058 // i.e., same type and zero extended (U and not S)
4059 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
4060 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
4061 continue;
4062
4063 // Compute the width of the bitfield insertion
4064 DstLSB = 0;
4065 Width = ImmS - ImmR + 1;
4066 // FIXME: This constraint is to catch bitfield insertion we may
4067 // want to widen the pattern if we want to grab general bitfield
4068 // move case
4069 if (Width <= 0)
4070 continue;
4071
4072 // If the mask on the insertee is correct, we have a BFXIL operation. We
4073 // can share the ImmR and ImmS values from the already-computed UBFM.
4074 } else if (isBitfieldPositioningOp(CurDAG, Op: OrOpd0Val,
4075 BiggerPattern,
4076 Src, DstLSB, Width)) {
4077 ImmR = (BitWidth - DstLSB) % BitWidth;
4078 ImmS = Width - 1;
4079 } else
4080 continue;
4081
4082 // Check the second part of the pattern
4083 EVT VT = OrOpd1Val.getValueType();
4084 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
4085
4086 // Compute the Known Zero for the candidate of the first operand.
4087 // This allows to catch more general case than just looking for
4088 // AND with imm. Indeed, simplify-demanded-bits may have removed
4089 // the AND instruction because it proves it was useless.
4090 KnownBits Known = CurDAG->computeKnownBits(Op: OrOpd1Val);
4091
4092 // Check if there is enough room for the second operand to appear
4093 // in the first one
4094 APInt BitsToBeInserted =
4095 APInt::getBitsSet(numBits: Known.getBitWidth(), loBit: DstLSB, hiBit: DstLSB + Width);
4096
4097 if ((BitsToBeInserted & ~Known.Zero) != 0)
4098 continue;
4099
4100 // Set the first operand
4101 uint64_t Imm;
4102 if (isOpcWithIntImmediate(N: OrOpd1, Opc: ISD::AND, Imm) &&
4103 isBitfieldDstMask(DstMask: Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
4104 // In that case, we can eliminate the AND
4105 Dst = OrOpd1->getOperand(Num: 0);
4106 else
4107 // Maybe the AND has been removed by simplify-demanded-bits
4108 // or is useful because it discards more bits
4109 Dst = OrOpd1Val;
4110
4111 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
4112 // with shifted operand is more efficient.
4113 if (tryOrrWithShift(N, OrOpd0: OrOpd0Val, OrOpd1: OrOpd1Val, Src, Dst, CurDAG,
4114 BiggerPattern))
4115 return true;
4116
4117 // both parts match
4118 SDLoc DL(N);
4119 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4120 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4121 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4122 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4123 return true;
4124 }
4125
4126 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
4127 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
4128 // mask (e.g., 0x000ffff0).
4129 uint64_t Mask0Imm, Mask1Imm;
4130 SDValue And0 = N->getOperand(Num: 0);
4131 SDValue And1 = N->getOperand(Num: 1);
4132 if (And0.hasOneUse() && And1.hasOneUse() &&
4133 isOpcWithIntImmediate(N: And0.getNode(), Opc: ISD::AND, Imm&: Mask0Imm) &&
4134 isOpcWithIntImmediate(N: And1.getNode(), Opc: ISD::AND, Imm&: Mask1Imm) &&
4135 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
4136 (isShiftedMask(Mask: Mask0Imm, VT) || isShiftedMask(Mask: Mask1Imm, VT))) {
4137
4138 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
4139 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
4140 // bits to be inserted.
4141 if (isShiftedMask(Mask: Mask0Imm, VT)) {
4142 std::swap(a&: And0, b&: And1);
4143 std::swap(a&: Mask0Imm, b&: Mask1Imm);
4144 }
4145
4146 SDValue Src = And1->getOperand(Num: 0);
4147 SDValue Dst = And0->getOperand(Num: 0);
4148 unsigned LSB = llvm::countr_zero(Val: Mask1Imm);
4149 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
4150
4151 // The BFXIL inserts the low-order bits from a source register, so right
4152 // shift the needed bits into place.
4153 SDLoc DL(N);
4154 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4155 uint64_t LsrImm = LSB;
4156 if (Src->hasOneUse() &&
4157 isOpcWithIntImmediate(N: Src.getNode(), Opc: ISD::SRL, Imm&: LsrImm) &&
4158 (LsrImm + LSB) < BitWidth) {
4159 Src = Src->getOperand(Num: 0);
4160 LsrImm += LSB;
4161 }
4162
4163 SDNode *LSR = CurDAG->getMachineNode(
4164 Opcode: ShiftOpc, dl: DL, VT, Op1: Src, Op2: CurDAG->getTargetConstant(Val: LsrImm, DL, VT),
4165 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT));
4166
4167 // BFXIL is an alias of BFM, so translate to BFM operands.
4168 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4169 unsigned ImmS = Width - 1;
4170
4171 // Create the BFXIL instruction.
4172 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4173 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4174 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4175 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4176 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4177 return true;
4178 }
4179
4180 return false;
4181}
4182
4183bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4184 if (N->getOpcode() != ISD::OR)
4185 return false;
4186
4187 APInt NUsefulBits;
4188 getUsefulBits(Op: SDValue(N, 0), UsefulBits&: NUsefulBits);
4189
4190 // If all bits are not useful, just return UNDEF.
4191 if (!NUsefulBits) {
4192 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::IMPLICIT_DEF, VT: N->getValueType(ResNo: 0));
4193 return true;
4194 }
4195
4196 if (tryBitfieldInsertOpFromOr(N, UsefulBits: NUsefulBits, CurDAG))
4197 return true;
4198
4199 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4200}
4201
4202/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4203/// equivalent of a left shift by a constant amount followed by an and masking
4204/// out a contiguous set of bits.
4205bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4206 if (N->getOpcode() != ISD::AND)
4207 return false;
4208
4209 EVT VT = N->getValueType(ResNo: 0);
4210 if (VT != MVT::i32 && VT != MVT::i64)
4211 return false;
4212
4213 SDValue Op0;
4214 int DstLSB, Width;
4215 if (!isBitfieldPositioningOp(CurDAG, Op: SDValue(N, 0), /*BiggerPattern=*/false,
4216 Src&: Op0, DstLSB, Width))
4217 return false;
4218
4219 // ImmR is the rotate right amount.
4220 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4221 // ImmS is the most significant bit of the source to be moved.
4222 unsigned ImmS = Width - 1;
4223
4224 SDLoc DL(N);
4225 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4226 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4227 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4228 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4229 return true;
4230}
4231
4232static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
4233 SDValue &FixedPos,
4234 unsigned RegWidth,
4235 bool isReciprocal) {
4236 APFloat FVal(0.0);
4237 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
4238 FVal = CN->getValueAPF();
4239 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(Val&: N)) {
4240 // Some otherwise illegal constants are allowed in this case.
4241 if (LN->getOperand(Num: 1).getOpcode() != AArch64ISD::ADDlow ||
4242 !isa<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1)))
4243 return false;
4244
4245 ConstantPoolSDNode *CN =
4246 dyn_cast<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1));
4247 FVal = cast<ConstantFP>(Val: CN->getConstVal())->getValueAPF();
4248 } else
4249 return false;
4250
4251 if (unsigned FBits =
4252 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4253 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4254 return true;
4255 }
4256
4257 return false;
4258}
4259
4260static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
4261 SDValue N,
4262 SDValue &FixedPos,
4263 unsigned RegWidth,
4264 bool isReciprocal) {
4265 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4266 N.getValueType().getScalarSizeInBits() ==
4267 N.getOperand(i: 0).getValueType().getScalarSizeInBits())
4268 N = N.getOperand(i: 0);
4269
4270 auto ImmToFloat = [RegWidth](APInt Imm) {
4271 switch (RegWidth) {
4272 case 16:
4273 return APFloat(APFloat::IEEEhalf(), Imm);
4274 case 32:
4275 return APFloat(APFloat::IEEEsingle(), Imm);
4276 case 64:
4277 return APFloat(APFloat::IEEEdouble(), Imm);
4278 default:
4279 llvm_unreachable("Unexpected RegWidth!");
4280 };
4281 };
4282
4283 APFloat FVal(0.0);
4284 switch (N->getOpcode()) {
4285 case AArch64ISD::MOVIshift:
4286 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(i: 0)
4287 << N.getConstantOperandVal(i: 1)));
4288 break;
4289 case AArch64ISD::FMOV:
4290 FVal = ImmToFloat(DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth));
4291 break;
4292 case AArch64ISD::DUP:
4293 if (isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
4294 FVal = ImmToFloat(N.getConstantOperandAPInt(i: 0).trunc(width: RegWidth));
4295 else
4296 return false;
4297 break;
4298 default:
4299 return false;
4300 }
4301
4302 if (unsigned FBits =
4303 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4304 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4305 return true;
4306 }
4307
4308 return false;
4309}
4310
4311bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4312 unsigned RegWidth) {
4313 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4314 /*isReciprocal*/ false);
4315}
4316
4317bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4318 unsigned RegWidth) {
4319 return checkCVTFixedPointOperandWithFBitsForVectors(
4320 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4321}
4322
4323bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4324 SDValue &FixedPos,
4325 unsigned RegWidth) {
4326 return checkCVTFixedPointOperandWithFBitsForVectors(
4327 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4328}
4329
4330bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4331 SDValue &FixedPos,
4332 unsigned RegWidth) {
4333 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4334 /*isReciprocal*/ true);
4335}
4336
4337// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4338// of the string and obtains the integer values from them and combines these
4339// into a single value to be used in the MRS/MSR instruction.
4340static int getIntOperandFromRegisterString(StringRef RegString) {
4341 SmallVector<StringRef, 5> Fields;
4342 RegString.split(A&: Fields, Separator: ':');
4343
4344 if (Fields.size() == 1)
4345 return -1;
4346
4347 assert(Fields.size() == 5
4348 && "Invalid number of fields in read register string");
4349
4350 SmallVector<int, 5> Ops;
4351 bool AllIntFields = true;
4352
4353 for (StringRef Field : Fields) {
4354 unsigned IntField;
4355 AllIntFields &= !Field.getAsInteger(Radix: 10, Result&: IntField);
4356 Ops.push_back(Elt: IntField);
4357 }
4358
4359 assert(AllIntFields &&
4360 "Unexpected non-integer value in special register string.");
4361 (void)AllIntFields;
4362
4363 // Need to combine the integer fields of the string into a single value
4364 // based on the bit encoding of MRS/MSR instruction.
4365 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4366 (Ops[4]);
4367}
4368
4369// Lower the read_register intrinsic to an MRS instruction node if the special
4370// register string argument is either of the form detailed in the ALCE (the
4371// form described in getIntOperandsFromRegisterString) or is a named register
4372// known by the MRS SysReg mapper.
4373bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4374 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4375 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4376 SDLoc DL(N);
4377
4378 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4379
4380 unsigned Opcode64Bit = AArch64::MRS;
4381 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4382 if (Imm == -1) {
4383 // No match, Use the sysreg mapper to map the remaining possible strings to
4384 // the value for the register to be used for the instruction operand.
4385 const auto *TheReg =
4386 AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4387 if (TheReg && TheReg->Readable &&
4388 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4389 Imm = TheReg->Encoding;
4390 else
4391 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4392
4393 if (Imm == -1) {
4394 // Still no match, see if this is "pc" or give up.
4395 if (!ReadIs128Bit && RegString->getString() == "pc") {
4396 Opcode64Bit = AArch64::ADR;
4397 Imm = 0;
4398 } else {
4399 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4400 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4401 // carries the source register as an immediate so the read does not
4402 // reference an undefined physical register (which the machine verifier
4403 // rejects); the AsmPrinter materializes the real mov/fmov.
4404 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4405 RegName: RegString->getString());
4406 unsigned PseudoOp = 0;
4407 if (AArch64::GPR64RegClass.contains(Reg: PReg))
4408 PseudoOp = AArch64::READ_REGISTER_GPR64;
4409 else if (AArch64::FPR64RegClass.contains(Reg: PReg))
4410 PseudoOp = AArch64::READ_REGISTER_FPR64;
4411 if (!ReadIs128Bit && PseudoOp && N->getValueType(ResNo: 0) == MVT::i64) {
4412 CurDAG->SelectNodeTo(N, MachineOpc: PseudoOp, VT1: MVT::i64, VT2: MVT::Other,
4413 Ops: {CurDAG->getTargetConstant(Val: PReg, DL, VT: MVT::i32),
4414 N->getOperand(Num: 0)});
4415 return true;
4416 }
4417 return false;
4418 }
4419 }
4420 }
4421
4422 SDValue InChain = N->getOperand(Num: 0);
4423 SDValue SysRegImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
4424 if (!ReadIs128Bit) {
4425 CurDAG->SelectNodeTo(N, MachineOpc: Opcode64Bit, VT1: MVT::i64, VT2: MVT::Other /* Chain */,
4426 Ops: {SysRegImm, InChain});
4427 } else {
4428 SDNode *MRRS = CurDAG->getMachineNode(
4429 Opcode: AArch64::MRRS, dl: DL,
4430 ResultTys: {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4431 Ops: {SysRegImm, InChain});
4432
4433 // Sysregs are not endian. The even register always contains the low half
4434 // of the register.
4435 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sube64, DL, VT: MVT::i64,
4436 Operand: SDValue(MRRS, 0));
4437 SDValue Hi = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::subo64, DL, VT: MVT::i64,
4438 Operand: SDValue(MRRS, 0));
4439 SDValue OutChain = SDValue(MRRS, 1);
4440
4441 ReplaceUses(F: SDValue(N, 0), T: Lo);
4442 ReplaceUses(F: SDValue(N, 1), T: Hi);
4443 ReplaceUses(F: SDValue(N, 2), T: OutChain);
4444 };
4445 return true;
4446}
4447
4448// Lower the write_register intrinsic to an MSR instruction node if the special
4449// register string argument is either of the form detailed in the ALCE (the
4450// form described in getIntOperandsFromRegisterString) or is a named register
4451// known by the MSR SysReg mapper.
4452bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4453 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4454 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4455 SDLoc DL(N);
4456
4457 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4458
4459 if (!WriteIs128Bit) {
4460 // Check if the register was one of those allowed as the pstatefield value
4461 // in the MSR (immediate) instruction. To accept the values allowed in the
4462 // pstatefield for the MSR (immediate) instruction, we also require that an
4463 // immediate value has been provided as an argument, we know that this is
4464 // the case as it has been ensured by semantic checking.
4465 auto trySelectPState = [&](auto PMapper, unsigned State) {
4466 if (PMapper) {
4467 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4468 "Expected a constant integer expression.");
4469 unsigned Reg = PMapper->Encoding;
4470 uint64_t Immed = N->getConstantOperandVal(Num: 2);
4471 CurDAG->SelectNodeTo(
4472 N, MachineOpc: State, VT: MVT::Other, Op1: CurDAG->getTargetConstant(Val: Reg, DL, VT: MVT::i32),
4473 Op2: CurDAG->getTargetConstant(Val: Immed, DL, VT: MVT::i16), Op3: N->getOperand(Num: 0));
4474 return true;
4475 }
4476 return false;
4477 };
4478
4479 if (trySelectPState(
4480 AArch64PState::lookupPStateImm0_15ByName(Name: RegString->getString()),
4481 AArch64::MSRpstateImm4))
4482 return true;
4483 if (trySelectPState(
4484 AArch64PState::lookupPStateImm0_1ByName(Name: RegString->getString()),
4485 AArch64::MSRpstateImm1))
4486 return true;
4487 }
4488
4489 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4490 if (Imm == -1) {
4491 // Use the sysreg mapper to attempt to map the remaining possible strings
4492 // to the value for the register to be used for the MSR (register)
4493 // instruction operand.
4494 auto TheReg = AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4495 if (TheReg && TheReg->Writeable &&
4496 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4497 Imm = TheReg->Encoding;
4498 else
4499 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4500
4501 if (Imm == -1) {
4502 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4503 // the physical register and keep it live with a FAKE_USE so the write is
4504 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4505 // so the generic write path cannot handle these.)
4506 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4507 RegName: RegString->getString());
4508 bool IsGPR = AArch64::GPR64RegClass.contains(Reg: PReg);
4509 bool IsFPR = AArch64::FPR64RegClass.contains(Reg: PReg);
4510 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4511 N->getOperand(Num: 2).getValueType() == MVT::i64) {
4512 SDValue Copy =
4513 CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: DL, Reg: PReg, N: N->getOperand(Num: 2));
4514 SDValue RegOp = CurDAG->getRegister(Reg: PReg, VT: MVT::i64);
4515 SDNode *FakeUse = CurDAG->getMachineNode(Opcode: TargetOpcode::FAKE_USE, dl: DL,
4516 VT: MVT::Other, Ops: {RegOp, Copy});
4517 ReplaceUses(F: SDValue(N, 0), T: SDValue(FakeUse, 0));
4518 CurDAG->RemoveDeadNode(N);
4519 return true;
4520 }
4521 return false;
4522 }
4523 }
4524
4525 SDValue InChain = N->getOperand(Num: 0);
4526 if (!WriteIs128Bit) {
4527 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSR, VT: MVT::Other,
4528 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4529 Op2: N->getOperand(Num: 2), Op3: InChain);
4530 } else {
4531 // No endian swap. The lower half always goes into the even subreg, and the
4532 // higher half always into the odd supreg.
4533 SDNode *Pair = CurDAG->getMachineNode(
4534 Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped /* XSeqPair */,
4535 Ops: {CurDAG->getTargetConstant(Val: AArch64::XSeqPairsClassRegClass.getID(), DL,
4536 VT: MVT::i32),
4537 N->getOperand(Num: 2),
4538 CurDAG->getTargetConstant(Val: AArch64::sube64, DL, VT: MVT::i32),
4539 N->getOperand(Num: 3),
4540 CurDAG->getTargetConstant(Val: AArch64::subo64, DL, VT: MVT::i32)});
4541
4542 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSRR, VT: MVT::Other,
4543 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4544 Op2: SDValue(Pair, 0), Op3: InChain);
4545 }
4546
4547 return true;
4548}
4549
4550/// We've got special pseudo-instructions for these
4551bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4552 unsigned Opcode;
4553 EVT MemTy = cast<MemSDNode>(Val: N)->getMemoryVT();
4554
4555 // Leave IR for LSE if subtarget supports it.
4556 if (Subtarget->hasLSE()) return false;
4557
4558 if (MemTy == MVT::i8)
4559 Opcode = AArch64::CMP_SWAP_8;
4560 else if (MemTy == MVT::i16)
4561 Opcode = AArch64::CMP_SWAP_16;
4562 else if (MemTy == MVT::i32)
4563 Opcode = AArch64::CMP_SWAP_32;
4564 else if (MemTy == MVT::i64)
4565 Opcode = AArch64::CMP_SWAP_64;
4566 else
4567 llvm_unreachable("Unknown AtomicCmpSwap type");
4568
4569 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4570 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3),
4571 N->getOperand(Num: 0)};
4572 SDNode *CmpSwap = CurDAG->getMachineNode(
4573 Opcode, dl: SDLoc(N),
4574 VTs: CurDAG->getVTList(VT1: RegTy, VT2: MVT::i32, VT3: MVT::Other), Ops);
4575
4576 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
4577 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
4578
4579 ReplaceUses(F: SDValue(N, 0), T: SDValue(CmpSwap, 0));
4580 ReplaceUses(F: SDValue(N, 1), T: SDValue(CmpSwap, 2));
4581 CurDAG->RemoveDeadNode(N);
4582
4583 return true;
4584}
4585
4586AArch64MemoryHint
4587AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
4588 int MemoryHint = -1;
4589 const MDNode *MemCacheHint = MMO->getMemCacheHint();
4590 if (!MemCacheHint)
4591 return AArch64MemoryHint::NONE;
4592
4593 for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
4594 if (MemCacheHint->getOperand(I).equalsStr(Str: "aarch64.mem_hint")) {
4595 const Metadata *Val = MemCacheHint->getOperand(I: I + 1).get();
4596 MemoryHint = cast<ConstantInt>(Val: cast<ConstantAsMetadata>(Val)->getValue())
4597 ->getZExtValue();
4598 }
4599 }
4600
4601 return toAArch64MemoryHint(I: MemoryHint);
4602}
4603
4604bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N,
4605 AArch64MemoryHint Hint) const {
4606 return decodeMemoryHintFlags(MMO: cast<MemSDNode>(Val: N)->getMemOperand()) == Hint;
4607}
4608
4609bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4610 SDValue &Shift, bool Negate) {
4611 if (!isa<ConstantSDNode>(Val: N))
4612 return false;
4613
4614 APInt Val =
4615 cast<ConstantSDNode>(Val&: N)->getAPIntValue().trunc(width: VT.getFixedSizeInBits());
4616
4617 return SelectSVEAddSubImm(DL: SDLoc(N), Value: Val, VT, Imm, Shift, Negate);
4618}
4619
4620bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4621 SDValue &Imm, SDValue &Shift,
4622 bool Negate) {
4623 if (Negate)
4624 Val = -Val;
4625
4626 switch (VT.SimpleTy) {
4627 case MVT::i8:
4628 // All immediates are supported.
4629 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4630 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4631 return true;
4632 case MVT::i16:
4633 case MVT::i32:
4634 case MVT::i64:
4635 // Support 8bit unsigned immediates.
4636 if ((Val & ~0xff) == 0) {
4637 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4638 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4639 return true;
4640 }
4641 // Support 16bit unsigned immediates that are a multiple of 256.
4642 if ((Val & ~0xff00) == 0) {
4643 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4644 Imm = CurDAG->getTargetConstant(Val: Val.lshr(shiftAmt: 8).getZExtValue(), DL, VT: MVT::i32);
4645 return true;
4646 }
4647 break;
4648 default:
4649 break;
4650 }
4651
4652 return false;
4653}
4654
4655bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4656 SDValue &Imm, SDValue &Shift,
4657 bool Negate) {
4658 if (!isa<ConstantSDNode>(Val: N))
4659 return false;
4660
4661 SDLoc DL(N);
4662 int64_t Val = cast<ConstantSDNode>(Val&: N)
4663 ->getAPIntValue()
4664 .trunc(width: VT.getFixedSizeInBits())
4665 .getSExtValue();
4666
4667 if (Negate)
4668 Val = -Val;
4669
4670 // Signed saturating instructions treat their immediate operand as unsigned,
4671 // whereas the related intrinsics define their operands to be signed. This
4672 // means we can only use the immediate form when the operand is non-negative.
4673 if (Val < 0)
4674 return false;
4675
4676 switch (VT.SimpleTy) {
4677 case MVT::i8:
4678 // All positive immediates are supported.
4679 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4680 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4681 return true;
4682 case MVT::i16:
4683 case MVT::i32:
4684 case MVT::i64:
4685 // Support 8bit positive immediates.
4686 if (Val <= 255) {
4687 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4688 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4689 return true;
4690 }
4691 // Support 16bit positive immediates that are a multiple of 256.
4692 if (Val <= 65280 && Val % 256 == 0) {
4693 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4694 Imm = CurDAG->getTargetConstant(Val: Val >> 8, DL, VT: MVT::i32);
4695 return true;
4696 }
4697 break;
4698 default:
4699 break;
4700 }
4701
4702 return false;
4703}
4704
4705bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4706 SDValue &Shift) {
4707 if (!isa<ConstantSDNode>(Val: N))
4708 return false;
4709
4710 SDLoc DL(N);
4711 int64_t Val = cast<ConstantSDNode>(Val&: N)
4712 ->getAPIntValue()
4713 .trunc(width: VT.getFixedSizeInBits())
4714 .getSExtValue();
4715 int32_t ImmVal, ShiftVal;
4716 if (!AArch64_AM::isSVECpyDupImm(SizeInBits: VT.getScalarSizeInBits(), Val, Imm&: ImmVal,
4717 Shift&: ShiftVal))
4718 return false;
4719
4720 Shift = CurDAG->getTargetConstant(Val: ShiftVal, DL, VT: MVT::i32);
4721 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4722 return true;
4723}
4724
4725bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4726 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N))
4727 return SelectSVESignedArithImm(DL: SDLoc(N), Value: CNode->getAPIntValue(), Imm);
4728 return false;
4729}
4730
4731bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4732 SDValue &Imm) {
4733 int64_t ImmVal = Val.getSExtValue();
4734 if (ImmVal >= -128 && ImmVal < 128) {
4735 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4736 return true;
4737 }
4738 return false;
4739}
4740
4741bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4742 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N)) {
4743 uint64_t ImmVal = CNode->getZExtValue();
4744
4745 switch (VT.SimpleTy) {
4746 case MVT::i8:
4747 ImmVal &= 0xFF;
4748 break;
4749 case MVT::i16:
4750 ImmVal &= 0xFFFF;
4751 break;
4752 case MVT::i32:
4753 ImmVal &= 0xFFFFFFFF;
4754 break;
4755 case MVT::i64:
4756 break;
4757 default:
4758 llvm_unreachable("Unexpected type");
4759 }
4760
4761 if (ImmVal < 256) {
4762 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4763 return true;
4764 }
4765 }
4766 return false;
4767}
4768
4769bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4770 bool Invert) {
4771 uint64_t ImmVal;
4772 if (auto CI = dyn_cast<ConstantSDNode>(Val&: N))
4773 ImmVal = CI->getZExtValue();
4774 else if (auto CFP = dyn_cast<ConstantFPSDNode>(Val&: N))
4775 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4776 else
4777 return false;
4778
4779 if (Invert)
4780 ImmVal = ~ImmVal;
4781
4782 uint64_t encoding;
4783 if (!AArch64_AM::isSVELogicalImm(SizeInBits: VT.getScalarSizeInBits(), ImmVal, Encoding&: encoding))
4784 return false;
4785
4786 Imm = CurDAG->getTargetConstant(Val: encoding, DL: SDLoc(N), VT: MVT::i64);
4787 return true;
4788}
4789
4790// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4791// Rather than attempt to normalise everything we can sometimes saturate the
4792// shift amount during selection. This function also allows for consistent
4793// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4794// required by the instructions.
4795bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4796 uint64_t High, bool AllowSaturation,
4797 SDValue &Imm) {
4798 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
4799 uint64_t ImmVal = CN->getZExtValue();
4800
4801 // Reject shift amounts that are too small.
4802 if (ImmVal < Low)
4803 return false;
4804
4805 // Reject or saturate shift amounts that are too big.
4806 if (ImmVal > High) {
4807 if (!AllowSaturation)
4808 return false;
4809 ImmVal = High;
4810 }
4811
4812 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4813 return true;
4814 }
4815
4816 return false;
4817}
4818
4819bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4820 // tagp(FrameIndex, IRGstack, tag_offset):
4821 // since the offset between FrameIndex and IRGstack is a compile-time
4822 // constant, this can be lowered to a single ADDG instruction.
4823 if (!(isa<FrameIndexSDNode>(Val: N->getOperand(Num: 1)))) {
4824 return false;
4825 }
4826
4827 SDValue IRG_SP = N->getOperand(Num: 2);
4828 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4829 IRG_SP->getConstantOperandVal(Num: 1) != Intrinsic::aarch64_irg_sp) {
4830 return false;
4831 }
4832
4833 const TargetLowering *TLI = getTargetLowering();
4834 SDLoc DL(N);
4835 int FI = cast<FrameIndexSDNode>(Val: N->getOperand(Num: 1))->getIndex();
4836 SDValue FiOp = CurDAG->getTargetFrameIndex(
4837 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
4838 int TagOffset = N->getConstantOperandVal(Num: 3);
4839
4840 SDNode *Out = CurDAG->getMachineNode(
4841 Opcode: AArch64::TAGPstack, dl: DL, VT: MVT::i64,
4842 Ops: {FiOp, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), N->getOperand(Num: 2),
4843 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4844 ReplaceNode(F: N, T: Out);
4845 return true;
4846}
4847
4848void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4849 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4850 "llvm.aarch64.tagp third argument must be an immediate");
4851 if (trySelectStackSlotTagP(N))
4852 return;
4853 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4854 // compile-time constant, not just for stack allocations.
4855
4856 // General case for unrelated pointers in Op1 and Op2.
4857 SDLoc DL(N);
4858 int TagOffset = N->getConstantOperandVal(Num: 3);
4859 SDNode *N1 = CurDAG->getMachineNode(Opcode: AArch64::SUBP, dl: DL, VT: MVT::i64,
4860 Ops: {N->getOperand(Num: 1), N->getOperand(Num: 2)});
4861 SDNode *N2 = CurDAG->getMachineNode(Opcode: AArch64::ADDXrr, dl: DL, VT: MVT::i64,
4862 Ops: {SDValue(N1, 0), N->getOperand(Num: 2)});
4863 SDNode *N3 = CurDAG->getMachineNode(
4864 Opcode: AArch64::ADDG, dl: DL, VT: MVT::i64,
4865 Ops: {SDValue(N2, 0), CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64),
4866 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4867 ReplaceNode(F: N, T: N3);
4868}
4869
4870bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4871 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4872
4873 // Bail when not a "cast" like insert_subvector.
4874 if (N->getConstantOperandVal(Num: 2) != 0)
4875 return false;
4876 if (!N->getOperand(Num: 0).isUndef())
4877 return false;
4878
4879 // Bail when normal isel should do the job.
4880 EVT VT = N->getValueType(ResNo: 0);
4881 EVT InVT = N->getOperand(Num: 1).getValueType();
4882 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4883 return false;
4884 if (InVT.getSizeInBits() <= 128)
4885 return false;
4886
4887 // NOTE: We can only get here when doing fixed length SVE code generation.
4888 // We do manual selection because the types involved are not linked to real
4889 // registers (despite being legal) and must be coerced into SVE registers.
4890
4891 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4892 "Expected to insert into a packed scalable vector!");
4893
4894 SDLoc DL(N);
4895 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4896 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4897 Op1: N->getOperand(Num: 1), Op2: RC));
4898 return true;
4899}
4900
4901bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4902 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4903
4904 // Bail when not a "cast" like extract_subvector.
4905 if (N->getConstantOperandVal(Num: 1) != 0)
4906 return false;
4907
4908 // Bail when normal isel can do the job.
4909 EVT VT = N->getValueType(ResNo: 0);
4910 EVT InVT = N->getOperand(Num: 0).getValueType();
4911 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4912 return false;
4913 if (VT.getSizeInBits() <= 128)
4914 return false;
4915
4916 // NOTE: We can only get here when doing fixed length SVE code generation.
4917 // We do manual selection because the types involved are not linked to real
4918 // registers (despite being legal) and must be coerced into SVE registers.
4919
4920 assert(InVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4921 "Expected to extract from a packed scalable vector!");
4922
4923 SDLoc DL(N);
4924 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4925 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4926 Op1: N->getOperand(Num: 0), Op2: RC));
4927 return true;
4928}
4929
4930bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4931 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4932
4933 SDValue N0 = N->getOperand(Num: 0);
4934 SDValue N1 = N->getOperand(Num: 1);
4935
4936 EVT VT = N->getValueType(ResNo: 0);
4937 SDLoc DL(N);
4938
4939 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4940 // Rotate by a constant is a funnel shift in IR which is expanded to
4941 // an OR with shifted operands.
4942 // We do the following transform:
4943 // OR N0, N1 -> xar (x, y, imm)
4944 // Where:
4945 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4946 // N0 = SHL_PRED true, V, splat(bits-imm)
4947 // V = (xor x, y)
4948 if (VT.isScalableVector() &&
4949 (Subtarget->hasSVE2() ||
4950 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4951 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4952 N1.getOpcode() != AArch64ISD::SRL_PRED)
4953 std::swap(a&: N0, b&: N1);
4954 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4955 N1.getOpcode() != AArch64ISD::SRL_PRED)
4956 return false;
4957
4958 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4959 if (!TLI->isAllActivePredicate(DAG: *CurDAG, N: N0.getOperand(i: 0)) ||
4960 !TLI->isAllActivePredicate(DAG: *CurDAG, N: N1.getOperand(i: 0)))
4961 return false;
4962
4963 if (N0.getOperand(i: 1) != N1.getOperand(i: 1))
4964 return false;
4965
4966 SDValue R1, R2;
4967 bool IsXOROperand = true;
4968 if (N0.getOperand(i: 1).getOpcode() != ISD::XOR) {
4969 IsXOROperand = false;
4970 } else {
4971 R1 = N0.getOperand(i: 1).getOperand(i: 0);
4972 R2 = N1.getOperand(i: 1).getOperand(i: 1);
4973 }
4974
4975 APInt ShlAmt, ShrAmt;
4976 if (!ISD::isConstantSplatVector(V: N0.getOperand(i: 2), SplatValue&: ShlAmt) ||
4977 !ISD::isConstantSplatVector(V: N1.getOperand(i: 2), SplatValue&: ShrAmt))
4978 return false;
4979
4980 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4981 return false;
4982
4983 if (!IsXOROperand) {
4984 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
4985 SDNode *MOV = CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT, Op1: Zero);
4986 SDValue MOVIV = SDValue(MOV, 0);
4987
4988 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
4989 SDNode *SubRegToReg =
4990 CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT, Op1: MOVIV, Op2: ZSub);
4991
4992 R1 = N1->getOperand(Num: 1);
4993 R2 = SDValue(SubRegToReg, 0);
4994 }
4995
4996 SDValue Imm =
4997 CurDAG->getTargetConstant(Val: ShrAmt.getZExtValue(), DL, VT: MVT::i32);
4998
4999 SDValue Ops[] = {R1, R2, Imm};
5000 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5001 VT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5002 AArch64::XAR_ZZZI_D})) {
5003 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
5004 return true;
5005 }
5006 return false;
5007 }
5008
5009 // We have Neon SHA3 XAR operation for v2i64 but for types
5010 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
5011 // is available.
5012 EVT SVT;
5013 switch (VT.getSimpleVT().SimpleTy) {
5014 case MVT::v4i32:
5015 case MVT::v2i32:
5016 SVT = MVT::nxv4i32;
5017 break;
5018 case MVT::v8i16:
5019 case MVT::v4i16:
5020 SVT = MVT::nxv8i16;
5021 break;
5022 case MVT::v16i8:
5023 case MVT::v8i8:
5024 SVT = MVT::nxv16i8;
5025 break;
5026 case MVT::v2i64:
5027 case MVT::v1i64:
5028 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5029 break;
5030 default:
5031 return false;
5032 }
5033
5034 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
5035 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
5036 return false;
5037
5038 if (N0->getOpcode() != AArch64ISD::VSHL ||
5039 N1->getOpcode() != AArch64ISD::VLSHR)
5040 return false;
5041
5042 if (N0->getOperand(Num: 0) != N1->getOperand(Num: 0))
5043 return false;
5044
5045 SDValue R1, R2;
5046 bool IsXOROperand = true;
5047 if (N1->getOperand(Num: 0)->getOpcode() != ISD::XOR) {
5048 IsXOROperand = false;
5049 } else {
5050 SDValue XOR = N0.getOperand(i: 0);
5051 R1 = XOR.getOperand(i: 0);
5052 R2 = XOR.getOperand(i: 1);
5053 }
5054
5055 unsigned HsAmt = N0.getConstantOperandVal(i: 1);
5056 unsigned ShAmt = N1.getConstantOperandVal(i: 1);
5057
5058 SDValue Imm = CurDAG->getTargetConstant(
5059 Val: ShAmt, DL, VT: N0.getOperand(i: 1).getValueType(), isOpaque: false);
5060
5061 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5062 if (ShAmt + HsAmt != VTSizeInBits)
5063 return false;
5064
5065 if (!IsXOROperand) {
5066 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
5067 SDNode *MOV =
5068 CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT: MVT::v2i64, Op1: Zero);
5069 SDValue MOVIV = SDValue(MOV, 0);
5070
5071 R1 = N1->getOperand(Num: 0);
5072 R2 = MOVIV;
5073 }
5074
5075 if (SVT != VT) {
5076 SDValue Undef =
5077 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: SVT), 0);
5078
5079 if (SVT.isScalableVector() && VT.is64BitVector()) {
5080 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5081
5082 SDValue UndefQ = SDValue(
5083 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: QVT), 0);
5084 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5085
5086 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5087 Op1: UndefQ, Op2: R1, Op3: DSub),
5088 0);
5089 if (R2.getValueType() == VT)
5090 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5091 Op1: UndefQ, Op2: R2, Op3: DSub),
5092 0);
5093 }
5094
5095 SDValue SubReg = CurDAG->getTargetConstant(
5096 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, VT: MVT::i32);
5097
5098 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT, Op1: Undef,
5099 Op2: R1, Op3: SubReg),
5100 0);
5101
5102 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5103 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT,
5104 Op1: Undef, Op2: R2, Op3: SubReg),
5105 0);
5106 }
5107
5108 SDValue Ops[] = {R1, R2, Imm};
5109 SDNode *XAR = nullptr;
5110
5111 if (SVT.isScalableVector()) {
5112 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5113 VT: SVT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5114 AArch64::XAR_ZZZI_D}))
5115 XAR = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: SVT, Ops);
5116 } else {
5117 XAR = CurDAG->getMachineNode(Opcode: AArch64::XAR, dl: DL, VT: SVT, Ops);
5118 }
5119
5120 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5121
5122 if (SVT != VT) {
5123 if (VT.is64BitVector() && SVT.isScalableVector()) {
5124 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5125
5126 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
5127 SDNode *Q = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT: QVT,
5128 Op1: SDValue(XAR, 0), Op2: ZSub);
5129
5130 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5131 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5132 Op1: SDValue(Q, 0), Op2: DSub);
5133 } else {
5134 SDValue SubReg = CurDAG->getTargetConstant(
5135 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5136 VT: MVT::i32);
5137 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5138 Op1: SDValue(XAR, 0), Op2: SubReg);
5139 }
5140 }
5141 ReplaceNode(F: N, T: XAR);
5142 return true;
5143}
5144
5145/// Returns a copy from WZR or XZR. This can be used during instruction
5146/// selection (it does not require any further selection/legalization).
5147static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
5148 assert(VT == MVT::i32 || VT == MVT::i64);
5149 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
5150 Reg: VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5151}
5152
5153void AArch64DAGToDAGISel::Select(SDNode *Node) {
5154 // If we have a custom node, we already have selected!
5155 if (Node->isMachineOpcode()) {
5156 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5157 Node->setNodeId(-1);
5158 return;
5159 }
5160
5161 // Few custom selection stuff.
5162 EVT VT = Node->getValueType(ResNo: 0);
5163
5164 switch (Node->getOpcode()) {
5165 default:
5166 break;
5167
5168 case ISD::ATOMIC_CMP_SWAP:
5169 if (SelectCMP_SWAP(N: Node))
5170 return;
5171 break;
5172
5173 case ISD::READ_REGISTER:
5174 case AArch64ISD::MRRS:
5175 if (tryReadRegister(N: Node))
5176 return;
5177 break;
5178
5179 case ISD::WRITE_REGISTER:
5180 case AArch64ISD::MSRR:
5181 if (tryWriteRegister(N: Node))
5182 return;
5183 break;
5184
5185 case ISD::LOAD: {
5186 // Try to select as an indexed load. Fall through to normal processing
5187 // if we can't.
5188 if (tryIndexedLoad(N: Node))
5189 return;
5190 break;
5191 }
5192
5193 case ISD::SRL:
5194 case ISD::AND:
5195 case ISD::SRA:
5196 case ISD::SIGN_EXTEND_INREG:
5197 if (tryBitfieldExtractOp(N: Node))
5198 return;
5199 if (tryBitfieldInsertInZeroOp(N: Node))
5200 return;
5201 [[fallthrough]];
5202 case ISD::ROTR:
5203 case ISD::SHL:
5204 break;
5205
5206 case ISD::OR:
5207 if (tryBitfieldInsertOp(N: Node))
5208 return;
5209 if (trySelectXAR(N: Node))
5210 return;
5211 break;
5212
5213 case ISD::EXTRACT_SUBVECTOR: {
5214 if (trySelectCastScalableToFixedLengthVector(N: Node))
5215 return;
5216 break;
5217 }
5218
5219 case ISD::INSERT_SUBVECTOR: {
5220 if (trySelectCastFixedLengthToScalableVector(N: Node))
5221 return;
5222 break;
5223 }
5224
5225 case AArch64ISD::CSEL:
5226 if (tryFoldCselToFMaxMin(N: Node))
5227 return;
5228 break;
5229
5230 case ISD::Constant: {
5231 // Materialize zero constants as copies from WZR/XZR. This allows
5232 // the coalescer to propagate these into other instructions.
5233 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Val: Node);
5234 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5235 ReplaceNode(F: Node, T: getZeroRegister(DAG&: *CurDAG, DL: SDLoc(Node), VT).getNode());
5236 return;
5237 }
5238 break;
5239 }
5240
5241 case ISD::FrameIndex: {
5242 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5243 int FI = cast<FrameIndexSDNode>(Val: Node)->getIndex();
5244 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
5245 const TargetLowering *TLI = getTargetLowering();
5246 SDValue TFI = CurDAG->getTargetFrameIndex(
5247 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
5248 SDLoc DL(Node);
5249 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32),
5250 CurDAG->getTargetConstant(Val: Shifter, DL, VT: MVT::i32) };
5251 CurDAG->SelectNodeTo(N: Node, MachineOpc: AArch64::ADDXri, VT: MVT::i64, Ops);
5252 return;
5253 }
5254 case ISD::INTRINSIC_W_CHAIN: {
5255 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5256 switch (IntNo) {
5257 default:
5258 break;
5259 case Intrinsic::aarch64_gcsss: {
5260 SDLoc DL(Node);
5261 SDValue Chain = Node->getOperand(Num: 0);
5262 SDValue Val = Node->getOperand(Num: 2);
5263 SDValue Zero = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::XZR, VT: MVT::i64);
5264 SDNode *SS1 =
5265 CurDAG->getMachineNode(Opcode: AArch64::GCSSS1, dl: DL, VT: MVT::Other, Op1: Val, Op2: Chain);
5266 SDNode *SS2 = CurDAG->getMachineNode(Opcode: AArch64::GCSSS2, dl: DL, VT1: MVT::i64,
5267 VT2: MVT::Other, Op1: Zero, Op2: SDValue(SS1, 0));
5268 ReplaceNode(F: Node, T: SS2);
5269 return;
5270 }
5271 case Intrinsic::aarch64_ldaxp:
5272 case Intrinsic::aarch64_ldxp: {
5273 unsigned Op =
5274 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5275 SDValue MemAddr = Node->getOperand(Num: 2);
5276 SDLoc DL(Node);
5277 SDValue Chain = Node->getOperand(Num: 0);
5278
5279 SDNode *Ld = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i64, VT2: MVT::i64,
5280 VT3: MVT::Other, Op1: MemAddr, Op2: Chain);
5281
5282 // Transfer memoperands.
5283 MachineMemOperand *MemOp =
5284 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5285 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
5286 ReplaceNode(F: Node, T: Ld);
5287 return;
5288 }
5289 case Intrinsic::aarch64_stlxp:
5290 case Intrinsic::aarch64_stxp: {
5291 unsigned Op =
5292 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5293 SDLoc DL(Node);
5294 SDValue Chain = Node->getOperand(Num: 0);
5295 SDValue ValLo = Node->getOperand(Num: 2);
5296 SDValue ValHi = Node->getOperand(Num: 3);
5297 SDValue MemAddr = Node->getOperand(Num: 4);
5298
5299 // Place arguments in the right order.
5300 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5301
5302 SDNode *St = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
5303 // Transfer memoperands.
5304 MachineMemOperand *MemOp =
5305 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5306 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
5307
5308 ReplaceNode(F: Node, T: St);
5309 return;
5310 }
5311 case Intrinsic::aarch64_neon_ld1x2:
5312 if (VT == MVT::v8i8) {
5313 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b, SubRegIdx: AArch64::dsub0);
5314 return;
5315 } else if (VT == MVT::v16i8) {
5316 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b, SubRegIdx: AArch64::qsub0);
5317 return;
5318 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5319 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h, SubRegIdx: AArch64::dsub0);
5320 return;
5321 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5322 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h, SubRegIdx: AArch64::qsub0);
5323 return;
5324 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5325 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s, SubRegIdx: AArch64::dsub0);
5326 return;
5327 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5328 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s, SubRegIdx: AArch64::qsub0);
5329 return;
5330 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5331 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5332 return;
5333 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5334 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d, SubRegIdx: AArch64::qsub0);
5335 return;
5336 }
5337 break;
5338 case Intrinsic::aarch64_neon_ld1x3:
5339 if (VT == MVT::v8i8) {
5340 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b, SubRegIdx: AArch64::dsub0);
5341 return;
5342 } else if (VT == MVT::v16i8) {
5343 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b, SubRegIdx: AArch64::qsub0);
5344 return;
5345 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5346 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h, SubRegIdx: AArch64::dsub0);
5347 return;
5348 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5349 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h, SubRegIdx: AArch64::qsub0);
5350 return;
5351 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5352 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s, SubRegIdx: AArch64::dsub0);
5353 return;
5354 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5355 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s, SubRegIdx: AArch64::qsub0);
5356 return;
5357 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5358 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5359 return;
5360 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5361 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d, SubRegIdx: AArch64::qsub0);
5362 return;
5363 }
5364 break;
5365 case Intrinsic::aarch64_neon_ld1x4:
5366 if (VT == MVT::v8i8) {
5367 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b, SubRegIdx: AArch64::dsub0);
5368 return;
5369 } else if (VT == MVT::v16i8) {
5370 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b, SubRegIdx: AArch64::qsub0);
5371 return;
5372 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5373 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h, SubRegIdx: AArch64::dsub0);
5374 return;
5375 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5376 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h, SubRegIdx: AArch64::qsub0);
5377 return;
5378 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5379 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s, SubRegIdx: AArch64::dsub0);
5380 return;
5381 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5382 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s, SubRegIdx: AArch64::qsub0);
5383 return;
5384 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5385 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5386 return;
5387 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5388 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d, SubRegIdx: AArch64::qsub0);
5389 return;
5390 }
5391 break;
5392 case Intrinsic::aarch64_neon_ld2:
5393 if (VT == MVT::v8i8) {
5394 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b, SubRegIdx: AArch64::dsub0);
5395 return;
5396 } else if (VT == MVT::v16i8) {
5397 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b, SubRegIdx: AArch64::qsub0);
5398 return;
5399 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5400 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h, SubRegIdx: AArch64::dsub0);
5401 return;
5402 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5403 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h, SubRegIdx: AArch64::qsub0);
5404 return;
5405 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5406 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s, SubRegIdx: AArch64::dsub0);
5407 return;
5408 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5409 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s, SubRegIdx: AArch64::qsub0);
5410 return;
5411 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5412 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5413 return;
5414 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5415 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d, SubRegIdx: AArch64::qsub0);
5416 return;
5417 }
5418 break;
5419 case Intrinsic::aarch64_neon_ld3:
5420 if (VT == MVT::v8i8) {
5421 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b, SubRegIdx: AArch64::dsub0);
5422 return;
5423 } else if (VT == MVT::v16i8) {
5424 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b, SubRegIdx: AArch64::qsub0);
5425 return;
5426 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5427 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h, SubRegIdx: AArch64::dsub0);
5428 return;
5429 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5430 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h, SubRegIdx: AArch64::qsub0);
5431 return;
5432 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5433 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s, SubRegIdx: AArch64::dsub0);
5434 return;
5435 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5436 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s, SubRegIdx: AArch64::qsub0);
5437 return;
5438 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5439 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5440 return;
5441 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5442 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d, SubRegIdx: AArch64::qsub0);
5443 return;
5444 }
5445 break;
5446 case Intrinsic::aarch64_neon_ld4:
5447 if (VT == MVT::v8i8) {
5448 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b, SubRegIdx: AArch64::dsub0);
5449 return;
5450 } else if (VT == MVT::v16i8) {
5451 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b, SubRegIdx: AArch64::qsub0);
5452 return;
5453 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5454 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h, SubRegIdx: AArch64::dsub0);
5455 return;
5456 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5457 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h, SubRegIdx: AArch64::qsub0);
5458 return;
5459 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5460 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s, SubRegIdx: AArch64::dsub0);
5461 return;
5462 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5463 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s, SubRegIdx: AArch64::qsub0);
5464 return;
5465 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5466 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5467 return;
5468 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5469 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d, SubRegIdx: AArch64::qsub0);
5470 return;
5471 }
5472 break;
5473 case Intrinsic::aarch64_neon_ld2r:
5474 if (VT == MVT::v8i8) {
5475 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b, SubRegIdx: AArch64::dsub0);
5476 return;
5477 } else if (VT == MVT::v16i8) {
5478 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b, SubRegIdx: AArch64::qsub0);
5479 return;
5480 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5481 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h, SubRegIdx: AArch64::dsub0);
5482 return;
5483 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5484 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h, SubRegIdx: AArch64::qsub0);
5485 return;
5486 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5487 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s, SubRegIdx: AArch64::dsub0);
5488 return;
5489 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5490 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s, SubRegIdx: AArch64::qsub0);
5491 return;
5492 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5493 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d, SubRegIdx: AArch64::dsub0);
5494 return;
5495 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5496 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d, SubRegIdx: AArch64::qsub0);
5497 return;
5498 }
5499 break;
5500 case Intrinsic::aarch64_neon_ld3r:
5501 if (VT == MVT::v8i8) {
5502 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b, SubRegIdx: AArch64::dsub0);
5503 return;
5504 } else if (VT == MVT::v16i8) {
5505 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b, SubRegIdx: AArch64::qsub0);
5506 return;
5507 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5508 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h, SubRegIdx: AArch64::dsub0);
5509 return;
5510 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5511 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h, SubRegIdx: AArch64::qsub0);
5512 return;
5513 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5514 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s, SubRegIdx: AArch64::dsub0);
5515 return;
5516 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5517 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s, SubRegIdx: AArch64::qsub0);
5518 return;
5519 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5520 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d, SubRegIdx: AArch64::dsub0);
5521 return;
5522 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5523 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d, SubRegIdx: AArch64::qsub0);
5524 return;
5525 }
5526 break;
5527 case Intrinsic::aarch64_neon_ld4r:
5528 if (VT == MVT::v8i8) {
5529 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b, SubRegIdx: AArch64::dsub0);
5530 return;
5531 } else if (VT == MVT::v16i8) {
5532 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b, SubRegIdx: AArch64::qsub0);
5533 return;
5534 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5535 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h, SubRegIdx: AArch64::dsub0);
5536 return;
5537 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5538 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h, SubRegIdx: AArch64::qsub0);
5539 return;
5540 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5541 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s, SubRegIdx: AArch64::dsub0);
5542 return;
5543 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5544 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s, SubRegIdx: AArch64::qsub0);
5545 return;
5546 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5547 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d, SubRegIdx: AArch64::dsub0);
5548 return;
5549 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5550 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d, SubRegIdx: AArch64::qsub0);
5551 return;
5552 }
5553 break;
5554 case Intrinsic::aarch64_neon_ld2lane:
5555 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5556 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8);
5557 return;
5558 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5559 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5560 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16);
5561 return;
5562 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5563 VT == MVT::v2f32) {
5564 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32);
5565 return;
5566 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5567 VT == MVT::v1f64) {
5568 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64);
5569 return;
5570 }
5571 break;
5572 case Intrinsic::aarch64_neon_ld3lane:
5573 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5574 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8);
5575 return;
5576 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5577 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5578 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16);
5579 return;
5580 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5581 VT == MVT::v2f32) {
5582 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32);
5583 return;
5584 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5585 VT == MVT::v1f64) {
5586 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64);
5587 return;
5588 }
5589 break;
5590 case Intrinsic::aarch64_neon_ld4lane:
5591 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5592 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8);
5593 return;
5594 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5595 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5596 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16);
5597 return;
5598 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5599 VT == MVT::v2f32) {
5600 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32);
5601 return;
5602 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5603 VT == MVT::v1f64) {
5604 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64);
5605 return;
5606 }
5607 break;
5608 case Intrinsic::aarch64_ld64b:
5609 SelectLoad(N: Node, NumVecs: 8, Opc: AArch64::LD64B, SubRegIdx: AArch64::x8sub_0);
5610 return;
5611 case Intrinsic::aarch64_sve_ld2q_sret: {
5612 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 4, Opc_ri: AArch64::LD2Q_IMM, Opc_rr: AArch64::LD2Q, IsIntr: true);
5613 return;
5614 }
5615 case Intrinsic::aarch64_sve_ld3q_sret: {
5616 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 4, Opc_ri: AArch64::LD3Q_IMM, Opc_rr: AArch64::LD3Q, IsIntr: true);
5617 return;
5618 }
5619 case Intrinsic::aarch64_sve_ld4q_sret: {
5620 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 4, Opc_ri: AArch64::LD4Q_IMM, Opc_rr: AArch64::LD4Q, IsIntr: true);
5621 return;
5622 }
5623 case Intrinsic::aarch64_sve_ld2_sret: {
5624 if (VT == MVT::nxv16i8) {
5625 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD2B_IMM, Opc_rr: AArch64::LD2B,
5626 IsIntr: true);
5627 return;
5628 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5629 VT == MVT::nxv8bf16) {
5630 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD2H_IMM, Opc_rr: AArch64::LD2H,
5631 IsIntr: true);
5632 return;
5633 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5634 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD2W_IMM, Opc_rr: AArch64::LD2W,
5635 IsIntr: true);
5636 return;
5637 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5638 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD2D_IMM, Opc_rr: AArch64::LD2D,
5639 IsIntr: true);
5640 return;
5641 }
5642 break;
5643 }
5644 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5645 if (VT == MVT::nxv16i8) {
5646 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5647 SelectContiguousMultiVectorLoad(
5648 N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_2Z_PSEUDO);
5649 else if (Subtarget->hasSVE2p1())
5650 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM,
5651 Opc_rr: AArch64::LD1B_2Z);
5652 else
5653 break;
5654 return;
5655 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5656 VT == MVT::nxv8bf16) {
5657 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5658 SelectContiguousMultiVectorLoad(
5659 N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_2Z_PSEUDO);
5660 else if (Subtarget->hasSVE2p1())
5661 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM,
5662 Opc_rr: AArch64::LD1H_2Z);
5663 else
5664 break;
5665 return;
5666 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5667 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5668 SelectContiguousMultiVectorLoad(
5669 N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_2Z_PSEUDO);
5670 else if (Subtarget->hasSVE2p1())
5671 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM,
5672 Opc_rr: AArch64::LD1W_2Z);
5673 else
5674 break;
5675 return;
5676 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5677 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5678 SelectContiguousMultiVectorLoad(
5679 N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_2Z_PSEUDO);
5680 else if (Subtarget->hasSVE2p1())
5681 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM,
5682 Opc_rr: AArch64::LD1D_2Z);
5683 else
5684 break;
5685 return;
5686 }
5687 break;
5688 }
5689 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5690 if (VT == MVT::nxv16i8) {
5691 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5692 SelectContiguousMultiVectorLoad(
5693 N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_4Z_PSEUDO);
5694 else if (Subtarget->hasSVE2p1())
5695 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM,
5696 Opc_rr: AArch64::LD1B_4Z);
5697 else
5698 break;
5699 return;
5700 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5701 VT == MVT::nxv8bf16) {
5702 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5703 SelectContiguousMultiVectorLoad(
5704 N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_4Z_PSEUDO);
5705 else if (Subtarget->hasSVE2p1())
5706 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM,
5707 Opc_rr: AArch64::LD1H_4Z);
5708 else
5709 break;
5710 return;
5711 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5712 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5713 SelectContiguousMultiVectorLoad(
5714 N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_4Z_PSEUDO);
5715 else if (Subtarget->hasSVE2p1())
5716 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM,
5717 Opc_rr: AArch64::LD1W_4Z);
5718 else
5719 break;
5720 return;
5721 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5722 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5723 SelectContiguousMultiVectorLoad(
5724 N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_4Z_PSEUDO);
5725 else if (Subtarget->hasSVE2p1())
5726 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM,
5727 Opc_rr: AArch64::LD1D_4Z);
5728 else
5729 break;
5730 return;
5731 }
5732 break;
5733 }
5734 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5735 if (VT == MVT::nxv16i8) {
5736 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5737 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0,
5738 Opc_ri: AArch64::LDNT1B_2Z_IMM_PSEUDO,
5739 Opc_rr: AArch64::LDNT1B_2Z_PSEUDO);
5740 else if (Subtarget->hasSVE2p1())
5741 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LDNT1B_2Z_IMM,
5742 Opc_rr: AArch64::LDNT1B_2Z);
5743 else
5744 break;
5745 return;
5746 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5747 VT == MVT::nxv8bf16) {
5748 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5749 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1,
5750 Opc_ri: AArch64::LDNT1H_2Z_IMM_PSEUDO,
5751 Opc_rr: AArch64::LDNT1H_2Z_PSEUDO);
5752 else if (Subtarget->hasSVE2p1())
5753 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LDNT1H_2Z_IMM,
5754 Opc_rr: AArch64::LDNT1H_2Z);
5755 else
5756 break;
5757 return;
5758 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5759 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5760 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2,
5761 Opc_ri: AArch64::LDNT1W_2Z_IMM_PSEUDO,
5762 Opc_rr: AArch64::LDNT1W_2Z_PSEUDO);
5763 else if (Subtarget->hasSVE2p1())
5764 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LDNT1W_2Z_IMM,
5765 Opc_rr: AArch64::LDNT1W_2Z);
5766 else
5767 break;
5768 return;
5769 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5770 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5771 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3,
5772 Opc_ri: AArch64::LDNT1D_2Z_IMM_PSEUDO,
5773 Opc_rr: AArch64::LDNT1D_2Z_PSEUDO);
5774 else if (Subtarget->hasSVE2p1())
5775 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LDNT1D_2Z_IMM,
5776 Opc_rr: AArch64::LDNT1D_2Z);
5777 else
5778 break;
5779 return;
5780 }
5781 break;
5782 }
5783 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5784 if (VT == MVT::nxv16i8) {
5785 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5786 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0,
5787 Opc_ri: AArch64::LDNT1B_4Z_IMM_PSEUDO,
5788 Opc_rr: AArch64::LDNT1B_4Z_PSEUDO);
5789 else if (Subtarget->hasSVE2p1())
5790 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LDNT1B_4Z_IMM,
5791 Opc_rr: AArch64::LDNT1B_4Z);
5792 else
5793 break;
5794 return;
5795 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5796 VT == MVT::nxv8bf16) {
5797 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5798 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1,
5799 Opc_ri: AArch64::LDNT1H_4Z_IMM_PSEUDO,
5800 Opc_rr: AArch64::LDNT1H_4Z_PSEUDO);
5801 else if (Subtarget->hasSVE2p1())
5802 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LDNT1H_4Z_IMM,
5803 Opc_rr: AArch64::LDNT1H_4Z);
5804 else
5805 break;
5806 return;
5807 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5808 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5809 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2,
5810 Opc_ri: AArch64::LDNT1W_4Z_IMM_PSEUDO,
5811 Opc_rr: AArch64::LDNT1W_4Z_PSEUDO);
5812 else if (Subtarget->hasSVE2p1())
5813 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LDNT1W_4Z_IMM,
5814 Opc_rr: AArch64::LDNT1W_4Z);
5815 else
5816 break;
5817 return;
5818 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5819 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5820 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3,
5821 Opc_ri: AArch64::LDNT1D_4Z_IMM_PSEUDO,
5822 Opc_rr: AArch64::LDNT1D_4Z_PSEUDO);
5823 else if (Subtarget->hasSVE2p1())
5824 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LDNT1D_4Z_IMM,
5825 Opc_rr: AArch64::LDNT1D_4Z);
5826 else
5827 break;
5828 return;
5829 }
5830 break;
5831 }
5832 case Intrinsic::aarch64_sve_ld3_sret: {
5833 if (VT == MVT::nxv16i8) {
5834 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 0, Opc_ri: AArch64::LD3B_IMM, Opc_rr: AArch64::LD3B,
5835 IsIntr: true);
5836 return;
5837 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5838 VT == MVT::nxv8bf16) {
5839 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 1, Opc_ri: AArch64::LD3H_IMM, Opc_rr: AArch64::LD3H,
5840 IsIntr: true);
5841 return;
5842 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5843 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 2, Opc_ri: AArch64::LD3W_IMM, Opc_rr: AArch64::LD3W,
5844 IsIntr: true);
5845 return;
5846 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5847 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 3, Opc_ri: AArch64::LD3D_IMM, Opc_rr: AArch64::LD3D,
5848 IsIntr: true);
5849 return;
5850 }
5851 break;
5852 }
5853 case Intrinsic::aarch64_sve_ld4_sret: {
5854 if (VT == MVT::nxv16i8) {
5855 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD4B_IMM, Opc_rr: AArch64::LD4B,
5856 IsIntr: true);
5857 return;
5858 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5859 VT == MVT::nxv8bf16) {
5860 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD4H_IMM, Opc_rr: AArch64::LD4H,
5861 IsIntr: true);
5862 return;
5863 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5864 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD4W_IMM, Opc_rr: AArch64::LD4W,
5865 IsIntr: true);
5866 return;
5867 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5868 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD4D_IMM, Opc_rr: AArch64::LD4D,
5869 IsIntr: true);
5870 return;
5871 }
5872 break;
5873 }
5874 case Intrinsic::aarch64_sme_read_hor_vg2: {
5875 if (VT == MVT::nxv16i8) {
5876 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5877 Op: AArch64::MOVA_2ZMXI_H_B);
5878 return;
5879 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5880 VT == MVT::nxv8bf16) {
5881 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5882 Op: AArch64::MOVA_2ZMXI_H_H);
5883 return;
5884 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5885 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5886 Op: AArch64::MOVA_2ZMXI_H_S);
5887 return;
5888 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5889 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5890 Op: AArch64::MOVA_2ZMXI_H_D);
5891 return;
5892 }
5893 break;
5894 }
5895 case Intrinsic::aarch64_sme_read_ver_vg2: {
5896 if (VT == MVT::nxv16i8) {
5897 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5898 Op: AArch64::MOVA_2ZMXI_V_B);
5899 return;
5900 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5901 VT == MVT::nxv8bf16) {
5902 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5903 Op: AArch64::MOVA_2ZMXI_V_H);
5904 return;
5905 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5906 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5907 Op: AArch64::MOVA_2ZMXI_V_S);
5908 return;
5909 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5910 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5911 Op: AArch64::MOVA_2ZMXI_V_D);
5912 return;
5913 }
5914 break;
5915 }
5916 case Intrinsic::aarch64_sme_read_hor_vg4: {
5917 if (VT == MVT::nxv16i8) {
5918 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5919 Op: AArch64::MOVA_4ZMXI_H_B);
5920 return;
5921 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5922 VT == MVT::nxv8bf16) {
5923 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5924 Op: AArch64::MOVA_4ZMXI_H_H);
5925 return;
5926 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5927 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5928 Op: AArch64::MOVA_4ZMXI_H_S);
5929 return;
5930 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5931 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5932 Op: AArch64::MOVA_4ZMXI_H_D);
5933 return;
5934 }
5935 break;
5936 }
5937 case Intrinsic::aarch64_sme_read_ver_vg4: {
5938 if (VT == MVT::nxv16i8) {
5939 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5940 Op: AArch64::MOVA_4ZMXI_V_B);
5941 return;
5942 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5943 VT == MVT::nxv8bf16) {
5944 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5945 Op: AArch64::MOVA_4ZMXI_V_H);
5946 return;
5947 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5948 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5949 Op: AArch64::MOVA_4ZMXI_V_S);
5950 return;
5951 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5952 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5953 Op: AArch64::MOVA_4ZMXI_V_D);
5954 return;
5955 }
5956 break;
5957 }
5958 case Intrinsic::aarch64_sme_read_vg1x2: {
5959 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 2, BaseReg: AArch64::ZA,
5960 Op: AArch64::MOVA_VG2_2ZMXI);
5961 return;
5962 }
5963 case Intrinsic::aarch64_sme_read_vg1x4: {
5964 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 4, BaseReg: AArch64::ZA,
5965 Op: AArch64::MOVA_VG4_4ZMXI);
5966 return;
5967 }
5968 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5969 if (VT == MVT::nxv16i8) {
5970 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_B_PSEUDO, MaxIdx: 14, Scale: 2);
5971 return;
5972 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5973 VT == MVT::nxv8bf16) {
5974 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_H_PSEUDO, MaxIdx: 6, Scale: 2);
5975 return;
5976 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5977 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_S_PSEUDO, MaxIdx: 2, Scale: 2);
5978 return;
5979 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5980 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 2);
5981 return;
5982 }
5983 break;
5984 }
5985 case Intrinsic::aarch64_sme_readz_vert_x2: {
5986 if (VT == MVT::nxv16i8) {
5987 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_B_PSEUDO, MaxIdx: 14, Scale: 2);
5988 return;
5989 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5990 VT == MVT::nxv8bf16) {
5991 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_H_PSEUDO, MaxIdx: 6, Scale: 2);
5992 return;
5993 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5994 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_S_PSEUDO, MaxIdx: 2, Scale: 2);
5995 return;
5996 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5997 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 2);
5998 return;
5999 }
6000 break;
6001 }
6002 case Intrinsic::aarch64_sme_readz_horiz_x4: {
6003 if (VT == MVT::nxv16i8) {
6004 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_B_PSEUDO, MaxIdx: 12, Scale: 4);
6005 return;
6006 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6007 VT == MVT::nxv8bf16) {
6008 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_H_PSEUDO, MaxIdx: 4, Scale: 4);
6009 return;
6010 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6011 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_S_PSEUDO, MaxIdx: 0, Scale: 4);
6012 return;
6013 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6014 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 4);
6015 return;
6016 }
6017 break;
6018 }
6019 case Intrinsic::aarch64_sme_readz_vert_x4: {
6020 if (VT == MVT::nxv16i8) {
6021 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_B_PSEUDO, MaxIdx: 12, Scale: 4);
6022 return;
6023 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6024 VT == MVT::nxv8bf16) {
6025 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_H_PSEUDO, MaxIdx: 4, Scale: 4);
6026 return;
6027 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6028 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_S_PSEUDO, MaxIdx: 0, Scale: 4);
6029 return;
6030 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6031 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 4);
6032 return;
6033 }
6034 break;
6035 }
6036 case Intrinsic::aarch64_sme_readz_x2: {
6037 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6038 BaseReg: AArch64::ZA);
6039 return;
6040 }
6041 case Intrinsic::aarch64_sme_readz_x4: {
6042 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6043 BaseReg: AArch64::ZA);
6044 return;
6045 }
6046 case Intrinsic::swift_async_context_addr: {
6047 SDLoc DL(Node);
6048 SDValue Chain = Node->getOperand(Num: 0);
6049 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::FP, VT: MVT::i64);
6050 SDValue Res = SDValue(
6051 CurDAG->getMachineNode(Opcode: AArch64::SUBXri, dl: DL, VT: MVT::i64, Op1: CopyFP,
6052 Op2: CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32),
6053 Op3: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32)),
6054 0);
6055 ReplaceUses(F: SDValue(Node, 0), T: Res);
6056 ReplaceUses(F: SDValue(Node, 1), T: CopyFP.getValue(R: 1));
6057 CurDAG->RemoveDeadNode(N: Node);
6058
6059 auto &MF = CurDAG->getMachineFunction();
6060 MF.getFrameInfo().setFrameAddressIsTaken(true);
6061 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6062 return;
6063 }
6064 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6065 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6066 VT: Node->getValueType(ResNo: 0),
6067 Opcodes: {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6068 AArch64::LUTI2_4ZTZI_S}))
6069 // Second Immediate must be <= 3:
6070 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 3);
6071 return;
6072 }
6073 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6074 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6075 VT: Node->getValueType(ResNo: 0),
6076 Opcodes: {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6077 // Second Immediate must be <= 1:
6078 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 1);
6079 return;
6080 }
6081 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6082 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6083 VT: Node->getValueType(ResNo: 0),
6084 Opcodes: {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6085 AArch64::LUTI2_2ZTZI_S}))
6086 // Second Immediate must be <= 7:
6087 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 7);
6088 return;
6089 }
6090 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6091 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6092 VT: Node->getValueType(ResNo: 0),
6093 Opcodes: {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6094 AArch64::LUTI4_2ZTZI_S}))
6095 // Second Immediate must be <= 3:
6096 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 3);
6097 return;
6098 }
6099 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6100 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI4_4ZZT2Z, NumInVecs: 2);
6101 return;
6102 }
6103 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6104 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI6_4ZT3Z, NumInVecs: 3);
6105 return;
6106 }
6107 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6108 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6109 VT: Node->getValueType(ResNo: 0),
6110 Opcodes: {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6111 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6112 return;
6113 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6114 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6115 VT: Node->getValueType(ResNo: 0),
6116 Opcodes: {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6117 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6118 return;
6119 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6120 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6121 VT: Node->getValueType(ResNo: 0),
6122 Opcodes: {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6123 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6124 return;
6125 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6126 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6127 VT: Node->getValueType(ResNo: 0),
6128 Opcodes: {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6129 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6130 return;
6131 case Intrinsic::ptrauth_resign_load_relative:
6132 SelectPtrauthResign(N: Node);
6133 return;
6134 }
6135 } break;
6136 case ISD::INTRINSIC_WO_CHAIN: {
6137 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
6138 switch (IntNo) {
6139 default:
6140 break;
6141 case Intrinsic::aarch64_tagp:
6142 SelectTagP(N: Node);
6143 return;
6144
6145 case Intrinsic::ptrauth_auth:
6146 SelectPtrauthAuth(N: Node);
6147 return;
6148
6149 case Intrinsic::ptrauth_resign:
6150 SelectPtrauthResign(N: Node);
6151 return;
6152
6153 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6154 SelectPtrauthResignWithPC(N: Node);
6155 return;
6156
6157 case Intrinsic::aarch64_neon_tbl2:
6158 SelectTable(N: Node, NumVecs: 2,
6159 Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6160 isExt: false);
6161 return;
6162 case Intrinsic::aarch64_neon_tbl3:
6163 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6164 : AArch64::TBLv16i8Three,
6165 isExt: false);
6166 return;
6167 case Intrinsic::aarch64_neon_tbl4:
6168 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6169 : AArch64::TBLv16i8Four,
6170 isExt: false);
6171 return;
6172 case Intrinsic::aarch64_neon_tbx2:
6173 SelectTable(N: Node, NumVecs: 2,
6174 Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6175 isExt: true);
6176 return;
6177 case Intrinsic::aarch64_neon_tbx3:
6178 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6179 : AArch64::TBXv16i8Three,
6180 isExt: true);
6181 return;
6182 case Intrinsic::aarch64_neon_tbx4:
6183 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6184 : AArch64::TBXv16i8Four,
6185 isExt: true);
6186 return;
6187 case Intrinsic::aarch64_sve_srshl_single_x2:
6188 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6189 VT: Node->getValueType(ResNo: 0),
6190 Opcodes: {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6191 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6192 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6193 return;
6194 case Intrinsic::aarch64_sve_srshl_single_x4:
6195 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6196 VT: Node->getValueType(ResNo: 0),
6197 Opcodes: {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6198 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6199 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6200 return;
6201 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6202 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 2);
6203 return;
6204 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6205 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 3);
6206 return;
6207 case Intrinsic::aarch64_sve_urshl_single_x2:
6208 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6209 VT: Node->getValueType(ResNo: 0),
6210 Opcodes: {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6211 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6212 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6213 return;
6214 case Intrinsic::aarch64_sve_urshl_single_x4:
6215 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6216 VT: Node->getValueType(ResNo: 0),
6217 Opcodes: {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6218 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6219 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6220 return;
6221 case Intrinsic::aarch64_sve_srshl_x2:
6222 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6223 VT: Node->getValueType(ResNo: 0),
6224 Opcodes: {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6225 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6226 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6227 return;
6228 case Intrinsic::aarch64_sve_srshl_x4:
6229 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6230 VT: Node->getValueType(ResNo: 0),
6231 Opcodes: {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6232 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6233 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6234 return;
6235 case Intrinsic::aarch64_sve_urshl_x2:
6236 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6237 VT: Node->getValueType(ResNo: 0),
6238 Opcodes: {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6239 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6240 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6241 return;
6242 case Intrinsic::aarch64_sve_urshl_x4:
6243 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6244 VT: Node->getValueType(ResNo: 0),
6245 Opcodes: {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6246 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6247 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6248 return;
6249 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6250 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6251 VT: Node->getValueType(ResNo: 0),
6252 Opcodes: {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6253 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6254 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6255 return;
6256 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6257 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6258 VT: Node->getValueType(ResNo: 0),
6259 Opcodes: {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6260 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6261 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6262 return;
6263 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6264 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6265 VT: Node->getValueType(ResNo: 0),
6266 Opcodes: {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6267 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6268 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6269 return;
6270 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6271 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6272 VT: Node->getValueType(ResNo: 0),
6273 Opcodes: {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6274 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6275 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6276 return;
6277 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6278 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6279 VT: Node->getValueType(ResNo: 0),
6280 Opcodes: {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6281 AArch64::FSCALE_2ZZ_D}))
6282 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6283 return;
6284 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6285 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6286 VT: Node->getValueType(ResNo: 0),
6287 Opcodes: {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6288 AArch64::FSCALE_4ZZ_D}))
6289 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6290 return;
6291 case Intrinsic::aarch64_sme_fp8_scale_x2:
6292 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6293 VT: Node->getValueType(ResNo: 0),
6294 Opcodes: {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6295 AArch64::FSCALE_2Z2Z_D}))
6296 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6297 return;
6298 case Intrinsic::aarch64_sme_fp8_scale_x4:
6299 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6300 VT: Node->getValueType(ResNo: 0),
6301 Opcodes: {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6302 AArch64::FSCALE_4Z4Z_D}))
6303 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6304 return;
6305 case Intrinsic::aarch64_sve_whilege_x2:
6306 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6307 VT: Node->getValueType(ResNo: 0),
6308 Opcodes: {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6309 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6310 SelectWhilePair(N: Node, Opc: Op);
6311 return;
6312 case Intrinsic::aarch64_sve_whilegt_x2:
6313 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6314 VT: Node->getValueType(ResNo: 0),
6315 Opcodes: {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6316 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6317 SelectWhilePair(N: Node, Opc: Op);
6318 return;
6319 case Intrinsic::aarch64_sve_whilehi_x2:
6320 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6321 VT: Node->getValueType(ResNo: 0),
6322 Opcodes: {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6323 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6324 SelectWhilePair(N: Node, Opc: Op);
6325 return;
6326 case Intrinsic::aarch64_sve_whilehs_x2:
6327 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6328 VT: Node->getValueType(ResNo: 0),
6329 Opcodes: {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6330 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6331 SelectWhilePair(N: Node, Opc: Op);
6332 return;
6333 case Intrinsic::aarch64_sve_whilele_x2:
6334 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6335 VT: Node->getValueType(ResNo: 0),
6336 Opcodes: {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6337 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6338 SelectWhilePair(N: Node, Opc: Op);
6339 return;
6340 case Intrinsic::aarch64_sve_whilelo_x2:
6341 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6342 VT: Node->getValueType(ResNo: 0),
6343 Opcodes: {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6344 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6345 SelectWhilePair(N: Node, Opc: Op);
6346 return;
6347 case Intrinsic::aarch64_sve_whilels_x2:
6348 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6349 VT: Node->getValueType(ResNo: 0),
6350 Opcodes: {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6351 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6352 SelectWhilePair(N: Node, Opc: Op);
6353 return;
6354 case Intrinsic::aarch64_sve_whilelt_x2:
6355 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6356 VT: Node->getValueType(ResNo: 0),
6357 Opcodes: {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6358 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6359 SelectWhilePair(N: Node, Opc: Op);
6360 return;
6361 case Intrinsic::aarch64_sve_smax_single_x2:
6362 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6363 VT: Node->getValueType(ResNo: 0),
6364 Opcodes: {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6365 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6366 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6367 return;
6368 case Intrinsic::aarch64_sve_umax_single_x2:
6369 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6370 VT: Node->getValueType(ResNo: 0),
6371 Opcodes: {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6372 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6373 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6374 return;
6375 case Intrinsic::aarch64_sve_fmax_single_x2:
6376 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6377 VT: Node->getValueType(ResNo: 0),
6378 Opcodes: {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6379 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6380 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6381 return;
6382 case Intrinsic::aarch64_sve_smax_single_x4:
6383 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6384 VT: Node->getValueType(ResNo: 0),
6385 Opcodes: {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6386 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6387 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6388 return;
6389 case Intrinsic::aarch64_sve_umax_single_x4:
6390 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6391 VT: Node->getValueType(ResNo: 0),
6392 Opcodes: {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6393 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6394 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6395 return;
6396 case Intrinsic::aarch64_sve_fmax_single_x4:
6397 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6398 VT: Node->getValueType(ResNo: 0),
6399 Opcodes: {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6400 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6401 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6402 return;
6403 case Intrinsic::aarch64_sve_smin_single_x2:
6404 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6405 VT: Node->getValueType(ResNo: 0),
6406 Opcodes: {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6407 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6408 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6409 return;
6410 case Intrinsic::aarch64_sve_umin_single_x2:
6411 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6412 VT: Node->getValueType(ResNo: 0),
6413 Opcodes: {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6414 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6415 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6416 return;
6417 case Intrinsic::aarch64_sve_fmin_single_x2:
6418 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6419 VT: Node->getValueType(ResNo: 0),
6420 Opcodes: {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6421 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6422 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6423 return;
6424 case Intrinsic::aarch64_sve_smin_single_x4:
6425 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6426 VT: Node->getValueType(ResNo: 0),
6427 Opcodes: {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6428 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6429 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6430 return;
6431 case Intrinsic::aarch64_sve_umin_single_x4:
6432 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6433 VT: Node->getValueType(ResNo: 0),
6434 Opcodes: {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6435 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6436 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6437 return;
6438 case Intrinsic::aarch64_sve_fmin_single_x4:
6439 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6440 VT: Node->getValueType(ResNo: 0),
6441 Opcodes: {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6442 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6443 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6444 return;
6445 case Intrinsic::aarch64_sve_smax_x2:
6446 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6447 VT: Node->getValueType(ResNo: 0),
6448 Opcodes: {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6449 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6450 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6451 return;
6452 case Intrinsic::aarch64_sve_umax_x2:
6453 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6454 VT: Node->getValueType(ResNo: 0),
6455 Opcodes: {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6456 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6457 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6458 return;
6459 case Intrinsic::aarch64_sve_fmax_x2:
6460 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6461 VT: Node->getValueType(ResNo: 0),
6462 Opcodes: {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6463 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6464 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6465 return;
6466 case Intrinsic::aarch64_sve_smax_x4:
6467 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6468 VT: Node->getValueType(ResNo: 0),
6469 Opcodes: {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6470 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6471 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6472 return;
6473 case Intrinsic::aarch64_sve_umax_x4:
6474 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6475 VT: Node->getValueType(ResNo: 0),
6476 Opcodes: {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6477 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6478 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6479 return;
6480 case Intrinsic::aarch64_sve_fmax_x4:
6481 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6482 VT: Node->getValueType(ResNo: 0),
6483 Opcodes: {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6484 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6485 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6486 return;
6487 case Intrinsic::aarch64_sme_famax_x2:
6488 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6489 VT: Node->getValueType(ResNo: 0),
6490 Opcodes: {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6491 AArch64::FAMAX_2Z2Z_D}))
6492 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6493 return;
6494 case Intrinsic::aarch64_sme_famax_x4:
6495 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6496 VT: Node->getValueType(ResNo: 0),
6497 Opcodes: {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6498 AArch64::FAMAX_4Z4Z_D}))
6499 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6500 return;
6501 case Intrinsic::aarch64_sme_famin_x2:
6502 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6503 VT: Node->getValueType(ResNo: 0),
6504 Opcodes: {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6505 AArch64::FAMIN_2Z2Z_D}))
6506 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6507 return;
6508 case Intrinsic::aarch64_sme_famin_x4:
6509 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6510 VT: Node->getValueType(ResNo: 0),
6511 Opcodes: {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6512 AArch64::FAMIN_4Z4Z_D}))
6513 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6514 return;
6515 case Intrinsic::aarch64_sve_smin_x2:
6516 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6517 VT: Node->getValueType(ResNo: 0),
6518 Opcodes: {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6519 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6520 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6521 return;
6522 case Intrinsic::aarch64_sve_umin_x2:
6523 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6524 VT: Node->getValueType(ResNo: 0),
6525 Opcodes: {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6526 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6527 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6528 return;
6529 case Intrinsic::aarch64_sve_fmin_x2:
6530 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6531 VT: Node->getValueType(ResNo: 0),
6532 Opcodes: {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6533 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6534 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6535 return;
6536 case Intrinsic::aarch64_sve_smin_x4:
6537 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6538 VT: Node->getValueType(ResNo: 0),
6539 Opcodes: {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6540 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6541 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6542 return;
6543 case Intrinsic::aarch64_sve_umin_x4:
6544 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6545 VT: Node->getValueType(ResNo: 0),
6546 Opcodes: {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6547 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6548 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6549 return;
6550 case Intrinsic::aarch64_sve_fmin_x4:
6551 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6552 VT: Node->getValueType(ResNo: 0),
6553 Opcodes: {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6554 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6555 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6556 return;
6557 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6558 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6559 VT: Node->getValueType(ResNo: 0),
6560 Opcodes: {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6561 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6562 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6563 return;
6564 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6565 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6566 VT: Node->getValueType(ResNo: 0),
6567 Opcodes: {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6568 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6569 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6570 return;
6571 case Intrinsic::aarch64_sve_fminnm_single_x2:
6572 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6573 VT: Node->getValueType(ResNo: 0),
6574 Opcodes: {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6575 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6576 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6577 return;
6578 case Intrinsic::aarch64_sve_fminnm_single_x4:
6579 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6580 VT: Node->getValueType(ResNo: 0),
6581 Opcodes: {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6582 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6583 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6584 return;
6585 case Intrinsic::aarch64_sve_fscale_single_x4:
6586 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::BFSCALE_4ZZ);
6587 return;
6588 case Intrinsic::aarch64_sve_fscale_single_x2:
6589 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::BFSCALE_2ZZ);
6590 return;
6591 case Intrinsic::aarch64_sve_fmul_single_x4:
6592 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6593 VT: Node->getValueType(ResNo: 0),
6594 Opcodes: {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6595 AArch64::FMUL_4ZZ_D}))
6596 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6597 return;
6598 case Intrinsic::aarch64_sve_fmul_single_x2:
6599 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6600 VT: Node->getValueType(ResNo: 0),
6601 Opcodes: {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6602 AArch64::FMUL_2ZZ_D}))
6603 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6604 return;
6605 case Intrinsic::aarch64_sve_fmaxnm_x2:
6606 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6607 VT: Node->getValueType(ResNo: 0),
6608 Opcodes: {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6609 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6610 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6611 return;
6612 case Intrinsic::aarch64_sve_fmaxnm_x4:
6613 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6614 VT: Node->getValueType(ResNo: 0),
6615 Opcodes: {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6616 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6617 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6618 return;
6619 case Intrinsic::aarch64_sve_fminnm_x2:
6620 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6621 VT: Node->getValueType(ResNo: 0),
6622 Opcodes: {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6623 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6624 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6625 return;
6626 case Intrinsic::aarch64_sve_fminnm_x4:
6627 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6628 VT: Node->getValueType(ResNo: 0),
6629 Opcodes: {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6630 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6631 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6632 return;
6633 case Intrinsic::aarch64_sve_aese_lane_x2:
6634 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESE_2ZZI_B);
6635 return;
6636 case Intrinsic::aarch64_sve_aesd_lane_x2:
6637 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESD_2ZZI_B);
6638 return;
6639 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6640 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESEMC_2ZZI_B);
6641 return;
6642 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6643 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESDIMC_2ZZI_B);
6644 return;
6645 case Intrinsic::aarch64_sve_aese_lane_x4:
6646 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESE_4ZZI_B);
6647 return;
6648 case Intrinsic::aarch64_sve_aesd_lane_x4:
6649 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESD_4ZZI_B);
6650 return;
6651 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6652 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESEMC_4ZZI_B);
6653 return;
6654 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6655 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESDIMC_4ZZI_B);
6656 return;
6657 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6658 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::PMLAL_2ZZZ_Q);
6659 return;
6660 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6661 SDLoc DL(Node);
6662 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 1, M: 2));
6663 SDNode *Res =
6664 CurDAG->getMachineNode(Opcode: AArch64::PMULL_2ZZZ_Q, dl: DL, VT: MVT::Untyped, Ops: Regs);
6665 SDValue SuperReg = SDValue(Res, 0);
6666 for (unsigned I = 0; I < 2; I++)
6667 ReplaceUses(F: SDValue(Node, I),
6668 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
6669 Operand: SuperReg));
6670 CurDAG->RemoveDeadNode(N: Node);
6671 return;
6672 }
6673 case Intrinsic::aarch64_sve_fscale_x4:
6674 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: AArch64::BFSCALE_4Z4Z);
6675 return;
6676 case Intrinsic::aarch64_sve_fscale_x2:
6677 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: AArch64::BFSCALE_2Z2Z);
6678 return;
6679 case Intrinsic::aarch64_sve_fmul_x4:
6680 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6681 VT: Node->getValueType(ResNo: 0),
6682 Opcodes: {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6683 AArch64::FMUL_4Z4Z_D}))
6684 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6685 return;
6686 case Intrinsic::aarch64_sve_fmul_x2:
6687 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6688 VT: Node->getValueType(ResNo: 0),
6689 Opcodes: {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6690 AArch64::FMUL_2Z2Z_D}))
6691 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6692 return;
6693 case Intrinsic::aarch64_sve_fcvtzs_x2:
6694 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZS_2Z2Z_StoS);
6695 return;
6696 case Intrinsic::aarch64_sve_scvtf_x2:
6697 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::SCVTF_2Z2Z_StoS);
6698 return;
6699 case Intrinsic::aarch64_sve_fcvtzu_x2:
6700 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZU_2Z2Z_StoS);
6701 return;
6702 case Intrinsic::aarch64_sve_ucvtf_x2:
6703 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::UCVTF_2Z2Z_StoS);
6704 return;
6705 case Intrinsic::aarch64_sve_fcvtzs_x4:
6706 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZS_4Z4Z_StoS);
6707 return;
6708 case Intrinsic::aarch64_sve_scvtf_x4:
6709 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::SCVTF_4Z4Z_StoS);
6710 return;
6711 case Intrinsic::aarch64_sve_fcvtzu_x4:
6712 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZU_4Z4Z_StoS);
6713 return;
6714 case Intrinsic::aarch64_sve_ucvtf_x4:
6715 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::UCVTF_4Z4Z_StoS);
6716 return;
6717 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6718 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVT_2ZZ_H_S);
6719 return;
6720 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6721 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVTL_2ZZ_H_S);
6722 return;
6723 case Intrinsic::aarch64_sve_sclamp_single_x2:
6724 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6725 VT: Node->getValueType(ResNo: 0),
6726 Opcodes: {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6727 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6728 SelectClamp(N: Node, NumVecs: 2, Op);
6729 return;
6730 case Intrinsic::aarch64_sve_uclamp_single_x2:
6731 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6732 VT: Node->getValueType(ResNo: 0),
6733 Opcodes: {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6734 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6735 SelectClamp(N: Node, NumVecs: 2, Op);
6736 return;
6737 case Intrinsic::aarch64_sve_fclamp_single_x2:
6738 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6739 VT: Node->getValueType(ResNo: 0),
6740 Opcodes: {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6741 AArch64::FCLAMP_VG2_2Z2Z_D}))
6742 SelectClamp(N: Node, NumVecs: 2, Op);
6743 return;
6744 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6745 SelectClamp(N: Node, NumVecs: 2, Op: AArch64::BFCLAMP_VG2_2ZZZ_H);
6746 return;
6747 case Intrinsic::aarch64_sve_sclamp_single_x4:
6748 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6749 VT: Node->getValueType(ResNo: 0),
6750 Opcodes: {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6751 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6752 SelectClamp(N: Node, NumVecs: 4, Op);
6753 return;
6754 case Intrinsic::aarch64_sve_uclamp_single_x4:
6755 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6756 VT: Node->getValueType(ResNo: 0),
6757 Opcodes: {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6758 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6759 SelectClamp(N: Node, NumVecs: 4, Op);
6760 return;
6761 case Intrinsic::aarch64_sve_fclamp_single_x4:
6762 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6763 VT: Node->getValueType(ResNo: 0),
6764 Opcodes: {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6765 AArch64::FCLAMP_VG4_4Z4Z_D}))
6766 SelectClamp(N: Node, NumVecs: 4, Op);
6767 return;
6768 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6769 SelectClamp(N: Node, NumVecs: 4, Op: AArch64::BFCLAMP_VG4_4ZZZ_H);
6770 return;
6771 case Intrinsic::aarch64_sve_add_single_x2:
6772 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6773 VT: Node->getValueType(ResNo: 0),
6774 Opcodes: {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6775 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6776 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6777 return;
6778 case Intrinsic::aarch64_sve_add_single_x4:
6779 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6780 VT: Node->getValueType(ResNo: 0),
6781 Opcodes: {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6782 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6783 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6784 return;
6785 case Intrinsic::aarch64_sve_zip_x2:
6786 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6787 VT: Node->getValueType(ResNo: 0),
6788 Opcodes: {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6789 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6790 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6791 return;
6792 case Intrinsic::aarch64_sve_zipq_x2:
6793 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6794 Opc: AArch64::ZIP_VG2_2ZZZ_Q);
6795 return;
6796 case Intrinsic::aarch64_sve_zip_x4:
6797 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6798 VT: Node->getValueType(ResNo: 0),
6799 Opcodes: {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6800 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6801 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6802 return;
6803 case Intrinsic::aarch64_sve_zipq_x4:
6804 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6805 Opc: AArch64::ZIP_VG4_4Z4Z_Q);
6806 return;
6807 case Intrinsic::aarch64_sve_uzp_x2:
6808 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6809 VT: Node->getValueType(ResNo: 0),
6810 Opcodes: {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6811 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6812 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6813 return;
6814 case Intrinsic::aarch64_sve_uzpq_x2:
6815 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6816 Opc: AArch64::UZP_VG2_2ZZZ_Q);
6817 return;
6818 case Intrinsic::aarch64_sve_uzp_x4:
6819 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6820 VT: Node->getValueType(ResNo: 0),
6821 Opcodes: {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6822 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6823 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6824 return;
6825 case Intrinsic::aarch64_sve_uzpq_x4:
6826 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6827 Opc: AArch64::UZP_VG4_4Z4Z_Q);
6828 return;
6829 case Intrinsic::aarch64_sve_sel_x2:
6830 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6831 VT: Node->getValueType(ResNo: 0),
6832 Opcodes: {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6833 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6834 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6835 return;
6836 case Intrinsic::aarch64_sve_sel_x4:
6837 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6838 VT: Node->getValueType(ResNo: 0),
6839 Opcodes: {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6840 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6841 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6842 return;
6843 case Intrinsic::aarch64_sve_frinta_x2:
6844 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTA_2Z2Z_S);
6845 return;
6846 case Intrinsic::aarch64_sve_frinta_x4:
6847 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTA_4Z4Z_S);
6848 return;
6849 case Intrinsic::aarch64_sve_frintm_x2:
6850 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTM_2Z2Z_S);
6851 return;
6852 case Intrinsic::aarch64_sve_frintm_x4:
6853 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTM_4Z4Z_S);
6854 return;
6855 case Intrinsic::aarch64_sve_frintn_x2:
6856 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTN_2Z2Z_S);
6857 return;
6858 case Intrinsic::aarch64_sve_frintn_x4:
6859 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTN_4Z4Z_S);
6860 return;
6861 case Intrinsic::aarch64_sve_frintp_x2:
6862 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTP_2Z2Z_S);
6863 return;
6864 case Intrinsic::aarch64_sve_frintp_x4:
6865 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTP_4Z4Z_S);
6866 return;
6867 case Intrinsic::aarch64_sve_sunpk_x2:
6868 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6869 VT: Node->getValueType(ResNo: 0),
6870 Opcodes: {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6871 AArch64::SUNPK_VG2_2ZZ_D}))
6872 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6873 return;
6874 case Intrinsic::aarch64_sve_uunpk_x2:
6875 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6876 VT: Node->getValueType(ResNo: 0),
6877 Opcodes: {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6878 AArch64::UUNPK_VG2_2ZZ_D}))
6879 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6880 return;
6881 case Intrinsic::aarch64_sve_sunpk_x4:
6882 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6883 VT: Node->getValueType(ResNo: 0),
6884 Opcodes: {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6885 AArch64::SUNPK_VG4_4Z2Z_D}))
6886 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6887 return;
6888 case Intrinsic::aarch64_sve_uunpk_x4:
6889 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6890 VT: Node->getValueType(ResNo: 0),
6891 Opcodes: {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6892 AArch64::UUNPK_VG4_4Z2Z_D}))
6893 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6894 return;
6895 case Intrinsic::aarch64_sve_pext_x2: {
6896 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6897 VT: Node->getValueType(ResNo: 0),
6898 Opcodes: {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6899 AArch64::PEXT_2PCI_D}))
6900 SelectPExtPair(N: Node, Opc: Op);
6901 return;
6902 }
6903 }
6904 break;
6905 }
6906 case ISD::INTRINSIC_VOID: {
6907 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
6908 if (Node->getNumOperands() >= 3)
6909 VT = Node->getOperand(Num: 2)->getValueType(ResNo: 0);
6910 switch (IntNo) {
6911 default:
6912 break;
6913 case Intrinsic::aarch64_neon_st1x2: {
6914 if (VT == MVT::v8i8) {
6915 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b);
6916 return;
6917 } else if (VT == MVT::v16i8) {
6918 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b);
6919 return;
6920 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6921 VT == MVT::v4bf16) {
6922 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h);
6923 return;
6924 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6925 VT == MVT::v8bf16) {
6926 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h);
6927 return;
6928 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6929 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s);
6930 return;
6931 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6932 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s);
6933 return;
6934 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6935 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d);
6936 return;
6937 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6938 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6939 return;
6940 }
6941 break;
6942 }
6943 case Intrinsic::aarch64_neon_st1x3: {
6944 if (VT == MVT::v8i8) {
6945 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b);
6946 return;
6947 } else if (VT == MVT::v16i8) {
6948 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b);
6949 return;
6950 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6951 VT == MVT::v4bf16) {
6952 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h);
6953 return;
6954 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6955 VT == MVT::v8bf16) {
6956 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h);
6957 return;
6958 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6959 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s);
6960 return;
6961 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6962 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s);
6963 return;
6964 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6965 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d);
6966 return;
6967 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6968 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
6969 return;
6970 }
6971 break;
6972 }
6973 case Intrinsic::aarch64_neon_st1x4: {
6974 if (VT == MVT::v8i8) {
6975 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b);
6976 return;
6977 } else if (VT == MVT::v16i8) {
6978 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b);
6979 return;
6980 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6981 VT == MVT::v4bf16) {
6982 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h);
6983 return;
6984 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6985 VT == MVT::v8bf16) {
6986 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h);
6987 return;
6988 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6989 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s);
6990 return;
6991 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6992 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s);
6993 return;
6994 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6995 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d);
6996 return;
6997 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6998 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
6999 return;
7000 }
7001 break;
7002 }
7003 case Intrinsic::aarch64_neon_st2: {
7004 if (VT == MVT::v8i8) {
7005 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b);
7006 return;
7007 } else if (VT == MVT::v16i8) {
7008 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b);
7009 return;
7010 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7011 VT == MVT::v4bf16) {
7012 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h);
7013 return;
7014 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7015 VT == MVT::v8bf16) {
7016 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h);
7017 return;
7018 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7019 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s);
7020 return;
7021 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7022 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s);
7023 return;
7024 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7025 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d);
7026 return;
7027 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7028 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
7029 return;
7030 }
7031 break;
7032 }
7033 case Intrinsic::aarch64_neon_st3: {
7034 if (VT == MVT::v8i8) {
7035 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b);
7036 return;
7037 } else if (VT == MVT::v16i8) {
7038 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b);
7039 return;
7040 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7041 VT == MVT::v4bf16) {
7042 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h);
7043 return;
7044 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7045 VT == MVT::v8bf16) {
7046 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h);
7047 return;
7048 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7049 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s);
7050 return;
7051 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7052 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s);
7053 return;
7054 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7055 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d);
7056 return;
7057 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7058 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
7059 return;
7060 }
7061 break;
7062 }
7063 case Intrinsic::aarch64_neon_st4: {
7064 if (VT == MVT::v8i8) {
7065 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b);
7066 return;
7067 } else if (VT == MVT::v16i8) {
7068 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b);
7069 return;
7070 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7071 VT == MVT::v4bf16) {
7072 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h);
7073 return;
7074 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7075 VT == MVT::v8bf16) {
7076 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h);
7077 return;
7078 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7079 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s);
7080 return;
7081 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7082 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s);
7083 return;
7084 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7085 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d);
7086 return;
7087 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7088 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7089 return;
7090 }
7091 break;
7092 }
7093 case Intrinsic::aarch64_neon_st2lane: {
7094 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7095 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8);
7096 return;
7097 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7098 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7099 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16);
7100 return;
7101 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7102 VT == MVT::v2f32) {
7103 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32);
7104 return;
7105 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7106 VT == MVT::v1f64) {
7107 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64);
7108 return;
7109 }
7110 break;
7111 }
7112 case Intrinsic::aarch64_neon_st3lane: {
7113 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7114 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8);
7115 return;
7116 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7117 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7118 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16);
7119 return;
7120 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7121 VT == MVT::v2f32) {
7122 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32);
7123 return;
7124 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7125 VT == MVT::v1f64) {
7126 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64);
7127 return;
7128 }
7129 break;
7130 }
7131 case Intrinsic::aarch64_neon_st4lane: {
7132 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7133 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8);
7134 return;
7135 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7136 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7137 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16);
7138 return;
7139 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7140 VT == MVT::v2f32) {
7141 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32);
7142 return;
7143 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7144 VT == MVT::v1f64) {
7145 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64);
7146 return;
7147 }
7148 break;
7149 }
7150 case Intrinsic::aarch64_sve_st2q: {
7151 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 4, Opc_rr: AArch64::ST2Q, Opc_ri: AArch64::ST2Q_IMM);
7152 return;
7153 }
7154 case Intrinsic::aarch64_sve_st3q: {
7155 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 4, Opc_rr: AArch64::ST3Q, Opc_ri: AArch64::ST3Q_IMM);
7156 return;
7157 }
7158 case Intrinsic::aarch64_sve_st4q: {
7159 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 4, Opc_rr: AArch64::ST4Q, Opc_ri: AArch64::ST4Q_IMM);
7160 return;
7161 }
7162 case Intrinsic::aarch64_sve_st2: {
7163 if (VT == MVT::nxv16i8) {
7164 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 0, Opc_rr: AArch64::ST2B, Opc_ri: AArch64::ST2B_IMM);
7165 return;
7166 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7167 VT == MVT::nxv8bf16) {
7168 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 1, Opc_rr: AArch64::ST2H, Opc_ri: AArch64::ST2H_IMM);
7169 return;
7170 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7171 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 2, Opc_rr: AArch64::ST2W, Opc_ri: AArch64::ST2W_IMM);
7172 return;
7173 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7174 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 3, Opc_rr: AArch64::ST2D, Opc_ri: AArch64::ST2D_IMM);
7175 return;
7176 }
7177 break;
7178 }
7179 case Intrinsic::aarch64_sve_st3: {
7180 if (VT == MVT::nxv16i8) {
7181 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 0, Opc_rr: AArch64::ST3B, Opc_ri: AArch64::ST3B_IMM);
7182 return;
7183 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7184 VT == MVT::nxv8bf16) {
7185 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 1, Opc_rr: AArch64::ST3H, Opc_ri: AArch64::ST3H_IMM);
7186 return;
7187 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7188 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 2, Opc_rr: AArch64::ST3W, Opc_ri: AArch64::ST3W_IMM);
7189 return;
7190 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7191 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 3, Opc_rr: AArch64::ST3D, Opc_ri: AArch64::ST3D_IMM);
7192 return;
7193 }
7194 break;
7195 }
7196 case Intrinsic::aarch64_sve_st4: {
7197 if (VT == MVT::nxv16i8) {
7198 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 0, Opc_rr: AArch64::ST4B, Opc_ri: AArch64::ST4B_IMM);
7199 return;
7200 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7201 VT == MVT::nxv8bf16) {
7202 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 1, Opc_rr: AArch64::ST4H, Opc_ri: AArch64::ST4H_IMM);
7203 return;
7204 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7205 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 2, Opc_rr: AArch64::ST4W, Opc_ri: AArch64::ST4W_IMM);
7206 return;
7207 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7208 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 3, Opc_rr: AArch64::ST4D, Opc_ri: AArch64::ST4D_IMM);
7209 return;
7210 }
7211 break;
7212 }
7213 }
7214 break;
7215 }
7216 case AArch64ISD::LD2post: {
7217 if (VT == MVT::v8i8) {
7218 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b_POST, SubRegIdx: AArch64::dsub0);
7219 return;
7220 } else if (VT == MVT::v16i8) {
7221 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b_POST, SubRegIdx: AArch64::qsub0);
7222 return;
7223 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7224 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h_POST, SubRegIdx: AArch64::dsub0);
7225 return;
7226 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7227 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h_POST, SubRegIdx: AArch64::qsub0);
7228 return;
7229 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7230 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s_POST, SubRegIdx: AArch64::dsub0);
7231 return;
7232 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7233 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s_POST, SubRegIdx: AArch64::qsub0);
7234 return;
7235 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7236 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7237 return;
7238 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7239 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d_POST, SubRegIdx: AArch64::qsub0);
7240 return;
7241 }
7242 break;
7243 }
7244 case AArch64ISD::LD3post: {
7245 if (VT == MVT::v8i8) {
7246 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b_POST, SubRegIdx: AArch64::dsub0);
7247 return;
7248 } else if (VT == MVT::v16i8) {
7249 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b_POST, SubRegIdx: AArch64::qsub0);
7250 return;
7251 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7252 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h_POST, SubRegIdx: AArch64::dsub0);
7253 return;
7254 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7255 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h_POST, SubRegIdx: AArch64::qsub0);
7256 return;
7257 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7258 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s_POST, SubRegIdx: AArch64::dsub0);
7259 return;
7260 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7261 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s_POST, SubRegIdx: AArch64::qsub0);
7262 return;
7263 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7264 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7265 return;
7266 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7267 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d_POST, SubRegIdx: AArch64::qsub0);
7268 return;
7269 }
7270 break;
7271 }
7272 case AArch64ISD::LD4post: {
7273 if (VT == MVT::v8i8) {
7274 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7275 return;
7276 } else if (VT == MVT::v16i8) {
7277 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7278 return;
7279 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7280 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7281 return;
7282 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7283 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7284 return;
7285 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7286 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7287 return;
7288 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7289 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7290 return;
7291 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7292 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7293 return;
7294 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7295 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7296 return;
7297 }
7298 break;
7299 }
7300 case AArch64ISD::LD1x2post: {
7301 if (VT == MVT::v8i8) {
7302 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b_POST, SubRegIdx: AArch64::dsub0);
7303 return;
7304 } else if (VT == MVT::v16i8) {
7305 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b_POST, SubRegIdx: AArch64::qsub0);
7306 return;
7307 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7308 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h_POST, SubRegIdx: AArch64::dsub0);
7309 return;
7310 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7311 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h_POST, SubRegIdx: AArch64::qsub0);
7312 return;
7313 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7314 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s_POST, SubRegIdx: AArch64::dsub0);
7315 return;
7316 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7317 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s_POST, SubRegIdx: AArch64::qsub0);
7318 return;
7319 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7320 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7321 return;
7322 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7323 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d_POST, SubRegIdx: AArch64::qsub0);
7324 return;
7325 }
7326 break;
7327 }
7328 case AArch64ISD::LD1x3post: {
7329 if (VT == MVT::v8i8) {
7330 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b_POST, SubRegIdx: AArch64::dsub0);
7331 return;
7332 } else if (VT == MVT::v16i8) {
7333 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b_POST, SubRegIdx: AArch64::qsub0);
7334 return;
7335 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7336 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h_POST, SubRegIdx: AArch64::dsub0);
7337 return;
7338 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7339 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h_POST, SubRegIdx: AArch64::qsub0);
7340 return;
7341 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7342 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s_POST, SubRegIdx: AArch64::dsub0);
7343 return;
7344 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7345 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s_POST, SubRegIdx: AArch64::qsub0);
7346 return;
7347 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7348 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7349 return;
7350 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7351 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d_POST, SubRegIdx: AArch64::qsub0);
7352 return;
7353 }
7354 break;
7355 }
7356 case AArch64ISD::LD1x4post: {
7357 if (VT == MVT::v8i8) {
7358 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7359 return;
7360 } else if (VT == MVT::v16i8) {
7361 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7362 return;
7363 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7364 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7365 return;
7366 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7367 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7368 return;
7369 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7370 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7371 return;
7372 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7373 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7374 return;
7375 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7376 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7377 return;
7378 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7379 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7380 return;
7381 }
7382 break;
7383 }
7384 case AArch64ISD::LD1DUPpost: {
7385 if (VT == MVT::v8i8) {
7386 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8b_POST, SubRegIdx: AArch64::dsub0);
7387 return;
7388 } else if (VT == MVT::v16i8) {
7389 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv16b_POST, SubRegIdx: AArch64::qsub0);
7390 return;
7391 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7392 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4h_POST, SubRegIdx: AArch64::dsub0);
7393 return;
7394 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7395 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8h_POST, SubRegIdx: AArch64::qsub0);
7396 return;
7397 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7398 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2s_POST, SubRegIdx: AArch64::dsub0);
7399 return;
7400 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7401 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4s_POST, SubRegIdx: AArch64::qsub0);
7402 return;
7403 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7404 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv1d_POST, SubRegIdx: AArch64::dsub0);
7405 return;
7406 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7407 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2d_POST, SubRegIdx: AArch64::qsub0);
7408 return;
7409 }
7410 break;
7411 }
7412 case AArch64ISD::LD2DUPpost: {
7413 if (VT == MVT::v8i8) {
7414 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b_POST, SubRegIdx: AArch64::dsub0);
7415 return;
7416 } else if (VT == MVT::v16i8) {
7417 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b_POST, SubRegIdx: AArch64::qsub0);
7418 return;
7419 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7420 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h_POST, SubRegIdx: AArch64::dsub0);
7421 return;
7422 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7423 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h_POST, SubRegIdx: AArch64::qsub0);
7424 return;
7425 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7426 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s_POST, SubRegIdx: AArch64::dsub0);
7427 return;
7428 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7429 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s_POST, SubRegIdx: AArch64::qsub0);
7430 return;
7431 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7432 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d_POST, SubRegIdx: AArch64::dsub0);
7433 return;
7434 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7435 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d_POST, SubRegIdx: AArch64::qsub0);
7436 return;
7437 }
7438 break;
7439 }
7440 case AArch64ISD::LD3DUPpost: {
7441 if (VT == MVT::v8i8) {
7442 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b_POST, SubRegIdx: AArch64::dsub0);
7443 return;
7444 } else if (VT == MVT::v16i8) {
7445 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b_POST, SubRegIdx: AArch64::qsub0);
7446 return;
7447 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7448 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h_POST, SubRegIdx: AArch64::dsub0);
7449 return;
7450 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7451 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h_POST, SubRegIdx: AArch64::qsub0);
7452 return;
7453 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7454 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s_POST, SubRegIdx: AArch64::dsub0);
7455 return;
7456 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7457 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s_POST, SubRegIdx: AArch64::qsub0);
7458 return;
7459 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7460 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d_POST, SubRegIdx: AArch64::dsub0);
7461 return;
7462 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7463 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d_POST, SubRegIdx: AArch64::qsub0);
7464 return;
7465 }
7466 break;
7467 }
7468 case AArch64ISD::LD4DUPpost: {
7469 if (VT == MVT::v8i8) {
7470 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b_POST, SubRegIdx: AArch64::dsub0);
7471 return;
7472 } else if (VT == MVT::v16i8) {
7473 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b_POST, SubRegIdx: AArch64::qsub0);
7474 return;
7475 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7476 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h_POST, SubRegIdx: AArch64::dsub0);
7477 return;
7478 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7479 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h_POST, SubRegIdx: AArch64::qsub0);
7480 return;
7481 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7482 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s_POST, SubRegIdx: AArch64::dsub0);
7483 return;
7484 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7485 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s_POST, SubRegIdx: AArch64::qsub0);
7486 return;
7487 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7488 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d_POST, SubRegIdx: AArch64::dsub0);
7489 return;
7490 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7491 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d_POST, SubRegIdx: AArch64::qsub0);
7492 return;
7493 }
7494 break;
7495 }
7496 case AArch64ISD::LD1LANEpost: {
7497 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7498 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i8_POST);
7499 return;
7500 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7501 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7502 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i16_POST);
7503 return;
7504 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7505 VT == MVT::v2f32) {
7506 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i32_POST);
7507 return;
7508 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7509 VT == MVT::v1f64) {
7510 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i64_POST);
7511 return;
7512 }
7513 break;
7514 }
7515 case AArch64ISD::LD2LANEpost: {
7516 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7517 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8_POST);
7518 return;
7519 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7520 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7521 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16_POST);
7522 return;
7523 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7524 VT == MVT::v2f32) {
7525 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32_POST);
7526 return;
7527 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7528 VT == MVT::v1f64) {
7529 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64_POST);
7530 return;
7531 }
7532 break;
7533 }
7534 case AArch64ISD::LD3LANEpost: {
7535 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7536 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8_POST);
7537 return;
7538 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7539 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7540 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16_POST);
7541 return;
7542 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7543 VT == MVT::v2f32) {
7544 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32_POST);
7545 return;
7546 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7547 VT == MVT::v1f64) {
7548 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64_POST);
7549 return;
7550 }
7551 break;
7552 }
7553 case AArch64ISD::LD4LANEpost: {
7554 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7555 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8_POST);
7556 return;
7557 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7558 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7559 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16_POST);
7560 return;
7561 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7562 VT == MVT::v2f32) {
7563 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32_POST);
7564 return;
7565 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7566 VT == MVT::v1f64) {
7567 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64_POST);
7568 return;
7569 }
7570 break;
7571 }
7572 case AArch64ISD::ST2post: {
7573 VT = Node->getOperand(Num: 1).getValueType();
7574 if (VT == MVT::v8i8) {
7575 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b_POST);
7576 return;
7577 } else if (VT == MVT::v16i8) {
7578 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b_POST);
7579 return;
7580 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7581 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h_POST);
7582 return;
7583 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7584 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h_POST);
7585 return;
7586 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7587 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s_POST);
7588 return;
7589 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7590 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s_POST);
7591 return;
7592 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7593 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d_POST);
7594 return;
7595 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7596 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7597 return;
7598 }
7599 break;
7600 }
7601 case AArch64ISD::ST3post: {
7602 VT = Node->getOperand(Num: 1).getValueType();
7603 if (VT == MVT::v8i8) {
7604 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b_POST);
7605 return;
7606 } else if (VT == MVT::v16i8) {
7607 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b_POST);
7608 return;
7609 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7610 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h_POST);
7611 return;
7612 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7613 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h_POST);
7614 return;
7615 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7616 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s_POST);
7617 return;
7618 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7619 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s_POST);
7620 return;
7621 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7622 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d_POST);
7623 return;
7624 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7625 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7626 return;
7627 }
7628 break;
7629 }
7630 case AArch64ISD::ST4post: {
7631 VT = Node->getOperand(Num: 1).getValueType();
7632 if (VT == MVT::v8i8) {
7633 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b_POST);
7634 return;
7635 } else if (VT == MVT::v16i8) {
7636 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b_POST);
7637 return;
7638 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7639 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h_POST);
7640 return;
7641 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7642 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h_POST);
7643 return;
7644 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7645 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s_POST);
7646 return;
7647 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7648 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s_POST);
7649 return;
7650 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7651 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d_POST);
7652 return;
7653 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7654 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7655 return;
7656 }
7657 break;
7658 }
7659 case AArch64ISD::ST1x2post: {
7660 VT = Node->getOperand(Num: 1).getValueType();
7661 if (VT == MVT::v8i8) {
7662 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b_POST);
7663 return;
7664 } else if (VT == MVT::v16i8) {
7665 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b_POST);
7666 return;
7667 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7668 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h_POST);
7669 return;
7670 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7671 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h_POST);
7672 return;
7673 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7674 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s_POST);
7675 return;
7676 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7677 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s_POST);
7678 return;
7679 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7680 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7681 return;
7682 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7683 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d_POST);
7684 return;
7685 }
7686 break;
7687 }
7688 case AArch64ISD::ST1x3post: {
7689 VT = Node->getOperand(Num: 1).getValueType();
7690 if (VT == MVT::v8i8) {
7691 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b_POST);
7692 return;
7693 } else if (VT == MVT::v16i8) {
7694 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b_POST);
7695 return;
7696 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7697 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h_POST);
7698 return;
7699 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7700 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h_POST);
7701 return;
7702 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7703 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s_POST);
7704 return;
7705 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7706 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s_POST);
7707 return;
7708 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7709 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7710 return;
7711 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7712 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d_POST);
7713 return;
7714 }
7715 break;
7716 }
7717 case AArch64ISD::ST1x4post: {
7718 VT = Node->getOperand(Num: 1).getValueType();
7719 if (VT == MVT::v8i8) {
7720 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b_POST);
7721 return;
7722 } else if (VT == MVT::v16i8) {
7723 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b_POST);
7724 return;
7725 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7726 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h_POST);
7727 return;
7728 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7729 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h_POST);
7730 return;
7731 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7732 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s_POST);
7733 return;
7734 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7735 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s_POST);
7736 return;
7737 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7738 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7739 return;
7740 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7741 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d_POST);
7742 return;
7743 }
7744 break;
7745 }
7746 case AArch64ISD::ST2LANEpost: {
7747 VT = Node->getOperand(Num: 1).getValueType();
7748 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7749 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8_POST);
7750 return;
7751 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7752 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7753 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16_POST);
7754 return;
7755 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7756 VT == MVT::v2f32) {
7757 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32_POST);
7758 return;
7759 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7760 VT == MVT::v1f64) {
7761 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64_POST);
7762 return;
7763 }
7764 break;
7765 }
7766 case AArch64ISD::ST3LANEpost: {
7767 VT = Node->getOperand(Num: 1).getValueType();
7768 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7769 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8_POST);
7770 return;
7771 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7772 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7773 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16_POST);
7774 return;
7775 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7776 VT == MVT::v2f32) {
7777 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32_POST);
7778 return;
7779 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7780 VT == MVT::v1f64) {
7781 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64_POST);
7782 return;
7783 }
7784 break;
7785 }
7786 case AArch64ISD::ST4LANEpost: {
7787 VT = Node->getOperand(Num: 1).getValueType();
7788 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7789 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8_POST);
7790 return;
7791 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7792 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7793 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16_POST);
7794 return;
7795 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7796 VT == MVT::v2f32) {
7797 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32_POST);
7798 return;
7799 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7800 VT == MVT::v1f64) {
7801 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64_POST);
7802 return;
7803 }
7804 break;
7805 }
7806 }
7807
7808 // Select the default instruction
7809 SelectCode(N: Node);
7810}
7811
7812/// createAArch64ISelDag - This pass converts a legalized DAG into a
7813/// AArch64-specific DAG, ready for instruction scheduling.
7814FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
7815 CodeGenOptLevel OptLevel) {
7816 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7817}
7818
7819/// When \p PredVT is a scalable vector predicate in the form
7820/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7821/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7822/// structured vectors (NumVec >1), the output data type is
7823/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7824/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7825/// EVT.
7826static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
7827 unsigned NumVec) {
7828 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7829 if (!PredVT.isScalableVectorOf(EltVT: MVT::i1))
7830 return EVT();
7831
7832 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7833 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7834 return EVT();
7835
7836 ElementCount EC = PredVT.getVectorElementCount();
7837 EVT ScalarVT =
7838 EVT::getIntegerVT(Context&: Ctx, BitWidth: AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7839 EVT MemVT = EVT::getVectorVT(Context&: Ctx, VT: ScalarVT, EC: EC * NumVec);
7840
7841 return MemVT;
7842}
7843
7844/// Builds an integer vector type large enough to hold \p NumVec instances
7845/// of \p VecVT.
7846static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7847 return EVT::getVectorVT(Context&: Ctx, VT: VecVT.getScalarType().changeTypeToInteger(),
7848 EC: VecVT.getVectorElementCount() * NumVec);
7849}
7850
7851/// Return the EVT of the data associated to a memory operation in \p
7852/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7853static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
7854 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: Root))
7855 return MemIntr->getMemoryVT();
7856
7857 if (isa<MemSDNode>(Val: Root)) {
7858 EVT MemVT = cast<MemSDNode>(Val: Root)->getMemoryVT();
7859
7860 EVT DataVT;
7861 if (auto *Load = dyn_cast<LoadSDNode>(Val: Root))
7862 DataVT = Load->getValueType(ResNo: 0);
7863 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Val: Root))
7864 DataVT = Load->getValueType(ResNo: 0);
7865 else if (auto *Store = dyn_cast<StoreSDNode>(Val: Root))
7866 DataVT = Store->getValue().getValueType();
7867 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Val: Root))
7868 DataVT = Store->getValue().getValueType();
7869 else
7870 llvm_unreachable("Unexpected MemSDNode!");
7871
7872 return DataVT.changeVectorElementType(Context&: Ctx, EltVT: MemVT.getVectorElementType());
7873 }
7874
7875 const unsigned Opcode = Root->getOpcode();
7876 // For custom ISD nodes, we have to look at them individually to extract the
7877 // type of the data moved to/from memory.
7878 switch (Opcode) {
7879 case AArch64ISD::LD1_MERGE_ZERO:
7880 case AArch64ISD::LD1S_MERGE_ZERO:
7881 case AArch64ISD::LDNF1_MERGE_ZERO:
7882 case AArch64ISD::LDNF1S_MERGE_ZERO:
7883 return cast<VTSDNode>(Val: Root->getOperand(Num: 3))->getVT();
7884 case AArch64ISD::ST1_PRED:
7885 return cast<VTSDNode>(Val: Root->getOperand(Num: 4))->getVT();
7886 default:
7887 break;
7888 }
7889
7890 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7891 return EVT();
7892
7893 switch (Root->getConstantOperandVal(Num: 1)) {
7894 default:
7895 return EVT();
7896 case Intrinsic::aarch64_sme_ldr:
7897 case Intrinsic::aarch64_sme_str:
7898 return MVT::nxv16i8;
7899 case Intrinsic::aarch64_sve_prf:
7900 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7901 // width of the predicate.
7902 return getPackedVectorTypeFromPredicateType(
7903 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/1);
7904 case Intrinsic::aarch64_sve_ld2_sret:
7905 case Intrinsic::aarch64_sve_ld2q_sret:
7906 return getPackedVectorTypeFromPredicateType(
7907 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/2);
7908 case Intrinsic::aarch64_sve_st2q:
7909 return getPackedVectorTypeFromPredicateType(
7910 Ctx, PredVT: Root->getOperand(Num: 4)->getValueType(ResNo: 0), /*NumVec=*/2);
7911 case Intrinsic::aarch64_sve_ld3_sret:
7912 case Intrinsic::aarch64_sve_ld3q_sret:
7913 return getPackedVectorTypeFromPredicateType(
7914 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/3);
7915 case Intrinsic::aarch64_sve_st3q:
7916 return getPackedVectorTypeFromPredicateType(
7917 Ctx, PredVT: Root->getOperand(Num: 5)->getValueType(ResNo: 0), /*NumVec=*/3);
7918 case Intrinsic::aarch64_sve_ld4_sret:
7919 case Intrinsic::aarch64_sve_ld4q_sret:
7920 return getPackedVectorTypeFromPredicateType(
7921 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/4);
7922 case Intrinsic::aarch64_sve_st4q:
7923 return getPackedVectorTypeFromPredicateType(
7924 Ctx, PredVT: Root->getOperand(Num: 6)->getValueType(ResNo: 0), /*NumVec=*/4);
7925 case Intrinsic::aarch64_sve_ld1_pn_x2:
7926 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7927 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7928 /*NumVec=*/2);
7929 case Intrinsic::aarch64_sve_ld1_pn_x4:
7930 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7931 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7932 /*NumVec=*/4);
7933 case Intrinsic::aarch64_sve_st1_pn_x2:
7934 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7935 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7936 /*NumVec=*/2);
7937 case Intrinsic::aarch64_sve_st1_pn_x4:
7938 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7939 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7940 /*NumVec=*/4);
7941 case Intrinsic::aarch64_sve_ld1udq:
7942 case Intrinsic::aarch64_sve_st1dq:
7943 return EVT(MVT::nxv1i64);
7944 case Intrinsic::aarch64_sve_ld1uwq:
7945 case Intrinsic::aarch64_sve_st1wq:
7946 return EVT(MVT::nxv1i32);
7947 }
7948}
7949
7950/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7951/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7952/// where Root is the memory access using N for its address.
7953template <int64_t Min, int64_t Max>
7954bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7955 SDValue &Base,
7956 SDValue &OffImm) {
7957 const EVT MemVT = getMemVTFromNode(Ctx&: *(CurDAG->getContext()), Root);
7958 const DataLayout &DL = CurDAG->getDataLayout();
7959 const MachineFrameInfo &MFI = MF->getFrameInfo();
7960
7961 if (N.getOpcode() == ISD::FrameIndex) {
7962 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
7963 // We can only encode VL scaled offsets, so only fold in frame indexes
7964 // referencing SVE objects.
7965 if (MFI.hasScalableStackID(ObjectIdx: FI)) {
7966 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7967 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
7968 return true;
7969 }
7970
7971 return false;
7972 }
7973
7974 if (MemVT == EVT())
7975 return false;
7976
7977 if (N.getOpcode() != ISD::ADD)
7978 return false;
7979
7980 SDValue VScale = N.getOperand(i: 1);
7981 int64_t MulImm = std::numeric_limits<int64_t>::max();
7982 if (VScale.getOpcode() == ISD::VSCALE) {
7983 MulImm = cast<ConstantSDNode>(Val: VScale.getOperand(i: 0))->getSExtValue();
7984 } else if (auto C = dyn_cast<ConstantSDNode>(Val&: VScale)) {
7985 int64_t ByteOffset = C->getSExtValue();
7986 const auto KnownVScale =
7987 Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
7988
7989 if (!KnownVScale || ByteOffset % KnownVScale != 0)
7990 return false;
7991
7992 MulImm = ByteOffset / KnownVScale;
7993 } else
7994 return false;
7995
7996 TypeSize TS = MemVT.getSizeInBits();
7997 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
7998
7999 if ((MulImm % MemWidthBytes) != 0)
8000 return false;
8001
8002 int64_t Offset = MulImm / MemWidthBytes;
8003 if (Offset < Min || Offset > Max)
8004 return false;
8005
8006 Base = N.getOperand(i: 0);
8007 if (Base.getOpcode() == ISD::FrameIndex) {
8008 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
8009 // We can only encode VL scaled offsets, so only fold in frame indexes
8010 // referencing SVE objects.
8011 if (MFI.hasScalableStackID(ObjectIdx: FI))
8012 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
8013 }
8014
8015 OffImm = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(N), VT: MVT::i64);
8016 return true;
8017}
8018
8019/// Select register plus register addressing mode for SVE, with scaled
8020/// offset.
8021bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
8022 SDValue &Base,
8023 SDValue &Offset) {
8024 if (N.getOpcode() != ISD::ADD)
8025 return false;
8026
8027 // Process an ADD node.
8028 const SDValue LHS = N.getOperand(i: 0);
8029 const SDValue RHS = N.getOperand(i: 1);
8030
8031 // 8 bit data does not come with the SHL node, so it is treated
8032 // separately.
8033 if (Scale == 0) {
8034 Base = LHS;
8035 Offset = RHS;
8036 return true;
8037 }
8038
8039 if (auto C = dyn_cast<ConstantSDNode>(Val: RHS)) {
8040 int64_t ImmOff = C->getSExtValue();
8041 unsigned Size = 1 << Scale;
8042
8043 // To use the reg+reg addressing mode, the immediate must be a multiple of
8044 // the vector element's byte size.
8045 if (ImmOff % Size)
8046 return false;
8047
8048 SDLoc DL(N);
8049 Base = LHS;
8050 Offset = CurDAG->getTargetConstant(Val: ImmOff >> Scale, DL, VT: MVT::i64);
8051 SDValue Ops[] = {Offset};
8052 SDNode *MI = CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
8053 Offset = SDValue(MI, 0);
8054 return true;
8055 }
8056
8057 // Check if the RHS is a shift node with a constant.
8058 if (RHS.getOpcode() != ISD::SHL)
8059 return false;
8060
8061 const SDValue ShiftRHS = RHS.getOperand(i: 1);
8062 if (auto *C = dyn_cast<ConstantSDNode>(Val: ShiftRHS))
8063 if (C->getZExtValue() == Scale) {
8064 Base = LHS;
8065 Offset = RHS.getOperand(i: 0);
8066 return true;
8067 }
8068
8069 return false;
8070}
8071
8072bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8073 const AArch64TargetLowering *TLI =
8074 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8075
8076 return TLI->isAllActivePredicate(DAG: *CurDAG, N);
8077}
8078
8079bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8080 return N.getValueType().isScalableVectorOf(EltVT: MVT::i1);
8081}
8082
8083bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8084 SDValue &Base, SDValue &Offset,
8085 unsigned Scale) {
8086 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8087 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CN)) {
8088 int64_t ImmOff = C->getSExtValue();
8089 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8090 return CurDAG->getTargetConstant(Val: ImmOff / Scale, DL: SDLoc(N), VT: MVT::i64);
8091 }
8092 return SDValue();
8093 };
8094
8095 if (SDValue C = MatchConstantOffset(N)) {
8096 Base = getZeroRegister(DAG&: *CurDAG, DL: SDLoc(N), VT: MVT::i32);
8097 Offset = C;
8098 return true;
8099 }
8100
8101 // Try to untangle an ADD node into a 'reg + offset'
8102 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
8103 if (SDValue C = MatchConstantOffset(N.getOperand(i: 1))) {
8104 Base = N.getOperand(i: 0);
8105 Offset = C;
8106 return true;
8107 }
8108 }
8109
8110 // By default, just match reg + 0.
8111 Base = N;
8112 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
8113 return true;
8114}
8115
8116bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8117 SDValue &Imm) {
8118 AArch64CC::CondCode CC =
8119 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(Num: 1));
8120 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
8121 // Check conservatively if the immediate fits the valid range [0, 64).
8122 // Immediate variants for GE and HS definitely need to be decremented
8123 // when lowering the pseudos later, so an immediate of 1 would become 0.
8124 // For the inverse conditions LT and LO we don't know for sure if they
8125 // will need a decrement but should the decision be made to reverse the
8126 // branch condition, we again end up with the need to decrement.
8127 // The same argument holds for LE, LS, GT and HI and possibly
8128 // incremented immediates. This can lead to slightly less optimal
8129 // codegen, e.g. we never codegen the legal case
8130 // cblt w0, #63, A
8131 // because we could end up with the illegal case
8132 // cbge w0, #64, B
8133 // should the decision to reverse the branch direction be made. For the
8134 // lower bound cases this is no problem since we can express comparisons
8135 // against 0 with either tbz/tnbz or using wzr/xzr.
8136 uint64_t LowerBound = 0, UpperBound = 64;
8137 switch (CC) {
8138 case AArch64CC::GE:
8139 case AArch64CC::HS:
8140 case AArch64CC::LT:
8141 case AArch64CC::LO:
8142 LowerBound = 1;
8143 break;
8144 case AArch64CC::LE:
8145 case AArch64CC::LS:
8146 case AArch64CC::GT:
8147 case AArch64CC::HI:
8148 UpperBound = 63;
8149 break;
8150 default:
8151 break;
8152 }
8153
8154 if (CN->getAPIntValue().uge(RHS: LowerBound) &&
8155 CN->getAPIntValue().ult(RHS: UpperBound)) {
8156 SDLoc DL(N);
8157 Imm = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: N.getValueType());
8158 return true;
8159 }
8160 }
8161
8162 return false;
8163}
8164
8165template <bool MatchCBB>
8166bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8167 SDValue &ExtType) {
8168
8169 // Use an invalid shift-extend value to indicate we don't need to extend later
8170 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8171 EVT Ty = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
8172 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8173 return false;
8174 Reg = N.getOperand(i: 0);
8175 ExtType = CurDAG->getSignedTargetConstant(Val: AArch64_AM::InvalidShiftExtend,
8176 DL: SDLoc(N), VT: MVT::i32);
8177 return true;
8178 }
8179
8180 AArch64_AM::ShiftExtendType ET = getExtendTypeForNode(N);
8181
8182 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8183 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8184 Reg = N.getOperand(i: 0);
8185 ExtType =
8186 CurDAG->getTargetConstant(Val: getExtendEncoding(ET), DL: SDLoc(N), VT: MVT::i32);
8187 return true;
8188 }
8189
8190 return false;
8191}
8192
8193/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8194///
8195/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8196/// doing this earlier based on the defining operation of X can be invalidated
8197/// by later DAG combines. At this point the DAG is being prepared for
8198/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8199/// final SDValue being selected.
8200/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8201/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8202/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8203/// differ from fcmp+fcsel for signaling NaN inputs.
8204bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *N) {
8205 EVT VT = N->getValueType(ResNo: 0);
8206
8207 // Scalar FP only.
8208 if (!VT.isFloatingPoint() || VT.isVector())
8209 return false;
8210
8211 SDValue TVal = N->getOperand(Num: 0);
8212 SDValue FVal = N->getOperand(Num: 1);
8213 SDValue CCVal = N->getOperand(Num: 2);
8214 SDValue Cmp = N->getOperand(Num: 3);
8215
8216 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8217 return false;
8218
8219 auto *CC = dyn_cast<ConstantSDNode>(Val&: CCVal);
8220 if (!CC)
8221 return false;
8222
8223 SDValue CmpLHS = Cmp.getOperand(i: 0);
8224 SDValue CmpRHS = Cmp.getOperand(i: 1);
8225 unsigned CondCode = CC->getZExtValue();
8226
8227 // Map VT and operation (max/min) to machine opcode.
8228 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8229 if (VT == MVT::f16)
8230 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8231 else if (VT == MVT::f32)
8232 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8233 else if (VT == MVT::f64)
8234 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8235 else
8236 return 0; // unsupported
8237 };
8238
8239 // Determine whether to use max or min based on condition code and operands.
8240 bool isMax;
8241 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8242 if (TVal == CmpLHS && FVal == CmpRHS)
8243 isMax = true;
8244 else
8245 return false;
8246 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8247 if (TVal == CmpLHS && FVal == CmpRHS)
8248 isMax = false;
8249 else
8250 return false;
8251 } else {
8252 return false;
8253 }
8254
8255 // Get the machine opcode for this VT and operation.
8256 unsigned Opc = getOpc(VT, isMax);
8257 if (!Opc)
8258 return false;
8259
8260 // Constant must be non-NaN.
8261 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: CmpRHS);
8262 if (!CFP || CFP->getValueAPF().isNaN())
8263 return false;
8264
8265 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8266 // fcmp+select's -0. For non-zero constants, semantics are identical.
8267 if (CFP->isZero() && !N->getFlags().hasNoSignedZeros())
8268 return false;
8269
8270 // Only fold if variable operand is never sNaN.
8271 // This runs after DAG combines, so later combines cannot remove a defining
8272 // operation used by isKnownNeverSNaN().
8273 if (!CurDAG->isKnownNeverSNaN(Op: CmpLHS))
8274 return false;
8275
8276 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Op1: CmpLHS, Op2: CmpRHS);
8277 return true;
8278}
8279
8280void AArch64DAGToDAGISel::PreprocessISelDAG() {
8281 bool MadeChange = false;
8282 for (SDNode &N : llvm::make_early_inc_range(Range: CurDAG->allnodes())) {
8283 if (N.use_empty())
8284 continue;
8285
8286 SDValue Result;
8287 switch (N.getOpcode()) {
8288 case ISD::SCALAR_TO_VECTOR: {
8289 EVT ScalarTy = N.getValueType(ResNo: 0).getVectorElementType();
8290 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8291 ScalarTy == N.getOperand(Num: 0).getValueType())
8292 Result = addBitcastHints(DAG&: *CurDAG, N);
8293
8294 break;
8295 }
8296 case AArch64ISD::VSHL: {
8297 // Undo mul(shl(A,C),B) -> shl(mul(A,B),C) canonicalisation when A is an
8298 // extend that can be folded into the shift.
8299 EVT VT = N.getValueType(ResNo: 0);
8300 SDValue A, B, C = N.getOperand(Num: 1);
8301 if (sd_match(N: N.getOperand(Num: 0),
8302 P: m_OneUse(P: m_Mul(L: m_Value(N&: A, P: m_AnyOf(preds: m_ZExt(Op: m_Value()),
8303 preds: m_SExt(Op: m_Value()))),
8304 R: m_Value(N&: B))))) {
8305 // If both mul operands are extended, preserve the smull/umull idiom.
8306 if (B.getOpcode() == A.getOpcode())
8307 break;
8308 SDLoc DL(&N);
8309 SDValue SHL = CurDAG->getNode(Opcode: AArch64ISD::VSHL, DL, VT, N1: A, N2: C);
8310 Result = CurDAG->getNode(Opcode: ISD::MUL, DL, VT, N1: SHL, N2: B);
8311 }
8312 break;
8313 }
8314 default:
8315 break;
8316 }
8317
8318 if (Result) {
8319 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8320 LLVM_DEBUG(N.dump(CurDAG));
8321 LLVM_DEBUG(dbgs() << "\nNew: ");
8322 LLVM_DEBUG(Result.dump(CurDAG));
8323 LLVM_DEBUG(dbgs() << "\n");
8324
8325 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(&N, 0), To: Result);
8326 MadeChange = true;
8327 }
8328 }
8329
8330 if (MadeChange)
8331 CurDAG->RemoveDeadNodes();
8332
8333 SelectionDAGISel::PreprocessISelDAG();
8334}
8335