1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64TargetMachine.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/SDPatternMatch.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/IR/Function.h" // To access function attributes.
23#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
26#include "llvm/Support/AArch64MemoryHints.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Support/KnownBits.h"
30#include "llvm/Support/MathExtras.h"
31#include "llvm/Support/raw_ostream.h"
32
33using namespace llvm;
34using namespace llvm::SDPatternMatch;
35
36#define DEBUG_TYPE "aarch64-isel"
37#define PASS_NAME "AArch64 Instruction Selection"
38
39// https://github.com/llvm/llvm-project/issues/114425
40#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
41#pragma inline_depth(0)
42#endif
43
44//===--------------------------------------------------------------------===//
45/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49
50class AArch64DAGToDAGISel : public SelectionDAGISel {
51
52 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
53 /// make the right decision when generating code for different targets.
54 const AArch64Subtarget *Subtarget;
55
56public:
57 AArch64DAGToDAGISel() = delete;
58
59 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
60 CodeGenOptLevel OptLevel)
61 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
62
63 bool runOnMachineFunction(MachineFunction &MF) override {
64 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
65 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
66 }
67
68 void Select(SDNode *Node) override;
69 void PreprocessISelDAG() override;
70
71 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
72 /// inline asm expressions.
73 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
74 InlineAsm::ConstraintCode ConstraintID,
75 std::vector<SDValue> &OutOps) override;
76
77 template <signed Low, signed High, signed Scale>
78 bool SelectRDVLImm(SDValue N, SDValue &Imm);
79
80 template <signed Low, signed High>
81 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
82
83 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
84 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
85 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
86 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
87 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
88 return SelectShiftedRegister(N, AllowROR: false, Reg, Shift);
89 }
90 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
91 return SelectShiftedRegister(N, AllowROR: true, Reg, Shift);
92 }
93 template <unsigned ShiftWidth>
94 bool SelectShiftMask(SDValue N, SDValue &ShAmt);
95
96 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
97 return SelectAddrModeIndexed7S(N, Size: 1, Base, OffImm);
98 }
99 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
100 return SelectAddrModeIndexed7S(N, Size: 2, Base, OffImm);
101 }
102 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
103 return SelectAddrModeIndexed7S(N, Size: 4, Base, OffImm);
104 }
105 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
106 return SelectAddrModeIndexed7S(N, Size: 8, Base, OffImm);
107 }
108 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
109 return SelectAddrModeIndexed7S(N, Size: 16, Base, OffImm);
110 }
111 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
112 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 9, Size: 16, Base, OffImm);
113 }
114 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
115 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: false, BW: 6, Size: 16, Base, OffImm);
116 }
117 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
118 return SelectAddrModeIndexed(N, Size: 1, Base, OffImm);
119 }
120 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
121 return SelectAddrModeIndexed(N, Size: 2, Base, OffImm);
122 }
123 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
124 return SelectAddrModeIndexed(N, Size: 4, Base, OffImm);
125 }
126 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
127 return SelectAddrModeIndexed(N, Size: 8, Base, OffImm);
128 }
129 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
130 return SelectAddrModeIndexed(N, Size: 16, Base, OffImm);
131 }
132 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
133 return SelectAddrModeUnscaled(N, Size: 1, Base, OffImm);
134 }
135 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
136 return SelectAddrModeUnscaled(N, Size: 2, Base, OffImm);
137 }
138 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
139 return SelectAddrModeUnscaled(N, Size: 4, Base, OffImm);
140 }
141 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
142 return SelectAddrModeUnscaled(N, Size: 8, Base, OffImm);
143 }
144 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
145 return SelectAddrModeUnscaled(N, Size: 16, Base, OffImm);
146 }
147 template <unsigned Size, unsigned Max>
148 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
149 // Test if there is an appropriate addressing mode and check if the
150 // immediate fits.
151 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
152 if (Found) {
153 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: OffImm)) {
154 int64_t C = CI->getSExtValue();
155 if (C <= Max)
156 return true;
157 }
158 }
159
160 // Otherwise, base only, materialize address in register.
161 Base = N;
162 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
163 return true;
164 }
165
166 template<int Width>
167 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
168 SDValue &SignExtend, SDValue &DoShift) {
169 return SelectAddrModeWRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
170 }
171
172 template<int Width>
173 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
174 SDValue &SignExtend, SDValue &DoShift) {
175 return SelectAddrModeXRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
176 }
177
178 bool SelectExtractHigh(SDValue N, SDValue &Res) {
179 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
180 N = N->getOperand(Num: 0);
181 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
182 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
183 return false;
184 EVT VT = N->getValueType(ResNo: 0);
185 EVT LVT = N->getOperand(Num: 0).getValueType();
186 unsigned Index = N->getConstantOperandVal(Num: 1);
187 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
188 Index != VT.getVectorNumElements())
189 return false;
190 Res = N->getOperand(Num: 0);
191 return true;
192 }
193
194 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
195 if (N.getOpcode() != AArch64ISD::VLSHR)
196 return false;
197 SDValue Op = N->getOperand(Num: 0);
198 EVT VT = Op.getValueType();
199 unsigned ShtAmt = N->getConstantOperandVal(Num: 1);
200 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
201 return false;
202
203 APInt Imm;
204 if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::MOVIshift)
205 Imm = APInt(VT.getScalarSizeInBits(),
206 Op.getOperand(i: 1).getConstantOperandVal(i: 0)
207 << Op.getOperand(i: 1).getConstantOperandVal(i: 1));
208 else if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::DUP &&
209 isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getOperand(i: 0)))
210 Imm = APInt(VT.getScalarSizeInBits(),
211 Op.getOperand(i: 1).getConstantOperandVal(i: 0));
212 else
213 return false;
214
215 if (Imm != 1ULL << (ShtAmt - 1))
216 return false;
217
218 Res1 = Op.getOperand(i: 0);
219 Res2 = CurDAG->getTargetConstant(Val: ShtAmt, DL: SDLoc(N), VT: MVT::i32);
220 return true;
221 }
222
223 bool SelectDupZeroOrUndef(SDValue N) {
224 switch(N->getOpcode()) {
225 case ISD::UNDEF:
226 case ISD::POISON:
227 return true;
228 case AArch64ISD::DUP:
229 case ISD::SPLAT_VECTOR: {
230 auto Opnd0 = N->getOperand(Num: 0);
231 if (isNullConstant(V: Opnd0))
232 return true;
233 if (isNullFPConstant(V: Opnd0))
234 return true;
235 break;
236 }
237 default:
238 break;
239 }
240
241 return false;
242 }
243
244 bool SelectAny(SDValue) { return true; }
245
246 bool SelectDupZero(SDValue N) {
247 switch(N->getOpcode()) {
248 case AArch64ISD::DUP:
249 case ISD::SPLAT_VECTOR: {
250 auto Opnd0 = N->getOperand(Num: 0);
251 if (isNullConstant(V: Opnd0))
252 return true;
253 if (isNullFPConstant(V: Opnd0))
254 return true;
255 break;
256 }
257 }
258
259 return false;
260 }
261
262 template <MVT::SimpleValueType VT, bool Negate>
263 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
264 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
265 }
266
267 template <MVT::SimpleValueType VT, bool Negate>
268 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
269 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
270 }
271
272 template <MVT::SimpleValueType VT>
273 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
274 return SelectSVECpyDupImm(N, VT, Imm, Shift);
275 }
276
277 template <MVT::SimpleValueType VT, bool Invert = false>
278 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
279 return SelectSVELogicalImm(N, VT, Imm, Invert);
280 }
281
282 template <MVT::SimpleValueType VT>
283 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
284 return SelectSVEArithImm(N, VT, Imm);
285 }
286
287 template <unsigned Low, unsigned High, bool AllowSaturation = false>
288 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
289 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
290 }
291
292 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
293 if (N->getOpcode() != ISD::SPLAT_VECTOR)
294 return false;
295
296 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
297 return SelectSVEShiftImm(N: N->getOperand(Num: 0), /* Low */ 1,
298 /* High */ EltVT.getFixedSizeInBits(),
299 /* AllowSaturation */ true, Imm);
300 }
301
302 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
303 template<signed Min, signed Max, signed Scale, bool Shift>
304 bool SelectCntImm(SDValue N, SDValue &Imm) {
305 if (!isa<ConstantSDNode>(Val: N))
306 return false;
307
308 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
309 if (Shift)
310 MulImm = 1LL << MulImm;
311
312 if ((MulImm % std::abs(x: Scale)) != 0)
313 return false;
314
315 MulImm /= Scale;
316 if ((MulImm >= Min) && (MulImm <= Max)) {
317 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
318 return true;
319 }
320
321 return false;
322 }
323
324 template <signed Max, signed Scale>
325 bool SelectEXTImm(SDValue N, SDValue &Imm) {
326 if (!isa<ConstantSDNode>(Val: N))
327 return false;
328
329 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
330
331 if (MulImm >= 0 && MulImm <= Max) {
332 MulImm *= Scale;
333 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
334 return true;
335 }
336
337 return false;
338 }
339
340 template <unsigned BaseReg, unsigned Max>
341 bool ImmToReg(SDValue N, SDValue &Imm) {
342 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: N)) {
343 uint64_t C = CI->getZExtValue();
344
345 if (C > Max)
346 return false;
347
348 Imm = CurDAG->getRegister(Reg: BaseReg + C, VT: MVT::Other);
349 return true;
350 }
351 return false;
352 }
353
354 /// Form sequences of consecutive 64/128-bit registers for use in NEON
355 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
356 /// between 1 and 4 elements. If it contains a single element that is returned
357 /// unchanged; otherwise a REG_SEQUENCE value is returned.
358 SDValue createDTuple(ArrayRef<SDValue> Vecs);
359 SDValue createQTuple(ArrayRef<SDValue> Vecs);
360 // Form a sequence of SVE registers for instructions using list of vectors,
361 // e.g. structured loads and stores (ldN, stN).
362 SDValue createZTuple(ArrayRef<SDValue> Vecs);
363
364 // Similar to above, except the register must start at a multiple of the
365 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
366 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
367
368 /// Generic helper for the createDTuple/createQTuple
369 /// functions. Those should almost always be called instead.
370 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
371 const unsigned SubRegs[]);
372
373 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
374
375 bool tryIndexedLoad(SDNode *N);
376
377 void SelectPtrauthAuth(SDNode *N);
378 void SelectPtrauthResign(SDNode *N);
379 void SelectPtrauthResignWithPC(SDNode *N);
380
381 bool trySelectStackSlotTagP(SDNode *N);
382 void SelectTagP(SDNode *N);
383
384 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
385 unsigned SubRegIdx);
386 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
387 unsigned SubRegIdx);
388 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
389 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
390 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
391 unsigned Opc_rr, unsigned Opc_ri,
392 bool IsIntr = false);
393 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
394 unsigned Scale, unsigned Opc_ri,
395 unsigned Opc_rr);
396 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
397 bool IsZmMulti, unsigned Opcode,
398 bool HasPred = false);
399 void SelectPExtPair(SDNode *N, unsigned Opc);
400 void SelectWhilePair(SDNode *N, unsigned Opc);
401 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
402 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
403 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
404 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
405 bool IsTupleInput, unsigned Opc);
406 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
407
408 template <unsigned MaxIdx, unsigned Scale>
409 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
410 unsigned Op);
411 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
412 unsigned Op, unsigned MaxIdx, unsigned Scale,
413 unsigned BaseReg = 0);
414 /// SVE Reg+Imm addressing mode.
415 template <int64_t Min, int64_t Max>
416 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
417 SDValue &OffImm);
418 /// SVE Reg+Reg address mode.
419 template <unsigned Scale>
420 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
421 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
422 }
423
424 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
425 unsigned Opc, uint32_t MaxImm);
426 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
427
428 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
429 unsigned NumInVecs);
430
431 template <unsigned MaxIdx, unsigned Scale>
432 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
433 return SelectSMETileSlice(N, MaxSize: MaxIdx, Vector, Offset, Scale);
434 }
435
436 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
437 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
438 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
439 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
440 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
441 unsigned Opc_rr, unsigned Opc_ri);
442 std::tuple<unsigned, SDValue, SDValue>
443 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
444 const SDValue &OldBase, const SDValue &OldOffset,
445 unsigned Scale);
446
447 bool tryBitfieldExtractOp(SDNode *N);
448 bool tryBitfieldInsertOp(SDNode *N);
449 bool tryBitfieldInsertInZeroOp(SDNode *N);
450
451 bool tryReadRegister(SDNode *N);
452 bool tryWriteRegister(SDNode *N);
453
454 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
455 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
456
457 bool trySelectXAR(SDNode *N);
458
459 bool tryFoldCselToFMaxMin(SDNode *N);
460
461// Include the pieces autogenerated from the target description.
462#include "AArch64GenDAGISel.inc"
463
464private:
465 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
466 SDValue &Shift);
467 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
468 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
469 SDValue &OffImm) {
470 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 7, Size, Base, OffImm);
471 }
472 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
473 unsigned Size, SDValue &Base,
474 SDValue &OffImm);
475 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
476 SDValue &OffImm);
477 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
478 SDValue &OffImm);
479 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
480 SDValue &Offset, SDValue &SignExtend,
481 SDValue &DoShift);
482 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
483 SDValue &Offset, SDValue &SignExtend,
484 SDValue &DoShift);
485 bool isWorthNegatingImm(SDValue V) const;
486 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
487 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
488 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
489 SDValue &Offset, SDValue &SignExtend);
490
491 template<unsigned RegWidth>
492 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
493 return SelectCVTFixedPosOperand(N, FixedPos, Width: RegWidth);
494 }
495 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
496
497 template <unsigned RegWidth>
498 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
499 return SelectCVTFixedPointVec(N, FixedPos, Width: RegWidth);
500 }
501 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
502
503 template<unsigned RegWidth>
504 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
505 return SelectCVTFixedPosRecipOperand(N, FixedPos, Width: RegWidth);
506 }
507
508 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
509 unsigned Width);
510
511 template <unsigned FloatWidth>
512 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
513 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, Width: FloatWidth);
514 }
515
516 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
517 unsigned Width);
518
519 bool SelectCMP_SWAP(SDNode *N);
520
521 AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
522 bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const;
523
524 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
525 bool Negate);
526 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
527 SDValue &Shift, bool Negate);
528 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
529 bool Negate);
530 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
531 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
532
533 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
534 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
535 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
536 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
537 bool SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm);
538 bool SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm);
539
540 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
541 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
542 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
543 bool AllowSaturation, SDValue &Imm);
544
545 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
546 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
547 SDValue &Offset);
548 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
549 SDValue &Offset, unsigned Scale = 1);
550
551 bool SelectAllActivePredicate(SDValue N);
552 bool SelectAnyPredicate(SDValue N);
553
554 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
555
556 template <bool MatchCBB>
557 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
558};
559
560class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
561public:
562 static char ID;
563 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
564 CodeGenOptLevel OptLevel)
565 : SelectionDAGISelLegacy(
566 ID, std::make_unique<AArch64DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
567};
568} // end anonymous namespace
569
570char AArch64DAGToDAGISelLegacy::ID = 0;
571
572INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
573
574AArch64DAGToDAGISelPass::AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
575 : SelectionDAGISelPass(
576 std::make_unique<AArch64DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
577
578/// addBitcastHints - This method adds bitcast hints to the operands of a node
579/// to help instruction selector determine which operands are in Neon registers.
580static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N) {
581 SDLoc DL(&N);
582 auto getFloatVT = [&](EVT VT) {
583 EVT ScalarVT = VT.getScalarType();
584 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
585 return VT.changeElementType(Context&: *(DAG.getContext()),
586 EltVT: ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
587 };
588 SmallVector<SDValue, 2> NewOps;
589 NewOps.reserve(N: N.getNumOperands());
590
591 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
592 auto bitcasted = DAG.getBitcast(VT: getFloatVT(N.getOperand(Num: I).getValueType()),
593 V: N.getOperand(Num: I));
594 NewOps.push_back(Elt: bitcasted);
595 }
596 EVT OrigVT = N.getValueType(ResNo: 0);
597 SDValue OpNode = DAG.getNode(Opcode: N.getOpcode(), DL, VT: getFloatVT(OrigVT), Ops: NewOps);
598 return DAG.getBitcast(VT: OrigVT, V: OpNode);
599}
600
601/// isIntImmediate - This method tests to see if the node is a constant
602/// operand. If so Imm will receive the 64-bit value.
603static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
604 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(Val: N)) {
605 Imm = C->getZExtValue();
606 return true;
607 }
608 return false;
609}
610
611// isIntImmediate - This method tests to see if a constant operand.
612// If so Imm will receive the value.
613static bool isIntImmediate(SDValue N, uint64_t &Imm) {
614 return isIntImmediate(N: N.getNode(), Imm);
615}
616
617// isOpcWithIntImmediate - This method tests to see if the node is a specific
618// opcode and that it has a immediate integer right operand.
619// If so Imm will receive the 32 bit value.
620static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
621 uint64_t &Imm) {
622 return N->getOpcode() == Opc &&
623 isIntImmediate(N: N->getOperand(Num: 1).getNode(), Imm);
624}
625
626// isIntImmediateEq - This method tests to see if N is a constant operand that
627// is equivalent to 'ImmExpected'.
628#ifndef NDEBUG
629static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
630 uint64_t Imm;
631 if (!isIntImmediate(N.getNode(), Imm))
632 return false;
633 return Imm == ImmExpected;
634}
635#endif
636
637static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
638 assert(RegWidth == 32 || RegWidth == 64);
639 if (RegWidth == 32)
640 return APInt(RegWidth,
641 uint32_t(AArch64_AM::decodeAdvSIMDModImmType11(Imm)));
642 return APInt(RegWidth, AArch64_AM::decodeAdvSIMDModImmType12(Imm));
643}
644
645// Decodes the raw integer splat value from a NEON splat operation.
646static std::optional<APInt> DecodeNEONSplat(SDValue N,
647 const AArch64Subtarget *Subtarget) {
648 assert(N.getValueType().isInteger() && "Only integers are supported");
649 if (N->getOpcode() == AArch64ISD::NVCAST ||
650 (N->getOpcode() == ISD::BITCAST && Subtarget->isLittleEndian()))
651 N = N->getOperand(Num: 0);
652 unsigned SplatWidth = N.getScalarValueSizeInBits();
653 if (N.getOpcode() == AArch64ISD::FMOV)
654 return DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth: SplatWidth);
655 if (N->getOpcode() == AArch64ISD::MOVI)
656 return APInt(SplatWidth, N.getConstantOperandVal(i: 0));
657 if (N->getOpcode() == AArch64ISD::MOVIshift)
658 return APInt(SplatWidth, N.getConstantOperandVal(i: 0)
659 << N.getConstantOperandVal(i: 1));
660 if (N->getOpcode() == AArch64ISD::MVNIshift)
661 return ~APInt(SplatWidth, N.getConstantOperandVal(i: 0)
662 << N.getConstantOperandVal(i: 1));
663 if (N->getOpcode() == AArch64ISD::MOVIedit)
664 return APInt(SplatWidth, AArch64_AM::decodeAdvSIMDModImmType10(
665 Imm: N.getConstantOperandVal(i: 0)));
666 if (N->getOpcode() == AArch64ISD::DUP)
667 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
668 return Const->getAPIntValue().trunc(width: SplatWidth);
669 APInt SplatVal;
670 if (ISD::isConstantSplatVector(N: N.getNode(), SplatValue&: SplatVal))
671 return SplatVal.trunc(width: SplatWidth);
672 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
673 // in AArch64ISelLowering.
674 return std::nullopt;
675}
676
677// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
678// matching the element size of N.
679static std::optional<APInt>
680GetNEONSplatValue(SDValue N, const AArch64Subtarget *Subtarget) {
681 unsigned SplatWidth = N.getScalarValueSizeInBits();
682 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N, Subtarget)) {
683 if (SplatVal->getBitWidth() <= SplatWidth)
684 return APInt::getSplat(NewLen: SplatWidth, V: *SplatVal);
685 if (SplatVal->isSplat(SplatSizeInBits: SplatWidth))
686 return SplatVal->trunc(width: SplatWidth);
687 }
688 return std::nullopt;
689}
690
691bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
692 SDValue &Imm) {
693 std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget);
694 if (!ImmVal)
695 return false;
696 uint64_t Encoding;
697 if (!AArch64_AM::isSVELogicalImm(SizeInBits: N.getScalarValueSizeInBits(),
698 ImmVal: ImmVal->getZExtValue(), Encoding))
699 return false;
700
701 Imm = CurDAG->getTargetConstant(Val: Encoding, DL: SDLoc(N), VT: MVT::i64);
702 return true;
703}
704
705bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
706 SDValue &Shift) {
707 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
708 return SelectSVEAddSubImm(DL: SDLoc(N), Value: *ImmVal,
709 VT: N.getValueType().getScalarType().getSimpleVT(),
710 Imm, Shift,
711 /*Negate=*/false);
712 return false;
713}
714
715bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
716 SDValue &Imm) {
717 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
718 return SelectSVESignedArithImm(DL: SDLoc(N), Value: *ImmVal, Imm);
719 return false;
720}
721
722bool AArch64DAGToDAGISel::SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm) {
723 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
724 if (!ImmAPIntVal)
725 return false;
726
727 int64_t ImmVal = ImmAPIntVal->getSExtValue();
728 if (ImmVal < -128 || ImmVal > 127)
729 return false;
730
731 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
732 return true;
733}
734
735bool AArch64DAGToDAGISel::SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm) {
736 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
737 if (!ImmAPIntVal)
738 return false;
739
740 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
741 if (ImmVal > 255)
742 return false;
743
744 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
745 return true;
746}
747
748bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
749 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
750 std::vector<SDValue> &OutOps) {
751 switch(ConstraintID) {
752 default:
753 llvm_unreachable("Unexpected asm memory constraint");
754 case InlineAsm::ConstraintCode::m:
755 case InlineAsm::ConstraintCode::o:
756 case InlineAsm::ConstraintCode::Q:
757 // We need to make sure that this one operand does not end up in XZR, thus
758 // require the address to be in a pointer register.
759 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
760 const TargetRegisterClass *TRC =
761 TII->getInlineAsmMemoryOperandRegClass(C: ConstraintID);
762 SDLoc dl(Op);
763 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL: dl, VT: MVT::i64);
764 SDValue NewOp =
765 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
766 dl, VT: Op.getValueType(),
767 Op1: Op, Op2: RC), 0);
768 OutOps.push_back(x: NewOp);
769 return false;
770 }
771 return true;
772}
773
774// Returns true if every use of N is a variable shift (SHL/SRL/SRA/ROTR) that
775// uses N as its shift-amount operand. When that holds, folding N into the
776// shift amount (e.g. generating a NEG or MVN) is safe even with multiple
777// uses, because each shift only reads the low bits of the amount.
778static bool allUsesAreShiftAmounts(SDValue N) {
779 return all_of(Range: N.getNode()->users(), P: [&](SDNode *User) {
780 switch (User->getOpcode()) {
781 case ISD::SHL:
782 case ISD::SRL:
783 case ISD::SRA:
784 case ISD::ROTR:
785 return User->getOperand(Num: 1).getNode() == N.getNode();
786 case ISD::ANY_EXTEND:
787 case ISD::ZERO_EXTEND:
788 case ISD::SIGN_EXTEND:
789 // A 32-bit shift amount may be widened to i64 before it reaches the
790 // shift; look through the extend to the ultimate users.
791 return allUsesAreShiftAmounts(N: SDValue(User, 0));
792 default:
793 return false;
794 }
795 });
796}
797
798template <unsigned ShiftWidth>
799bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
800 // AArch64 shift instructions only use the low log2(ShiftWidth) bits of the
801 // shift amount. The matcher may run on a node wider or narrower than the
802 // shift result (an i32 amount widened to i64, or an i64 modulus reduced to
803 // an i32 subtract), so the redundancy threshold and modulus come from the
804 // ShiftWidth template parameter, while the NEG/NOT opcode follows the node's
805 // own type. Return false if nothing was combined so other patterns (e.g.
806 // zext/sext GPR32 -> SUBREG_TO_REG) can match.
807 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(Val: N.getOperand(i: 1)) &&
808 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
809 uint64_t Mask = N.getConstantOperandVal(i: 1);
810 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
811 if ((unsigned)llvm::countr_one(Value: Mask) >= Log2_32(Value: ShiftWidth)) {
812 ShAmt = N.getOperand(i: 0);
813 return true;
814 }
815 }
816 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
817 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
818 // shift can use X directly; the original ADD/SUB stays for any other users.
819 if ((N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) &&
820 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
821 uint64_t Imm;
822 if (isIntImmediate(N: N.getOperand(i: 1).getNode(), Imm) &&
823 (Imm % ShiftWidth == 0)) {
824 ShAmt = N.getOperand(i: 0);
825 return true;
826 }
827 }
828
829 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
830 // to generate a NEG instead of a SUB from a constant.
831 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
832 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
833 uint64_t Imm;
834 if (isIntImmediate(N: N.getOperand(i: 0).getNode(), Imm) && Imm != 0 &&
835 (Imm % ShiftWidth == 0)) {
836 SDLoc DL(N);
837 EVT VT = N.getValueType();
838 bool Is32 = VT == MVT::i32;
839 unsigned NegOpc = Is32 ? AArch64::SUBWrr : AArch64::SUBXrr;
840 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
841 SDValue Zero =
842 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT);
843 MachineSDNode *Neg =
844 CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT, Op1: Zero, Op2: N.getOperand(i: 1));
845 ShAmt = SDValue(Neg, 0);
846 return true;
847 }
848 }
849
850 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
851 // to generate a NOT (MVN) instead of a SUB from a constant.
852 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
853 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
854 uint64_t Imm;
855 if (isIntImmediate(N: N.getOperand(i: 0).getNode(), Imm) &&
856 (Imm % ShiftWidth == ShiftWidth - 1)) {
857 SDLoc DL(N);
858 EVT VT = N.getValueType();
859 bool Is32 = VT == MVT::i32;
860 unsigned NotOpc = Is32 ? AArch64::ORNWrr : AArch64::ORNXrr;
861 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
862 SDValue Zero =
863 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT);
864 MachineSDNode *Not =
865 CurDAG->getMachineNode(Opcode: NotOpc, dl: DL, VT, Op1: Zero, Op2: N.getOperand(i: 1));
866 ShAmt = SDValue(Not, 0);
867 return true;
868 }
869 }
870
871 return false;
872}
873
874/// SelectArithImmed - Select an immediate value that can be represented as
875/// a 12-bit value shifted left by either 0 or 12. If so, return true with
876/// Val set to the 12-bit value and Shift set to the shifter operand.
877bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
878 SDValue &Shift) {
879 // This function is called from the addsub_shifted_imm ComplexPattern,
880 // which lists [imm] as the list of opcode it's interested in, however
881 // we still need to check whether the operand is actually an immediate
882 // here because the ComplexPattern opcode list is only used in
883 // root-level opcode matching.
884 if (!isa<ConstantSDNode>(Val: N.getNode()))
885 return false;
886
887 uint64_t Immed = N.getNode()->getAsZExtVal();
888
889 if (!AArch64_AM::isLegalArithImmed(C: Immed))
890 return false;
891
892 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(C: Immed);
893 Immed >>= ShiftAmt;
894
895 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
896 SDLoc dl(N);
897 Val = CurDAG->getTargetConstant(Val: Immed, DL: dl, VT: MVT::i32);
898 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: dl, VT: MVT::i32);
899 return true;
900}
901
902/// SelectNegArithImmed - As above, but negates the value before trying to
903/// select it.
904bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
905 SDValue &Shift) {
906 // This function is called from the addsub_shifted_imm ComplexPattern,
907 // which lists [imm] as the list of opcode it's interested in, however
908 // we still need to check whether the operand is actually an immediate
909 // here because the ComplexPattern opcode list is only used in
910 // root-level opcode matching.
911 if (!isa<ConstantSDNode>(Val: N.getNode()))
912 return false;
913
914 // The immediate operand must be a 24-bit zero-extended immediate.
915 uint64_t Immed = N.getNode()->getAsZExtVal();
916
917 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
918 // have the opposite effect on the C flag, so this pattern mustn't match under
919 // those circumstances.
920 if (Immed == 0)
921 return false;
922
923 if (N.getValueType() == MVT::i32)
924 Immed = ~((uint32_t)Immed) + 1;
925 else
926 Immed = ~Immed + 1ULL;
927 if (Immed & 0xFFFFFFFFFF000000ULL)
928 return false;
929
930 Immed &= 0xFFFFFFULL;
931 return SelectArithImmed(N: CurDAG->getConstant(Val: Immed, DL: SDLoc(N), VT: MVT::i32), Val,
932 Shift);
933}
934
935/// getShiftTypeForNode - Translate a shift node to the corresponding
936/// ShiftType value.
937static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
938 switch (N.getOpcode()) {
939 default:
940 return AArch64_AM::InvalidShiftExtend;
941 case ISD::SHL:
942 return AArch64_AM::LSL;
943 case ISD::SRL:
944 return AArch64_AM::LSR;
945 case ISD::SRA:
946 return AArch64_AM::ASR;
947 case ISD::ROTR:
948 return AArch64_AM::ROR;
949 }
950}
951
952static bool isMemOpOrPrefetch(SDNode *N) {
953 return isa<MemSDNode>(Val: *N) || N->getOpcode() == AArch64ISD::PREFETCH;
954}
955
956/// Determine whether it is worth it to fold SHL into the addressing
957/// mode.
958static bool isWorthFoldingSHL(SDValue V) {
959 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
960 // It is worth folding logical shift of up to three places.
961 auto *CSD = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
962 if (!CSD)
963 return false;
964 unsigned ShiftVal = CSD->getZExtValue();
965 if (ShiftVal > 3)
966 return false;
967
968 // Check if this particular node is reused in any non-memory related
969 // operation. If yes, do not try to fold this node into the address
970 // computation, since the computation will be kept.
971 const SDNode *Node = V.getNode();
972 for (SDNode *UI : Node->users())
973 if (!isMemOpOrPrefetch(N: UI))
974 for (SDNode *UII : UI->users())
975 if (!isMemOpOrPrefetch(N: UII))
976 return false;
977 return true;
978}
979
980/// Determine whether it is worth to fold V into an extended register addressing
981/// mode.
982bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
983 // Trivial if we are optimizing for code size or if there is only
984 // one use of the value.
985 if (CurDAG->shouldOptForSize() || V.hasOneUse())
986 return true;
987
988 // If a subtarget has a slow shift, folding a shift into multiple loads
989 // costs additional micro-ops.
990 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
991 return false;
992
993 // Check whether we're going to emit the address arithmetic anyway because
994 // it's used by a non-address operation.
995 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
996 return true;
997 if (V.getOpcode() == ISD::ADD) {
998 const SDValue LHS = V.getOperand(i: 0);
999 const SDValue RHS = V.getOperand(i: 1);
1000 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: LHS))
1001 return true;
1002 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: RHS))
1003 return true;
1004 }
1005
1006 // It hurts otherwise, since the value will be reused.
1007 return false;
1008}
1009
1010/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
1011/// to select more shifted register
1012bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
1013 SDValue &Shift) {
1014 EVT VT = N.getValueType();
1015 if (VT != MVT::i32 && VT != MVT::i64)
1016 return false;
1017
1018 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
1019 return false;
1020 SDValue LHS = N.getOperand(i: 0);
1021 if (!LHS->hasOneUse())
1022 return false;
1023
1024 unsigned LHSOpcode = LHS->getOpcode();
1025 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
1026 return false;
1027
1028 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
1029 if (!ShiftAmtNode)
1030 return false;
1031
1032 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
1033 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1034 if (!RHSC)
1035 return false;
1036
1037 APInt AndMask = RHSC->getAPIntValue();
1038 unsigned LowZBits, MaskLen;
1039 if (!AndMask.isShiftedMask(MaskIdx&: LowZBits, MaskLen))
1040 return false;
1041
1042 unsigned BitWidth = N.getValueSizeInBits();
1043 SDLoc DL(LHS);
1044 uint64_t NewShiftC;
1045 unsigned NewShiftOp;
1046 if (LHSOpcode == ISD::SHL) {
1047 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
1048 // BitWidth != LowZBits + MaskLen doesn't match the pattern
1049 if (LowZBits <= ShiftAmtC || (BitWidth != LowZBits + MaskLen))
1050 return false;
1051
1052 NewShiftC = LowZBits - ShiftAmtC;
1053 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1054 } else {
1055 if (LowZBits == 0)
1056 return false;
1057
1058 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
1059 NewShiftC = LowZBits + ShiftAmtC;
1060 if (NewShiftC >= BitWidth)
1061 return false;
1062
1063 // SRA need all high bits
1064 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
1065 return false;
1066
1067 // SRL high bits can be 0 or 1
1068 if (LHSOpcode == ISD::SRL && (BitWidth > (NewShiftC + MaskLen)))
1069 return false;
1070
1071 if (LHSOpcode == ISD::SRL)
1072 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1073 else
1074 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
1075 }
1076
1077 assert(NewShiftC < BitWidth && "Invalid shift amount");
1078 SDValue NewShiftAmt = CurDAG->getTargetConstant(Val: NewShiftC, DL, VT);
1079 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT);
1080 Reg = SDValue(CurDAG->getMachineNode(Opcode: NewShiftOp, dl: DL, VT, Op1: LHS->getOperand(Num: 0),
1081 Op2: NewShiftAmt, Op3: BitWidthMinus1),
1082 0);
1083 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LowZBits);
1084 Shift = CurDAG->getTargetConstant(Val: ShVal, DL, VT: MVT::i32);
1085 return true;
1086}
1087
1088/// getExtendTypeForNode - Translate an extend node to the corresponding
1089/// ExtendType value.
1090static AArch64_AM::ShiftExtendType
1091getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
1092 if (N.getOpcode() == ISD::SIGN_EXTEND ||
1093 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1094 EVT SrcVT;
1095 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
1096 SrcVT = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
1097 else
1098 SrcVT = N.getOperand(i: 0).getValueType();
1099
1100 if (!IsLoadStore && SrcVT == MVT::i8)
1101 return AArch64_AM::SXTB;
1102 else if (!IsLoadStore && SrcVT == MVT::i16)
1103 return AArch64_AM::SXTH;
1104 else if (SrcVT == MVT::i32)
1105 return AArch64_AM::SXTW;
1106 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1107
1108 return AArch64_AM::InvalidShiftExtend;
1109 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
1110 N.getOpcode() == ISD::ANY_EXTEND) {
1111 EVT SrcVT = N.getOperand(i: 0).getValueType();
1112 if (!IsLoadStore && SrcVT == MVT::i8)
1113 return AArch64_AM::UXTB;
1114 else if (!IsLoadStore && SrcVT == MVT::i16)
1115 return AArch64_AM::UXTH;
1116 else if (SrcVT == MVT::i32)
1117 return AArch64_AM::UXTW;
1118 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1119
1120 return AArch64_AM::InvalidShiftExtend;
1121 } else if (N.getOpcode() == ISD::AND) {
1122 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1123 if (!CSD)
1124 return AArch64_AM::InvalidShiftExtend;
1125 uint64_t AndMask = CSD->getZExtValue();
1126
1127 switch (AndMask) {
1128 default:
1129 return AArch64_AM::InvalidShiftExtend;
1130 case 0xFF:
1131 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
1132 case 0xFFFF:
1133 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
1134 case 0xFFFFFFFF:
1135 return AArch64_AM::UXTW;
1136 }
1137 }
1138
1139 return AArch64_AM::InvalidShiftExtend;
1140}
1141
1142/// Determine whether constant -V is cheaper to materialise than V.
1143bool AArch64DAGToDAGISel::isWorthNegatingImm(SDValue V) const {
1144 assert(isa<ConstantSDNode>(V) && "invalid node");
1145
1146 EVT VT = V.getValueType();
1147 assert((VT == MVT::i32 || VT == MVT::i64) && "invalid type");
1148
1149 // It's only worth negating the constant if it doesn't have other uses.
1150 if (!V.hasOneUse())
1151 return false;
1152
1153 uint64_t Imm = cast<ConstantSDNode>(Val&: V)->getZExtValue();
1154 unsigned BitSize = VT.getSizeInBits();
1155 SmallVector<AArch64_IMM::ImmInsnModel, 4> OrigCost, NewCost;
1156 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: OrigCost);
1157 AArch64_IMM::expandMOVImm(Imm: -Imm, BitSize, Insn&: NewCost);
1158 return NewCost.size() < OrigCost.size();
1159}
1160
1161/// Determine whether it is worth to fold V into an extended register of an
1162/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
1163/// instruction, and the shift should be treated as worth folding even if has
1164/// multiple uses.
1165bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1166 // Trivial if we are optimizing for code size or if there is only
1167 // one use of the value.
1168 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1169 return true;
1170
1171 // If a subtarget has a fastpath LSL we can fold a logical shift into
1172 // the add/sub and save a cycle.
1173 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1174 V.getConstantOperandVal(i: 1) <= 4 &&
1175 getExtendTypeForNode(N: V.getOperand(i: 0)) == AArch64_AM::InvalidShiftExtend)
1176 return true;
1177
1178 // It hurts otherwise, since the value will be reused.
1179 return false;
1180}
1181
1182/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1183/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1184/// instructions allow the shifted register to be rotated, but the arithmetic
1185/// instructions do not. The AllowROR parameter specifies whether ROR is
1186/// supported.
1187bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1188 SDValue &Reg, SDValue &Shift) {
1189 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1190 return true;
1191
1192 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
1193 if (ShType == AArch64_AM::InvalidShiftExtend)
1194 return false;
1195 if (!AllowROR && ShType == AArch64_AM::ROR)
1196 return false;
1197
1198 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1199 unsigned BitSize = N.getValueSizeInBits();
1200 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1201 unsigned ShVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
1202
1203 Reg = N.getOperand(i: 0);
1204 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: SDLoc(N), VT: MVT::i32);
1205 return isWorthFoldingALU(V: N, LSL: true);
1206 }
1207
1208 return false;
1209}
1210
1211/// Instructions that accept extend modifiers like UXTW expect the register
1212/// being extended to be a GPR32, but the incoming DAG might be acting on a
1213/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1214/// this is the case.
1215static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
1216 if (N.getValueType() == MVT::i32)
1217 return N;
1218
1219 SDLoc dl(N);
1220 return CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i32, Operand: N);
1221}
1222
1223// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1224template<signed Low, signed High, signed Scale>
1225bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1226 if (!isa<ConstantSDNode>(Val: N))
1227 return false;
1228
1229 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
1230 if ((MulImm % std::abs(x: Scale)) == 0) {
1231 int64_t RDVLImm = MulImm / Scale;
1232 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1233 Imm = CurDAG->getSignedTargetConstant(Val: RDVLImm, DL: SDLoc(N), VT: MVT::i32);
1234 return true;
1235 }
1236 }
1237
1238 return false;
1239}
1240
1241// Returns a suitable RDSVL multiplier from a left shift.
1242template <signed Low, signed High>
1243bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1244 if (!isa<ConstantSDNode>(Val: N))
1245 return false;
1246
1247 int64_t MulImm = 1LL << cast<ConstantSDNode>(Val&: N)->getSExtValue();
1248 if (MulImm >= Low && MulImm <= High) {
1249 Imm = CurDAG->getSignedTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
1250 return true;
1251 }
1252
1253 return false;
1254}
1255
1256/// SelectArithExtendedRegister - Select a "extended register" operand. This
1257/// operand folds in an extend followed by an optional left shift.
1258bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1259 SDValue &Shift) {
1260 unsigned ShiftVal = 0;
1261 AArch64_AM::ShiftExtendType Ext;
1262
1263 if (N.getOpcode() == ISD::SHL) {
1264 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1265 if (!CSD)
1266 return false;
1267 ShiftVal = CSD->getZExtValue();
1268 if (ShiftVal > 4)
1269 return false;
1270
1271 Ext = getExtendTypeForNode(N: N.getOperand(i: 0));
1272 if (Ext == AArch64_AM::InvalidShiftExtend)
1273 return false;
1274
1275 Reg = N.getOperand(i: 0).getOperand(i: 0);
1276 } else {
1277 Ext = getExtendTypeForNode(N);
1278 if (Ext == AArch64_AM::InvalidShiftExtend)
1279 return false;
1280
1281 // Don't match sext of vector extracts. These can use SMOV, but if we match
1282 // this as an extended register, we'll always fold the extend into an ALU op
1283 // user of the extend (which results in a UMOV).
1284 if (AArch64_AM::isSignExtendShiftType(Type: Ext)) {
1285 SDValue Op = N.getOperand(i: 0);
1286 if (Op->getOpcode() == ISD::ANY_EXTEND)
1287 Op = Op->getOperand(Num: 0);
1288 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1289 Op.getOperand(i: 0).getValueType().isFixedLengthVector())
1290 return false;
1291 }
1292
1293 Reg = N.getOperand(i: 0);
1294
1295 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1296 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1297 auto isDef32 = [](SDValue N) {
1298 unsigned Opc = N.getOpcode();
1299 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1300 Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
1301 Opc != ISD::AssertZext && Opc != ISD::AssertAlign &&
1302 Opc != ISD::FREEZE;
1303 };
1304 if (Ext == AArch64_AM::UXTW && Reg->getValueType(ResNo: 0).getSizeInBits() == 32 &&
1305 isDef32(Reg))
1306 return false;
1307 }
1308
1309 // Don't match if the sext can be folded with an asr to form an SBFX.
1310 if (Ext == AArch64_AM::SXTW && Reg.getOpcode() == ISD::SRA &&
1311 Reg.getValueType() == MVT::i32 &&
1312 isa<ConstantSDNode>(Val: Reg.getOperand(i: 1)) && Reg.hasOneUse())
1313 return false;
1314
1315 // AArch64 mandates that the RHS of the operation must use the smallest
1316 // register class that could contain the size being extended from. Thus,
1317 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1318 // there might not be an actual 32-bit value in the program. We can
1319 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1320 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1321 Reg = narrowIfNeeded(CurDAG, N: Reg);
1322 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1323 VT: MVT::i32);
1324 return isWorthFoldingALU(V: N);
1325}
1326
1327/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1328/// operand is referred by the instructions have SP operand
1329bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1330 SDValue &Shift) {
1331 unsigned ShiftVal = 0;
1332 AArch64_AM::ShiftExtendType Ext;
1333
1334 if (N.getOpcode() != ISD::SHL)
1335 return false;
1336
1337 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1338 if (!CSD)
1339 return false;
1340 ShiftVal = CSD->getZExtValue();
1341 if (ShiftVal > 4)
1342 return false;
1343
1344 Ext = AArch64_AM::UXTX;
1345 Reg = N.getOperand(i: 0);
1346 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1347 VT: MVT::i32);
1348 return isWorthFoldingALU(V: N);
1349}
1350
1351/// If there's a use of this ADDlow that's not itself a load/store then we'll
1352/// need to create a real ADD instruction from it anyway and there's no point in
1353/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1354/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1355/// leads to duplicated ADRP instructions.
1356static bool isWorthFoldingADDlow(SDValue N) {
1357 for (auto *User : N->users()) {
1358 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1359 User->getOpcode() != ISD::ATOMIC_LOAD &&
1360 User->getOpcode() != ISD::ATOMIC_STORE)
1361 return false;
1362
1363 // ldar and stlr have much more restrictive addressing modes (just a
1364 // register).
1365 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
1366 return false;
1367 }
1368
1369 return true;
1370}
1371
1372/// Check if the immediate offset is valid as a scaled immediate.
1373static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1374 unsigned Size) {
1375 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1376 Offset < (Range << Log2_32(Value: Size)))
1377 return true;
1378 return false;
1379}
1380
1381/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1382/// immediate" address. The "Size" argument is the size in bytes of the memory
1383/// reference, which determines the scale.
1384bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1385 unsigned BW, unsigned Size,
1386 SDValue &Base,
1387 SDValue &OffImm) {
1388 SDLoc dl(N);
1389 const DataLayout &DL = CurDAG->getDataLayout();
1390 const TargetLowering *TLI = getTargetLowering();
1391 if (N.getOpcode() == ISD::FrameIndex) {
1392 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1393 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1394 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1395 return true;
1396 }
1397
1398 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1399 // selected here doesn't support labels/immediates, only base+offset.
1400 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1401 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1402 if (IsSignedImm) {
1403 int64_t RHSC = RHS->getSExtValue();
1404 unsigned Scale = Log2_32(Value: Size);
1405 int64_t Range = 0x1LL << (BW - 1);
1406
1407 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1408 RHSC < (Range << Scale)) {
1409 Base = N.getOperand(i: 0);
1410 if (Base.getOpcode() == ISD::FrameIndex) {
1411 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1412 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1413 }
1414 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1415 return true;
1416 }
1417 } else {
1418 // unsigned Immediate
1419 uint64_t RHSC = RHS->getZExtValue();
1420 unsigned Scale = Log2_32(Value: Size);
1421 uint64_t Range = 0x1ULL << BW;
1422
1423 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1424 Base = N.getOperand(i: 0);
1425 if (Base.getOpcode() == ISD::FrameIndex) {
1426 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1427 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1428 }
1429 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1430 return true;
1431 }
1432 }
1433 }
1434 }
1435 // Base only. The address will be materialized into a register before
1436 // the memory is accessed.
1437 // add x0, Xbase, #offset
1438 // stp x1, x2, [x0]
1439 Base = N;
1440 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1441 return true;
1442}
1443
1444/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1445/// immediate" address. The "Size" argument is the size in bytes of the memory
1446/// reference, which determines the scale.
1447bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1448 SDValue &Base, SDValue &OffImm) {
1449 SDLoc dl(N);
1450 const DataLayout &DL = CurDAG->getDataLayout();
1451 const TargetLowering *TLI = getTargetLowering();
1452 if (N.getOpcode() == ISD::FrameIndex) {
1453 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1454 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1455 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1456 return true;
1457 }
1458
1459 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1460 GlobalAddressSDNode *GAN =
1461 dyn_cast<GlobalAddressSDNode>(Val: N.getOperand(i: 1).getNode());
1462 Base = N.getOperand(i: 0);
1463 OffImm = N.getOperand(i: 1);
1464 if (!GAN)
1465 return true;
1466
1467 if (GAN->getOffset() % Size == 0 &&
1468 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1469 return true;
1470 }
1471
1472 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1473 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1474 int64_t RHSC = (int64_t)RHS->getZExtValue();
1475 unsigned Scale = Log2_32(Value: Size);
1476 if (isValidAsScaledImmediate(Offset: RHSC, Range: 0x1000, Size)) {
1477 Base = N.getOperand(i: 0);
1478 if (Base.getOpcode() == ISD::FrameIndex) {
1479 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1480 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1481 }
1482 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1483 return true;
1484 }
1485 }
1486 }
1487
1488 // Before falling back to our general case, check if the unscaled
1489 // instructions can handle this. If so, that's preferable.
1490 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1491 return false;
1492
1493 // Base only. The address will be materialized into a register before
1494 // the memory is accessed.
1495 // add x0, Xbase, #offset
1496 // ldr x0, [x0]
1497 Base = N;
1498 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1499 return true;
1500}
1501
1502/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1503/// immediate" address. This should only match when there is an offset that
1504/// is not valid for a scaled immediate addressing mode. The "Size" argument
1505/// is the size in bytes of the memory reference, which is needed here to know
1506/// what is valid for a scaled immediate.
1507bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1508 SDValue &Base,
1509 SDValue &OffImm) {
1510 if (!CurDAG->isBaseWithConstantOffset(Op: N))
1511 return false;
1512 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1513 int64_t RHSC = RHS->getSExtValue();
1514 if (RHSC >= -256 && RHSC < 256) {
1515 Base = N.getOperand(i: 0);
1516 if (Base.getOpcode() == ISD::FrameIndex) {
1517 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1518 const TargetLowering *TLI = getTargetLowering();
1519 Base = CurDAG->getTargetFrameIndex(
1520 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1521 }
1522 OffImm = CurDAG->getTargetConstant(Val: RHSC, DL: SDLoc(N), VT: MVT::i64);
1523 return true;
1524 }
1525 }
1526 return false;
1527}
1528
1529static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
1530 SDLoc dl(N);
1531 SDValue ImpDef = SDValue(
1532 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl, VT: MVT::i64), 0);
1533 return CurDAG->getTargetInsertSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i64, Operand: ImpDef,
1534 Subreg: N);
1535}
1536
1537/// Check if the given SHL node (\p N), can be used to form an
1538/// extended register for an addressing mode.
1539bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1540 bool WantExtend, SDValue &Offset,
1541 SDValue &SignExtend) {
1542 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1543 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1544 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1545 return false;
1546
1547 SDLoc dl(N);
1548 if (WantExtend) {
1549 AArch64_AM::ShiftExtendType Ext =
1550 getExtendTypeForNode(N: N.getOperand(i: 0), IsLoadStore: true);
1551 if (Ext == AArch64_AM::InvalidShiftExtend)
1552 return false;
1553
1554 Offset = narrowIfNeeded(CurDAG, N: N.getOperand(i: 0).getOperand(i: 0));
1555 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1556 VT: MVT::i32);
1557 } else {
1558 Offset = N.getOperand(i: 0);
1559 SignExtend = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1560 }
1561
1562 unsigned LegalShiftVal = Log2_32(Value: Size);
1563 unsigned ShiftVal = CSD->getZExtValue();
1564
1565 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1566 return false;
1567
1568 return isWorthFoldingAddr(V: N, Size);
1569}
1570
1571bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1572 SDValue &Base, SDValue &Offset,
1573 SDValue &SignExtend,
1574 SDValue &DoShift) {
1575 if (N.getOpcode() != ISD::ADD)
1576 return false;
1577 SDValue LHS = N.getOperand(i: 0);
1578 SDValue RHS = N.getOperand(i: 1);
1579 SDLoc dl(N);
1580
1581 // We don't want to match immediate adds here, because they are better lowered
1582 // to the register-immediate addressing modes.
1583 if (isa<ConstantSDNode>(Val: LHS) || isa<ConstantSDNode>(Val: RHS))
1584 return false;
1585
1586 // Check if this particular node is reused in any non-memory related
1587 // operation. If yes, do not try to fold this node into the address
1588 // computation, since the computation will be kept.
1589 const SDNode *Node = N.getNode();
1590 for (SDNode *UI : Node->users()) {
1591 if (!isMemOpOrPrefetch(N: UI))
1592 return false;
1593 }
1594
1595 // Remember if it is worth folding N when it produces extended register.
1596 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1597
1598 // Try to match a shifted extend on the RHS.
1599 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1600 SelectExtendedSHL(N: RHS, Size, WantExtend: true, Offset, SignExtend)) {
1601 Base = LHS;
1602 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1603 return true;
1604 }
1605
1606 // Try to match a shifted extend on the LHS.
1607 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1608 SelectExtendedSHL(N: LHS, Size, WantExtend: true, Offset, SignExtend)) {
1609 Base = RHS;
1610 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1611 return true;
1612 }
1613
1614 // There was no shift, whatever else we find.
1615 DoShift = CurDAG->getTargetConstant(Val: false, DL: dl, VT: MVT::i32);
1616
1617 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1618 // Try to match an unshifted extend on the LHS.
1619 if (IsExtendedRegisterWorthFolding &&
1620 (Ext = getExtendTypeForNode(N: LHS, IsLoadStore: true)) !=
1621 AArch64_AM::InvalidShiftExtend) {
1622 Base = RHS;
1623 Offset = narrowIfNeeded(CurDAG, N: LHS.getOperand(i: 0));
1624 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1625 VT: MVT::i32);
1626 if (isWorthFoldingAddr(V: LHS, Size))
1627 return true;
1628 }
1629
1630 // Try to match an unshifted extend on the RHS.
1631 if (IsExtendedRegisterWorthFolding &&
1632 (Ext = getExtendTypeForNode(N: RHS, IsLoadStore: true)) !=
1633 AArch64_AM::InvalidShiftExtend) {
1634 Base = LHS;
1635 Offset = narrowIfNeeded(CurDAG, N: RHS.getOperand(i: 0));
1636 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1637 VT: MVT::i32);
1638 if (isWorthFoldingAddr(V: RHS, Size))
1639 return true;
1640 }
1641
1642 return false;
1643}
1644
1645// Check if the given immediate is preferred by ADD. If an immediate can be
1646// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1647// encoded by one MOVZ, return true.
1648static bool isPreferredADD(int64_t ImmOff) {
1649 // Constant in [0x0, 0xfff] can be encoded in ADD.
1650 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1651 return true;
1652 // Check if it can be encoded in an "ADD LSL #12".
1653 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1654 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1655 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1656 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1657 return false;
1658}
1659
1660bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1661 SDValue &Base, SDValue &Offset,
1662 SDValue &SignExtend,
1663 SDValue &DoShift) {
1664 if (N.getOpcode() != ISD::ADD)
1665 return false;
1666 SDValue LHS = N.getOperand(i: 0);
1667 SDValue RHS = N.getOperand(i: 1);
1668 SDLoc DL(N);
1669
1670 // Check if this particular node is reused in any non-memory related
1671 // operation. If yes, do not try to fold this node into the address
1672 // computation, since the computation will be kept.
1673 const SDNode *Node = N.getNode();
1674 for (SDNode *UI : Node->users()) {
1675 if (!isMemOpOrPrefetch(N: UI))
1676 return false;
1677 }
1678
1679 // Watch out if RHS is a wide immediate, it can not be selected into
1680 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1681 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1682 // instructions like:
1683 // MOV X0, WideImmediate
1684 // ADD X1, BaseReg, X0
1685 // LDR X2, [X1, 0]
1686 // For such situation, using [BaseReg, XReg] addressing mode can save one
1687 // ADD/SUB:
1688 // MOV X0, WideImmediate
1689 // LDR X2, [BaseReg, X0]
1690 if (isa<ConstantSDNode>(Val: RHS)) {
1691 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1692 // Skip the immediate can be selected by load/store addressing mode.
1693 // Also skip the immediate can be encoded by a single ADD (SUB is also
1694 // checked by using -ImmOff).
1695 if (isValidAsScaledImmediate(Offset: ImmOff, Range: 0x1000, Size) ||
1696 isPreferredADD(ImmOff) || isPreferredADD(ImmOff: -ImmOff))
1697 return false;
1698
1699 SDValue Ops[] = { RHS };
1700 SDNode *MOVI =
1701 CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
1702 SDValue MOVIV = SDValue(MOVI, 0);
1703 // This ADD of two X register will be selected into [Reg+Reg] mode.
1704 N = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: LHS, N2: MOVIV);
1705 }
1706
1707 // Remember if it is worth folding N when it produces extended register.
1708 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1709
1710 // Try to match a shifted extend on the RHS.
1711 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1712 SelectExtendedSHL(N: RHS, Size, WantExtend: false, Offset, SignExtend)) {
1713 Base = LHS;
1714 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1715 return true;
1716 }
1717
1718 // Try to match a shifted extend on the LHS.
1719 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1720 SelectExtendedSHL(N: LHS, Size, WantExtend: false, Offset, SignExtend)) {
1721 Base = RHS;
1722 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1723 return true;
1724 }
1725
1726 // Match any non-shifted, non-extend, non-immediate add expression.
1727 Base = LHS;
1728 Offset = RHS;
1729 SignExtend = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1730 DoShift = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1731 // Reg1 + Reg2 is free: no check needed.
1732 return true;
1733}
1734
1735SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1736 static const unsigned RegClassIDs[] = {
1737 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1738 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1739 AArch64::dsub2, AArch64::dsub3};
1740
1741 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1742}
1743
1744SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1745 static const unsigned RegClassIDs[] = {
1746 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1747 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1748 AArch64::qsub2, AArch64::qsub3};
1749
1750 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1751}
1752
1753SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1754 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1755 AArch64::ZPR3RegClassID,
1756 AArch64::ZPR4RegClassID};
1757 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1758 AArch64::zsub2, AArch64::zsub3};
1759
1760 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1761}
1762
1763SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1764 assert(Regs.size() == 2 || Regs.size() == 4);
1765
1766 // The createTuple interface requires 3 RegClassIDs for each possible
1767 // tuple type even though we only have them for ZPR2 and ZPR4.
1768 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1769 AArch64::ZPR4Mul4RegClassID};
1770 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1771 AArch64::zsub2, AArch64::zsub3};
1772 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1773}
1774
1775SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1776 const unsigned RegClassIDs[],
1777 const unsigned SubRegs[]) {
1778 // There's no special register-class for a vector-list of 1 element: it's just
1779 // a vector.
1780 if (Regs.size() == 1)
1781 return Regs[0];
1782
1783 assert(Regs.size() >= 2 && Regs.size() <= 4);
1784
1785 SDLoc DL(Regs[0]);
1786
1787 SmallVector<SDValue, 4> Ops;
1788
1789 // First operand of REG_SEQUENCE is the desired RegClass.
1790 Ops.push_back(
1791 Elt: CurDAG->getTargetConstant(Val: RegClassIDs[Regs.size() - 2], DL, VT: MVT::i32));
1792
1793 // Then we get pairs of source & subregister-position for the components.
1794 for (unsigned i = 0; i < Regs.size(); ++i) {
1795 Ops.push_back(Elt: Regs[i]);
1796 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: SubRegs[i], DL, VT: MVT::i32));
1797 }
1798
1799 SDNode *N =
1800 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped, Ops);
1801 return SDValue(N, 0);
1802}
1803
1804void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1805 bool isExt) {
1806 SDLoc dl(N);
1807 EVT VT = N->getValueType(ResNo: 0);
1808
1809 unsigned ExtOff = isExt;
1810
1811 // Form a REG_SEQUENCE to force register allocation.
1812 unsigned Vec0Off = ExtOff + 1;
1813 SmallVector<SDValue, 4> Regs(N->ops().slice(N: Vec0Off, M: NumVecs));
1814 SDValue RegSeq = createQTuple(Regs);
1815
1816 SmallVector<SDValue, 6> Ops;
1817 if (isExt)
1818 Ops.push_back(Elt: N->getOperand(Num: 1));
1819 Ops.push_back(Elt: RegSeq);
1820 Ops.push_back(Elt: N->getOperand(Num: NumVecs + ExtOff + 1));
1821 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT, Ops));
1822}
1823
1824static std::tuple<SDValue, SDValue>
1825extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1826 SDLoc DL(Disc);
1827 SDValue AddrDisc;
1828 SDValue ConstDisc;
1829
1830 // If this is a blend, remember the constant and address discriminators.
1831 // Otherwise, it's either a constant discriminator, or a non-blended
1832 // address discriminator.
1833 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1834 Disc->getConstantOperandVal(Num: 0) == Intrinsic::ptrauth_blend) {
1835 AddrDisc = Disc->getOperand(Num: 1);
1836 ConstDisc = Disc->getOperand(Num: 2);
1837 } else {
1838 ConstDisc = Disc;
1839 }
1840
1841 // If the constant discriminator (either the blend RHS, or the entire
1842 // discriminator value) isn't a 16-bit constant, bail out, and let the
1843 // discriminator be computed separately.
1844 auto *ConstDiscN = dyn_cast<ConstantSDNode>(Val&: ConstDisc);
1845 if (!ConstDiscN || !isUInt<16>(x: ConstDiscN->getZExtValue()))
1846 return std::make_tuple(args: DAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), args&: Disc);
1847
1848 // If there's no address discriminator, use XZR directly.
1849 if (!AddrDisc)
1850 AddrDisc = DAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64);
1851
1852 return std::make_tuple(
1853 args: DAG->getTargetConstant(Val: ConstDiscN->getZExtValue(), DL, VT: MVT::i64),
1854 args&: AddrDisc);
1855}
1856
1857void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1858 SDLoc DL(N);
1859 // IntrinsicID is operand #0
1860 SDValue Val = N->getOperand(Num: 1);
1861 SDValue AUTKey = N->getOperand(Num: 2);
1862 SDValue AUTDisc = N->getOperand(Num: 3);
1863
1864 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1865 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1866
1867 SDValue AUTAddrDisc, AUTConstDisc;
1868 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1869 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1870
1871 if (!Subtarget->isX16X17Safer()) {
1872 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1873 // Copy deactivation symbol if present.
1874 if (N->getNumOperands() > 4)
1875 Ops.push_back(x: N->getOperand(Num: 4));
1876
1877 SDNode *AUT =
1878 CurDAG->getMachineNode(Opcode: AArch64::AUTxMxN, dl: DL, VT1: MVT::i64, VT2: MVT::i64, Ops);
1879 ReplaceNode(F: N, T: AUT);
1880 } else {
1881 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1882 Reg: AArch64::X16, N: Val, Glue: SDValue());
1883 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(R: 1)};
1884
1885 SDNode *AUT = CurDAG->getMachineNode(Opcode: AArch64::AUTx16x17, dl: DL, VT: MVT::i64, Ops);
1886 ReplaceNode(F: N, T: AUT);
1887 }
1888}
1889
1890void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1891 SDLoc DL(N);
1892 // IntrinsicID is operand #0, if W_CHAIN it is #1
1893 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1894 SDValue Val = N->getOperand(Num: OffsetBase + 1);
1895 SDValue AUTKey = N->getOperand(Num: OffsetBase + 2);
1896 SDValue AUTDisc = N->getOperand(Num: OffsetBase + 3);
1897 SDValue PACKey = N->getOperand(Num: OffsetBase + 4);
1898 SDValue PACDisc = N->getOperand(Num: OffsetBase + 5);
1899 uint32_t IntNum = N->getConstantOperandVal(Num: OffsetBase + 0);
1900 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
1901
1902 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1903 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1904
1905 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1906 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1907
1908 SDValue AUTAddrDisc, AUTConstDisc;
1909 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1910 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1911
1912 SDValue PACAddrDisc, PACConstDisc;
1913 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1914 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1915
1916 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1917 Reg: AArch64::X16, N: Val, Glue: SDValue());
1918
1919 if (HasLoad) {
1920 SDValue Addend = N->getOperand(Num: OffsetBase + 6);
1921 SDValue IncomingChain = N->getOperand(Num: 0);
1922 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1923 PACKey, PACConstDisc, PACAddrDisc,
1924 Addend, IncomingChain, X16Copy.getValue(R: 1)};
1925
1926 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTRELLOADPAC, dl: DL,
1927 VT1: MVT::i64, VT2: MVT::Other, Ops);
1928 ReplaceNode(F: N, T: AUTRELLOADPAC);
1929 } else {
1930 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1931 PACConstDisc, PACAddrDisc, X16Copy.getValue(R: 1)};
1932
1933 SDNode *AUTPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTPAC, dl: DL, VT: MVT::i64, Ops);
1934 ReplaceNode(F: N, T: AUTPAC);
1935 }
1936}
1937
1938void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
1939 SDLoc DL(N);
1940 SDValue Val = N->getOperand(Num: 1);
1941 SDValue AUTKey = N->getOperand(Num: 2);
1942 SDValue AUTDisc = N->getOperand(Num: 3);
1943 SDValue AUTPC = N->getOperand(Num: 4);
1944 SDValue PACKey = N->getOperand(Num: 5);
1945 SDValue PACDisc = N->getOperand(Num: 6);
1946
1947 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1948 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1949
1950 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1951 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1952
1953 SDValue PACAddrDisc, PACConstDisc;
1954 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1955 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1956
1957 SDValue X17Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1958 Reg: AArch64::X17, N: Val, Glue: SDValue());
1959 SDValue X16Copy = CurDAG->getCopyToReg(
1960 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X16, N: AUTDisc, Glue: X17Copy.getValue(R: 1));
1961 SDValue X15Copy = CurDAG->getCopyToReg(
1962 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X15, N: AUTPC, Glue: X16Copy.getValue(R: 1));
1963
1964 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
1965 X15Copy.getValue(R: 1)};
1966 SDNode *AUTPCPAC =
1967 CurDAG->getMachineNode(Opcode: AArch64::AUTPCPAC, dl: DL, VT: MVT::i64, Ops);
1968 ReplaceNode(F: N, T: AUTPCPAC);
1969}
1970
1971bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1972 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1973 if (LD->isUnindexed())
1974 return false;
1975 EVT VT = LD->getMemoryVT();
1976 EVT DstVT = N->getValueType(ResNo: 0);
1977 ISD::MemIndexedMode AM = LD->getAddressingMode();
1978 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1979 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(Val: LD->getOffset());
1980 int OffsetVal = (int)OffsetOp->getZExtValue();
1981
1982 // We're not doing validity checking here. That was done when checking
1983 // if we should mark the load as indexed or not. We're just selecting
1984 // the right instruction.
1985 unsigned Opcode = 0;
1986
1987 ISD::LoadExtType ExtType = LD->getExtensionType();
1988 bool InsertTo64 = false;
1989 bool UseLd1 =
1990 (VT.is64BitVector() || VT.is128BitVector()) &&
1991 (!Subtarget->isLittleEndian() || (Subtarget->requiresStrictAlign() &&
1992 LD->getAlign() < VT.getStoreSize()));
1993 if (VT == MVT::i64)
1994 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1995 else if (VT == MVT::i32) {
1996 if (ExtType == ISD::NON_EXTLOAD)
1997 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1998 else if (ExtType == ISD::SEXTLOAD)
1999 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
2000 else {
2001 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2002 InsertTo64 = true;
2003 // The result of the load is only i32. It's the subreg_to_reg that makes
2004 // it into an i64.
2005 DstVT = MVT::i32;
2006 }
2007 } else if (VT == MVT::i16) {
2008 if (ExtType == ISD::SEXTLOAD) {
2009 if (DstVT == MVT::i64)
2010 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
2011 else
2012 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
2013 } else {
2014 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
2015 InsertTo64 = DstVT == MVT::i64;
2016 // The result of the load is only i32. It's the subreg_to_reg that makes
2017 // it into an i64.
2018 DstVT = MVT::i32;
2019 }
2020 } else if (VT == MVT::i8) {
2021 if (ExtType == ISD::SEXTLOAD) {
2022 if (DstVT == MVT::i64)
2023 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
2024 else
2025 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
2026 } else {
2027 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
2028 InsertTo64 = DstVT == MVT::i64;
2029 // The result of the load is only i32. It's the subreg_to_reg that makes
2030 // it into an i64.
2031 DstVT = MVT::i32;
2032 }
2033 } else if (VT == MVT::f16) {
2034 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2035 } else if (VT == MVT::bf16) {
2036 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2037 } else if (VT == MVT::f32) {
2038 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
2039 } else if (VT == MVT::f64 || (VT.is64BitVector() && !UseLd1)) {
2040 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
2041 } else if (VT.is128BitVector() && !UseLd1) {
2042 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
2043 } else if (VT.is64BitVector() && UseLd1) {
2044 if (IsPre || OffsetVal != 8)
2045 return false;
2046 switch (VT.getScalarSizeInBits()) {
2047 case 8:
2048 Opcode = AArch64::LD1Onev8b_POST;
2049 break;
2050 case 16:
2051 Opcode = AArch64::LD1Onev4h_POST;
2052 break;
2053 case 32:
2054 Opcode = AArch64::LD1Onev2s_POST;
2055 break;
2056 case 64:
2057 Opcode = AArch64::LD1Onev1d_POST;
2058 break;
2059 default:
2060 llvm_unreachable("Expected vector element to be a power of 2");
2061 }
2062 } else if (VT.is128BitVector() && UseLd1) {
2063 if (IsPre || OffsetVal != 16)
2064 return false;
2065 switch (VT.getScalarSizeInBits()) {
2066 case 8:
2067 Opcode = AArch64::LD1Onev16b_POST;
2068 break;
2069 case 16:
2070 Opcode = AArch64::LD1Onev8h_POST;
2071 break;
2072 case 32:
2073 Opcode = AArch64::LD1Onev4s_POST;
2074 break;
2075 case 64:
2076 Opcode = AArch64::LD1Onev2d_POST;
2077 break;
2078 default:
2079 llvm_unreachable("Expected vector element to be a power of 2");
2080 }
2081 } else
2082 return false;
2083 SDValue Chain = LD->getChain();
2084 SDValue Base = LD->getBasePtr();
2085 SDLoc dl(N);
2086 // LD1 encodes an immediate offset by using XZR as the offset register.
2087 SDValue Offset = UseLd1 ? CurDAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64)
2088 : CurDAG->getTargetConstant(Val: OffsetVal, DL: dl, VT: MVT::i64);
2089 SDValue Ops[] = { Base, Offset, Chain };
2090 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, VT1: MVT::i64, VT2: DstVT,
2091 VT3: MVT::Other, Ops);
2092
2093 // Transfer memoperands.
2094 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
2095 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Res), NewMemRefs: {MemOp});
2096
2097 // Either way, we're replacing the node, so tell the caller that.
2098 SDValue LoadedVal = SDValue(Res, 1);
2099 if (InsertTo64) {
2100 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL: dl, VT: MVT::i32);
2101 LoadedVal = SDValue(CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl,
2102 VT: MVT::i64, Op1: LoadedVal, Op2: SubReg),
2103 0);
2104 }
2105
2106 ReplaceUses(F: SDValue(N, 0), T: LoadedVal);
2107 ReplaceUses(F: SDValue(N, 1), T: SDValue(Res, 0));
2108 ReplaceUses(F: SDValue(N, 2), T: SDValue(Res, 2));
2109 CurDAG->RemoveDeadNode(N);
2110 return true;
2111}
2112
2113void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
2114 unsigned SubRegIdx) {
2115 SDLoc dl(N);
2116 EVT VT = N->getValueType(ResNo: 0);
2117 SDValue Chain = N->getOperand(Num: 0);
2118
2119 SDValue Ops[] = {N->getOperand(Num: 2), // Mem operand;
2120 Chain};
2121
2122 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2123
2124 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2125 SDValue SuperReg = SDValue(Ld, 0);
2126 for (unsigned i = 0; i < NumVecs; ++i)
2127 ReplaceUses(F: SDValue(N, i),
2128 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2129
2130 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2131
2132 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
2133 // because it's too simple to have needed special treatment during lowering.
2134 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
2135 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2136 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2137 }
2138
2139 CurDAG->RemoveDeadNode(N);
2140}
2141
2142void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
2143 unsigned Opc, unsigned SubRegIdx) {
2144 SDLoc dl(N);
2145 EVT VT = N->getValueType(ResNo: 0);
2146 SDValue Chain = N->getOperand(Num: 0);
2147
2148 SDValue Ops[] = {N->getOperand(Num: 1), // Mem operand
2149 N->getOperand(Num: 2), // Incremental
2150 Chain};
2151
2152 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2153 MVT::Untyped, MVT::Other};
2154
2155 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2156
2157 // Update uses of write back register
2158 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2159
2160 // Update uses of vector list
2161 SDValue SuperReg = SDValue(Ld, 1);
2162 if (NumVecs == 1)
2163 ReplaceUses(F: SDValue(N, 0), T: SuperReg);
2164 else
2165 for (unsigned i = 0; i < NumVecs; ++i)
2166 ReplaceUses(F: SDValue(N, i),
2167 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2168
2169 // Transfer memoperands.
2170 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2171 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2172
2173 // Update the chain
2174 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2175 CurDAG->RemoveDeadNode(N);
2176}
2177
2178/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2179/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2180/// new Base and an SDValue representing the new offset.
2181std::tuple<unsigned, SDValue, SDValue>
2182AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2183 unsigned Opc_ri,
2184 const SDValue &OldBase,
2185 const SDValue &OldOffset,
2186 unsigned Scale) {
2187 SDValue NewBase = OldBase;
2188 SDValue NewOffset = OldOffset;
2189 // Detect a possible Reg+Imm addressing mode.
2190 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2191 Root: N, N: OldBase, Base&: NewBase, OffImm&: NewOffset);
2192
2193 // Detect a possible reg+reg addressing mode, but only if we haven't already
2194 // detected a Reg+Imm one.
2195 const bool IsRegReg =
2196 !IsRegImm && SelectSVERegRegAddrMode(N: OldBase, Scale, Base&: NewBase, Offset&: NewOffset);
2197
2198 // Select the instruction.
2199 return std::make_tuple(args&: IsRegReg ? Opc_rr : Opc_ri, args&: NewBase, args&: NewOffset);
2200}
2201
2202enum class SelectTypeKind {
2203 Int1 = 0,
2204 Int = 1,
2205 FP = 2,
2206 AnyType = 3,
2207};
2208
2209/// This function selects an opcode from a list of opcodes, which is
2210/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2211/// element types, in this order.
2212template <SelectTypeKind Kind>
2213static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2214 // Only match scalable vector VTs
2215 if (!VT.isScalableVector())
2216 return 0;
2217
2218 EVT EltVT = VT.getVectorElementType();
2219 unsigned Key = VT.getVectorMinNumElements();
2220 switch (Kind) {
2221 case SelectTypeKind::AnyType:
2222 break;
2223 case SelectTypeKind::Int:
2224 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2225 EltVT != MVT::i64)
2226 return 0;
2227 break;
2228 case SelectTypeKind::Int1:
2229 if (EltVT != MVT::i1)
2230 return 0;
2231 break;
2232 case SelectTypeKind::FP:
2233 if (EltVT == MVT::bf16)
2234 Key = 16;
2235 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2236 EltVT != MVT::f64)
2237 return 0;
2238 break;
2239 }
2240
2241 unsigned Offset;
2242 switch (Key) {
2243 case 16: // 8-bit or bf16
2244 Offset = 0;
2245 break;
2246 case 8: // 16-bit
2247 Offset = 1;
2248 break;
2249 case 4: // 32-bit
2250 Offset = 2;
2251 break;
2252 case 2: // 64-bit
2253 Offset = 3;
2254 break;
2255 default:
2256 return 0;
2257 }
2258
2259 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2260}
2261
2262// This function is almost identical to SelectWhilePair, but has an
2263// extra check on the range of the immediate operand.
2264// TODO: Merge these two functions together at some point?
2265void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2266 // Immediate can be either 0 or 1.
2267 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
2268 if (Imm->getZExtValue() > 1)
2269 return;
2270
2271 SDLoc DL(N);
2272 EVT VT = N->getValueType(ResNo: 0);
2273 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2274 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2275 SDValue SuperReg = SDValue(WhilePair, 0);
2276
2277 for (unsigned I = 0; I < 2; ++I)
2278 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2279 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2280
2281 CurDAG->RemoveDeadNode(N);
2282}
2283
2284void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2285 SDLoc DL(N);
2286 EVT VT = N->getValueType(ResNo: 0);
2287
2288 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2289
2290 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2291 SDValue SuperReg = SDValue(WhilePair, 0);
2292
2293 for (unsigned I = 0; I < 2; ++I)
2294 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2295 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2296
2297 CurDAG->RemoveDeadNode(N);
2298}
2299
2300void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2301 unsigned Opcode) {
2302 EVT VT = N->getValueType(ResNo: 0);
2303 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2304 SDValue Ops = createZTuple(Regs);
2305 SDLoc DL(N);
2306 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Op1: Ops);
2307 SDValue SuperReg = SDValue(Intrinsic, 0);
2308 for (unsigned i = 0; i < NumVecs; ++i)
2309 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2310 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2311
2312 CurDAG->RemoveDeadNode(N);
2313}
2314
2315void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2316 unsigned Opcode) {
2317 SDLoc DL(N);
2318 EVT VT = N->getValueType(ResNo: 0);
2319 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2320 Ops.push_back(/*Chain*/ Elt: N->getOperand(Num: 0));
2321
2322 SDNode *Instruction =
2323 CurDAG->getMachineNode(Opcode, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2324 SDValue SuperReg = SDValue(Instruction, 0);
2325
2326 for (unsigned i = 0; i < NumVecs; ++i)
2327 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2328 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2329
2330 // Copy chain
2331 unsigned ChainIdx = NumVecs;
2332 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Instruction, 1));
2333 CurDAG->RemoveDeadNode(N);
2334}
2335
2336void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2337 unsigned NumVecs,
2338 bool IsZmMulti,
2339 unsigned Opcode,
2340 bool HasPred) {
2341 assert(Opcode != 0 && "Unexpected opcode");
2342
2343 SDLoc DL(N);
2344 EVT VT = N->getValueType(ResNo: 0);
2345 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2346 SmallVector<SDValue, 4> Ops;
2347
2348 auto GetMultiVecOperand = [&]() {
2349 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2350 OpsIter += NumVecs;
2351 return createZMulTuple(Regs);
2352 };
2353
2354 if (HasPred)
2355 Ops.push_back(Elt: *OpsIter++);
2356
2357 Ops.push_back(Elt: GetMultiVecOperand());
2358 if (IsZmMulti)
2359 Ops.push_back(Elt: GetMultiVecOperand());
2360 else
2361 Ops.push_back(Elt: *OpsIter++);
2362
2363 // Append any remaining operands.
2364 Ops.append(in_start: OpsIter, in_end: N->op_end());
2365 SDNode *Intrinsic;
2366 Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Ops);
2367 SDValue SuperReg = SDValue(Intrinsic, 0);
2368 for (unsigned i = 0; i < NumVecs; ++i)
2369 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2370 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2371
2372 CurDAG->RemoveDeadNode(N);
2373}
2374
2375void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2376 unsigned Scale, unsigned Opc_ri,
2377 unsigned Opc_rr, bool IsIntr) {
2378 assert(Scale < 5 && "Invalid scaling value.");
2379 SDLoc DL(N);
2380 EVT VT = N->getValueType(ResNo: 0);
2381 SDValue Chain = N->getOperand(Num: 0);
2382
2383 // Optimize addressing mode.
2384 SDValue Base, Offset;
2385 unsigned Opc;
2386 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2387 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: IsIntr ? 3 : 2),
2388 OldOffset: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), Scale);
2389
2390 SDValue Ops[] = {N->getOperand(Num: IsIntr ? 2 : 1), // Predicate
2391 Base, // Memory operand
2392 Offset, Chain};
2393
2394 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2395
2396 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2397 SDValue SuperReg = SDValue(Load, 0);
2398 for (unsigned i = 0; i < NumVecs; ++i)
2399 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2400 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2401
2402 // Copy chain
2403 unsigned ChainIdx = NumVecs;
2404 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2405 CurDAG->RemoveDeadNode(N);
2406}
2407
2408void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2409 unsigned NumVecs,
2410 unsigned Scale,
2411 unsigned Opc_ri,
2412 unsigned Opc_rr) {
2413 assert(Scale < 4 && "Invalid scaling value.");
2414 SDLoc DL(N);
2415 EVT VT = N->getValueType(ResNo: 0);
2416 SDValue Chain = N->getOperand(Num: 0);
2417
2418 SDValue PNg = N->getOperand(Num: 2);
2419 SDValue Base = N->getOperand(Num: 3);
2420 SDValue Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
2421 unsigned Opc;
2422 std::tie(args&: Opc, args&: Base, args&: Offset) =
2423 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, OldBase: Base, OldOffset: Offset, Scale);
2424
2425 SDValue Ops[] = {PNg, // Predicate-as-counter
2426 Base, // Memory operand
2427 Offset, Chain};
2428
2429 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2430
2431 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2432 SDValue SuperReg = SDValue(Load, 0);
2433 for (unsigned i = 0; i < NumVecs; ++i)
2434 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2435 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2436
2437 // Copy chain
2438 unsigned ChainIdx = NumVecs;
2439 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2440 CurDAG->RemoveDeadNode(N);
2441}
2442
2443void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2444 unsigned Opcode) {
2445 if (N->getValueType(ResNo: 0) != MVT::nxv4f32)
2446 return;
2447 SelectUnaryMultiIntrinsic(N, NumOutVecs: NumVecs, IsTupleInput: true, Opc: Opcode);
2448}
2449
2450void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2451 unsigned NumOutVecs,
2452 unsigned Opc,
2453 uint32_t MaxImm) {
2454 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 4)))
2455 if (Imm->getZExtValue() > MaxImm)
2456 return;
2457
2458 SDValue ZtValue;
2459 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2460 return;
2461
2462 SDValue Chain = Node->getOperand(Num: 0);
2463 SDValue Ops[] = {ZtValue, Node->getOperand(Num: 3), Node->getOperand(Num: 4), Chain};
2464 SDLoc DL(Node);
2465 EVT VT = Node->getValueType(ResNo: 0);
2466
2467 SDNode *Instruction =
2468 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2469 SDValue SuperReg = SDValue(Instruction, 0);
2470
2471 for (unsigned I = 0; I < NumOutVecs; ++I)
2472 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2473 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2474
2475 // Copy chain
2476 unsigned ChainIdx = NumOutVecs;
2477 ReplaceUses(F: SDValue(Node, ChainIdx), T: SDValue(Instruction, 1));
2478 CurDAG->RemoveDeadNode(N: Node);
2479}
2480
2481void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2482 unsigned NumIndexVecs) {
2483 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2484 "unexpected number of index vectors");
2485
2486 constexpr unsigned FirstIndexOp = 3;
2487 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2488 auto *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
2489 if (!Imm || Imm->getZExtValue() > 1)
2490 return;
2491
2492 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2493 // ACLE form provides three index vectors, so the lane selects which adjacent
2494 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2495 unsigned Lane = Imm->getZExtValue();
2496 unsigned IndexOp = FirstIndexOp;
2497 if (NumIndexVecs == 3)
2498 IndexOp += Lane;
2499
2500 SDValue TableTuple = createZTuple(Regs: {Node->getOperand(Num: 1), Node->getOperand(Num: 2)});
2501 SDValue IndexTuple =
2502 createZTuple(Regs: {Node->getOperand(Num: IndexOp), Node->getOperand(Num: IndexOp + 1)});
2503 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(Num: ImmOp)};
2504
2505 SDLoc DL(Node);
2506 EVT VT = Node->getValueType(ResNo: 0);
2507 SDNode *Instruction =
2508 CurDAG->getMachineNode(Opcode: AArch64::LUTI6_4Z2Z2ZI, dl: DL, VT: MVT::Untyped, Ops);
2509 SDValue SuperReg = SDValue(Instruction, 0);
2510
2511 for (unsigned I = 0; I < 4; ++I)
2512 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2513 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2514
2515 CurDAG->RemoveDeadNode(N: Node);
2516}
2517
2518void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2519 unsigned NumOutVecs,
2520 unsigned Opc,
2521 unsigned NumInVecs) {
2522 assert((NumInVecs == 2 || NumInVecs == 3) &&
2523 "unexpected number of input vectors");
2524
2525 SDValue ZtValue;
2526 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2527 return;
2528
2529 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 3, M: NumInVecs));
2530 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2531 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(Num: 0)};
2532
2533 SDLoc DL(Node);
2534 EVT VT = Node->getValueType(ResNo: 0);
2535
2536 SDNode *Instruction =
2537 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2538 SDValue SuperReg = SDValue(Instruction, 0);
2539
2540 for (unsigned I = 0; I < NumOutVecs; ++I)
2541 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2542 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2543
2544 ReplaceUses(F: SDValue(Node, NumOutVecs), T: SDValue(Instruction, 1));
2545 CurDAG->RemoveDeadNode(N: Node);
2546}
2547
2548void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2549 unsigned Op) {
2550 SDLoc DL(N);
2551 EVT VT = N->getValueType(ResNo: 0);
2552
2553 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2554 SDValue Zd = createZMulTuple(Regs);
2555 SDValue Zn = N->getOperand(Num: 1 + NumVecs);
2556 SDValue Zm = N->getOperand(Num: 2 + NumVecs);
2557
2558 SDValue Ops[] = {Zd, Zn, Zm};
2559
2560 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT: MVT::Untyped, Ops);
2561 SDValue SuperReg = SDValue(Intrinsic, 0);
2562 for (unsigned i = 0; i < NumVecs; ++i)
2563 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2564 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2565
2566 CurDAG->RemoveDeadNode(N);
2567}
2568
2569bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2570 switch (BaseReg) {
2571 default:
2572 return false;
2573 case AArch64::ZA:
2574 case AArch64::ZAB0:
2575 if (TileNum == 0)
2576 break;
2577 return false;
2578 case AArch64::ZAH0:
2579 if (TileNum <= 1)
2580 break;
2581 return false;
2582 case AArch64::ZAS0:
2583 if (TileNum <= 3)
2584 break;
2585 return false;
2586 case AArch64::ZAD0:
2587 if (TileNum <= 7)
2588 break;
2589 return false;
2590 }
2591
2592 BaseReg += TileNum;
2593 return true;
2594}
2595
2596template <unsigned MaxIdx, unsigned Scale>
2597void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2598 unsigned BaseReg, unsigned Op) {
2599 unsigned TileNum = 0;
2600 if (BaseReg != AArch64::ZA)
2601 TileNum = N->getConstantOperandVal(Num: 2);
2602
2603 if (!SelectSMETile(BaseReg, TileNum))
2604 return;
2605
2606 SDValue SliceBase, Base, Offset;
2607 if (BaseReg == AArch64::ZA)
2608 SliceBase = N->getOperand(Num: 2);
2609 else
2610 SliceBase = N->getOperand(Num: 3);
2611
2612 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2613 return;
2614
2615 SDLoc DL(N);
2616 SDValue SubReg = CurDAG->getRegister(Reg: BaseReg, VT: MVT::Other);
2617 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(Num: 0)};
2618 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2619
2620 EVT VT = N->getValueType(ResNo: 0);
2621 for (unsigned I = 0; I < NumVecs; ++I)
2622 ReplaceUses(F: SDValue(N, I),
2623 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2624 Operand: SDValue(Mov, 0)));
2625 // Copy chain
2626 unsigned ChainIdx = NumVecs;
2627 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2628 CurDAG->RemoveDeadNode(N);
2629}
2630
2631void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2632 unsigned Op, unsigned MaxIdx,
2633 unsigned Scale, unsigned BaseReg) {
2634 // Slice can be in different positions
2635 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2636 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2637 SDValue SliceBase = N->getOperand(Num: 2);
2638 if (BaseReg != AArch64::ZA)
2639 SliceBase = N->getOperand(Num: 3);
2640
2641 SDValue Base, Offset;
2642 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2643 return;
2644 // The correct Za tile number is computed in Machine Instruction
2645 // See EmitZAInstr
2646 // DAG cannot select Za tile as an output register with ZReg
2647 SDLoc DL(N);
2648 SmallVector<SDValue, 6> Ops;
2649 if (BaseReg != AArch64::ZA )
2650 Ops.push_back(Elt: N->getOperand(Num: 2));
2651 Ops.push_back(Elt: Base);
2652 Ops.push_back(Elt: Offset);
2653 Ops.push_back(Elt: N->getOperand(Num: 0)); //Chain
2654 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2655
2656 EVT VT = N->getValueType(ResNo: 0);
2657 for (unsigned I = 0; I < NumVecs; ++I)
2658 ReplaceUses(F: SDValue(N, I),
2659 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2660 Operand: SDValue(Mov, 0)));
2661
2662 // Copy chain
2663 unsigned ChainIdx = NumVecs;
2664 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2665 CurDAG->RemoveDeadNode(N);
2666}
2667
2668void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2669 unsigned NumOutVecs,
2670 bool IsTupleInput,
2671 unsigned Opc) {
2672 SDLoc DL(N);
2673 EVT VT = N->getValueType(ResNo: 0);
2674 unsigned NumInVecs = N->getNumOperands() - 1;
2675
2676 SmallVector<SDValue, 6> Ops;
2677 if (IsTupleInput) {
2678 assert((NumInVecs == 2 || NumInVecs == 4) &&
2679 "Don't know how to handle multi-register input!");
2680 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumInVecs));
2681 Ops.push_back(Elt: createZMulTuple(Regs));
2682 } else {
2683 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2684 for (unsigned I = 0; I < NumInVecs; I++)
2685 Ops.push_back(Elt: N->getOperand(Num: 1 + I));
2686 }
2687
2688 SDNode *Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2689 SDValue SuperReg = SDValue(Res, 0);
2690
2691 for (unsigned I = 0; I < NumOutVecs; I++)
2692 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2693 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2694 CurDAG->RemoveDeadNode(N);
2695}
2696
2697void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2698 unsigned Opc) {
2699 SDLoc dl(N);
2700 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2701
2702 // Form a REG_SEQUENCE to force register allocation.
2703 bool Is128Bit = VT.getSizeInBits() == 128;
2704 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2705 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2706
2707 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), N->getOperand(Num: 0)};
2708 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2709
2710 // Transfer memoperands.
2711 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2712 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2713
2714 ReplaceNode(F: N, T: St);
2715}
2716
2717void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2718 unsigned Scale, unsigned Opc_rr,
2719 unsigned Opc_ri) {
2720 SDLoc dl(N);
2721
2722 // Form a REG_SEQUENCE to force register allocation.
2723 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2724 SDValue RegSeq = createZTuple(Regs);
2725
2726 // Optimize addressing mode.
2727 unsigned Opc;
2728 SDValue Offset, Base;
2729 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2730 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: NumVecs + 3),
2731 OldOffset: CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64), Scale);
2732
2733 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), // predicate
2734 Base, // address
2735 Offset, // offset
2736 N->getOperand(Num: 0)}; // chain
2737 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2738
2739 // Transfer memoperands.
2740 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2741 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2742
2743 ReplaceNode(F: N, T: St);
2744}
2745
2746void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2747 unsigned Opc) {
2748 SDLoc dl(N);
2749 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2750 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2751 MVT::Other}; // Type for the Chain
2752
2753 // Form a REG_SEQUENCE to force register allocation.
2754 bool Is128Bit = VT.getSizeInBits() == 128;
2755 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2756 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2757
2758 SDValue Ops[] = {RegSeq,
2759 N->getOperand(Num: NumVecs + 1), // base register
2760 N->getOperand(Num: NumVecs + 2), // Incremental
2761 N->getOperand(Num: 0)}; // Chain
2762 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2763
2764 // Transfer memoperands.
2765 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2766 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2767
2768 ReplaceNode(F: N, T: St);
2769}
2770
2771namespace {
2772/// WidenVector - Given a value in the V64 register class, produce the
2773/// equivalent value in the V128 register class.
2774class WidenVector {
2775 SelectionDAG &DAG;
2776
2777public:
2778 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2779
2780 SDValue operator()(SDValue V64Reg) {
2781 EVT VT = V64Reg.getValueType();
2782 unsigned NarrowSize = VT.getVectorNumElements();
2783 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2784 MVT WideTy = MVT::getVectorVT(VT: EltTy, NumElements: 2 * NarrowSize);
2785 SDLoc DL(V64Reg);
2786
2787 SDValue Undef =
2788 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: WideTy), 0);
2789 return DAG.getTargetInsertSubreg(SRIdx: AArch64::dsub, DL, VT: WideTy, Operand: Undef, Subreg: V64Reg);
2790 }
2791};
2792} // namespace
2793
2794/// NarrowVector - Given a value in the V128 register class, produce the
2795/// equivalent value in the V64 register class.
2796static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
2797 EVT VT = V128Reg.getValueType();
2798 unsigned WideSize = VT.getVectorNumElements();
2799 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2800 MVT NarrowTy = MVT::getVectorVT(VT: EltTy, NumElements: WideSize / 2);
2801
2802 return DAG.getTargetExtractSubreg(SRIdx: AArch64::dsub, DL: SDLoc(V128Reg), VT: NarrowTy,
2803 Operand: V128Reg);
2804}
2805
2806void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2807 unsigned Opc) {
2808 SDLoc dl(N);
2809 EVT VT = N->getValueType(ResNo: 0);
2810 bool Narrow = VT.getSizeInBits() == 64;
2811
2812 // Form a REG_SEQUENCE to force register allocation.
2813 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2814
2815 if (Narrow)
2816 transform(Range&: Regs, d_first: Regs.begin(),
2817 F: WidenVector(*CurDAG));
2818
2819 SDValue RegSeq = createQTuple(Regs);
2820
2821 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2822
2823 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2824
2825 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2826 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2827 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2828 SDValue SuperReg = SDValue(Ld, 0);
2829
2830 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2831 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2832 AArch64::qsub2, AArch64::qsub3 };
2833 for (unsigned i = 0; i < NumVecs; ++i) {
2834 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT, Operand: SuperReg);
2835 if (Narrow)
2836 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2837 ReplaceUses(F: SDValue(N, i), T: NV);
2838 }
2839
2840 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2841 CurDAG->RemoveDeadNode(N);
2842}
2843
2844void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2845 unsigned Opc) {
2846 SDLoc dl(N);
2847 EVT VT = N->getValueType(ResNo: 0);
2848 bool Narrow = VT.getSizeInBits() == 64;
2849
2850 // Form a REG_SEQUENCE to force register allocation.
2851 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2852
2853 if (Narrow)
2854 transform(Range&: Regs, d_first: Regs.begin(),
2855 F: WidenVector(*CurDAG));
2856
2857 SDValue RegSeq = createQTuple(Regs);
2858
2859 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2860 RegSeq->getValueType(ResNo: 0), MVT::Other};
2861
2862 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2863
2864 SDValue Ops[] = {RegSeq,
2865 CurDAG->getTargetConstant(Val: LaneNo, DL: dl,
2866 VT: MVT::i64), // Lane Number
2867 N->getOperand(Num: NumVecs + 2), // Base register
2868 N->getOperand(Num: NumVecs + 3), // Incremental
2869 N->getOperand(Num: 0)};
2870 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2871
2872 // Update uses of the write back register
2873 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2874
2875 // Update uses of the vector list
2876 SDValue SuperReg = SDValue(Ld, 1);
2877 if (NumVecs == 1) {
2878 ReplaceUses(F: SDValue(N, 0),
2879 T: Narrow ? NarrowVector(V128Reg: SuperReg, DAG&: *CurDAG) : SuperReg);
2880 } else {
2881 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2882 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2883 AArch64::qsub2, AArch64::qsub3 };
2884 for (unsigned i = 0; i < NumVecs; ++i) {
2885 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT,
2886 Operand: SuperReg);
2887 if (Narrow)
2888 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2889 ReplaceUses(F: SDValue(N, i), T: NV);
2890 }
2891 }
2892
2893 // Update the Chain
2894 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2895 CurDAG->RemoveDeadNode(N);
2896}
2897
2898void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
2899 unsigned Opc) {
2900 SDLoc dl(N);
2901 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2902 bool Narrow = VT.getSizeInBits() == 64;
2903
2904 // Form a REG_SEQUENCE to force register allocation.
2905 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2906
2907 if (Narrow)
2908 transform(Range&: Regs, d_first: Regs.begin(),
2909 F: WidenVector(*CurDAG));
2910
2911 SDValue RegSeq = createQTuple(Regs);
2912
2913 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2914
2915 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2916 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2917 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
2918
2919 // Transfer memoperands.
2920 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2921 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2922
2923 ReplaceNode(F: N, T: St);
2924}
2925
2926void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
2927 unsigned Opc) {
2928 SDLoc dl(N);
2929 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2930 bool Narrow = VT.getSizeInBits() == 64;
2931
2932 // Form a REG_SEQUENCE to force register allocation.
2933 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2934
2935 if (Narrow)
2936 transform(Range&: Regs, d_first: Regs.begin(),
2937 F: WidenVector(*CurDAG));
2938
2939 SDValue RegSeq = createQTuple(Regs);
2940
2941 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2942 MVT::Other};
2943
2944 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2945
2946 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2947 N->getOperand(Num: NumVecs + 2), // Base Register
2948 N->getOperand(Num: NumVecs + 3), // Incremental
2949 N->getOperand(Num: 0)};
2950 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2951
2952 // Transfer memoperands.
2953 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2954 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2955
2956 ReplaceNode(F: N, T: St);
2957}
2958
2959static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
2960 unsigned &Opc, SDValue &Opd0,
2961 unsigned &LSB, unsigned &MSB,
2962 unsigned NumberOfIgnoredLowBits,
2963 bool BiggerPattern) {
2964 assert(N->getOpcode() == ISD::AND &&
2965 "N must be a AND operation to call this function");
2966
2967 EVT VT = N->getValueType(ResNo: 0);
2968
2969 // Here we can test the type of VT and return false when the type does not
2970 // match, but since it is done prior to that call in the current context
2971 // we turned that into an assert to avoid redundant code.
2972 assert((VT == MVT::i32 || VT == MVT::i64) &&
2973 "Type checking must have been done before calling this function");
2974
2975 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2976 // changed the AND node to a 32-bit mask operation. We'll have to
2977 // undo that as part of the transform here if we want to catch all
2978 // the opportunities.
2979 // Currently the NumberOfIgnoredLowBits argument helps to recover
2980 // from these situations when matching bigger pattern (bitfield insert).
2981
2982 // For unsigned extracts, check for a shift right and mask
2983 uint64_t AndImm = 0;
2984 if (!isOpcWithIntImmediate(N, Opc: ISD::AND, Imm&: AndImm))
2985 return false;
2986
2987 const SDNode *Op0 = N->getOperand(Num: 0).getNode();
2988
2989 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2990 // simplified. Try to undo that
2991 AndImm |= maskTrailingOnes<uint64_t>(N: NumberOfIgnoredLowBits);
2992
2993 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2994 if (AndImm & (AndImm + 1))
2995 return false;
2996
2997 bool ClampMSB = false;
2998 uint64_t SrlImm = 0;
2999 // Handle the SRL + ANY_EXTEND case.
3000 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
3001 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3002 // Extend the incoming operand of the SRL to 64-bit.
3003 Opd0 = Widen(CurDAG, N: Op0->getOperand(Num: 0).getOperand(i: 0));
3004 // Make sure to clamp the MSB so that we preserve the semantics of the
3005 // original operations.
3006 ClampMSB = true;
3007 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
3008 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL,
3009 Imm&: SrlImm)) {
3010 // If the shift result was truncated, we can still combine them.
3011 Opd0 = Op0->getOperand(Num: 0).getOperand(i: 0);
3012
3013 // Use the type of SRL node.
3014 VT = Opd0->getValueType(ResNo: 0);
3015 } else if (isOpcWithIntImmediate(N: Op0, Opc: ISD::SRL, Imm&: SrlImm)) {
3016 Opd0 = Op0->getOperand(Num: 0);
3017 ClampMSB = (VT == MVT::i32);
3018 } else if (BiggerPattern) {
3019 // Let's pretend a 0 shift right has been performed.
3020 // The resulting code will be at least as good as the original one
3021 // plus it may expose more opportunities for bitfield insert pattern.
3022 // FIXME: Currently we limit this to the bigger pattern, because
3023 // some optimizations expect AND and not UBFM.
3024 Opd0 = N->getOperand(Num: 0);
3025 } else
3026 return false;
3027
3028 // Bail out on large immediates. This happens when no proper
3029 // combining/constant folding was performed.
3030 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
3031 LLVM_DEBUG(
3032 (dbgs() << N
3033 << ": Found large shift immediate, this should not happen\n"));
3034 return false;
3035 }
3036
3037 LSB = SrlImm;
3038 MSB = SrlImm +
3039 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(Value: AndImm)
3040 : llvm::countr_one<uint64_t>(Value: AndImm)) -
3041 1;
3042 if (ClampMSB)
3043 // Since we're moving the extend before the right shift operation, we need
3044 // to clamp the MSB to make sure we don't shift in undefined bits instead of
3045 // the zeros which would get shifted in with the original right shift
3046 // operation.
3047 MSB = MSB > 31 ? 31 : MSB;
3048
3049 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3050 return true;
3051}
3052
3053static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
3054 SDValue &Opd0, unsigned &Immr,
3055 unsigned &Imms) {
3056 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
3057
3058 EVT VT = N->getValueType(ResNo: 0);
3059 unsigned BitWidth = VT.getSizeInBits();
3060 assert((VT == MVT::i32 || VT == MVT::i64) &&
3061 "Type checking must have been done before calling this function");
3062
3063 SDValue Op = N->getOperand(Num: 0);
3064 if (Op->getOpcode() == ISD::TRUNCATE) {
3065 Op = Op->getOperand(Num: 0);
3066 VT = Op->getValueType(ResNo: 0);
3067 BitWidth = VT.getSizeInBits();
3068 }
3069
3070 uint64_t ShiftImm;
3071 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRL, Imm&: ShiftImm) &&
3072 !isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
3073 return false;
3074
3075 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
3076 if (ShiftImm + Width > BitWidth)
3077 return false;
3078
3079 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
3080 Opd0 = Op.getOperand(i: 0);
3081 Immr = ShiftImm;
3082 Imms = ShiftImm + Width - 1;
3083 return true;
3084}
3085
3086static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
3087 SDValue &Opd0, unsigned &LSB,
3088 unsigned &MSB) {
3089 // We are looking for the following pattern which basically extracts several
3090 // continuous bits from the source value and places it from the LSB of the
3091 // destination value, all other bits of the destination value or set to zero:
3092 //
3093 // Value2 = AND Value, MaskImm
3094 // SRL Value2, ShiftImm
3095 //
3096 // with MaskImm >> ShiftImm to search for the bit width.
3097 //
3098 // This gets selected into a single UBFM:
3099 //
3100 // UBFM Value, ShiftImm, Log2_64(MaskImm)
3101 //
3102
3103 if (N->getOpcode() != ISD::SRL)
3104 return false;
3105
3106 uint64_t AndMask = 0;
3107 if (!isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm&: AndMask))
3108 return false;
3109
3110 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3111
3112 uint64_t SrlImm = 0;
3113 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3114 return false;
3115
3116 // Check whether we really have several bits extract here.
3117 if (!isMask_64(Value: AndMask >> SrlImm))
3118 return false;
3119
3120 Opc = N->getValueType(ResNo: 0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3121 LSB = SrlImm;
3122 MSB = llvm::Log2_64(Value: AndMask);
3123 return true;
3124}
3125
3126static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
3127 unsigned &Immr, unsigned &Imms,
3128 bool BiggerPattern) {
3129 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
3130 "N must be a SHR/SRA operation to call this function");
3131
3132 EVT VT = N->getValueType(ResNo: 0);
3133
3134 // Here we can test the type of VT and return false when the type does not
3135 // match, but since it is done prior to that call in the current context
3136 // we turned that into an assert to avoid redundant code.
3137 assert((VT == MVT::i32 || VT == MVT::i64) &&
3138 "Type checking must have been done before calling this function");
3139
3140 // Check for AND + SRL doing several bits extract.
3141 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB&: Immr, MSB&: Imms))
3142 return true;
3143
3144 // We're looking for a shift of a shift.
3145 uint64_t ShlImm = 0;
3146 uint64_t TruncBits = 0;
3147 if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3148 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3149 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
3150 N->getOperand(Num: 0).getNode()->getOpcode() == ISD::TRUNCATE) {
3151 // We are looking for a shift of truncate. Truncate from i64 to i32 could
3152 // be considered as setting high 32 bits as zero. Our strategy here is to
3153 // always generate 64bit UBFM. This consistency will help the CSE pass
3154 // later find more redundancy.
3155 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3156 TruncBits = Opd0->getValueType(ResNo: 0).getSizeInBits() - VT.getSizeInBits();
3157 VT = Opd0.getValueType();
3158 assert(VT == MVT::i64 && "the promoted type should be i64");
3159 } else if (BiggerPattern) {
3160 // Let's pretend a 0 shift left has been performed.
3161 // FIXME: Currently we limit this to the bigger pattern case,
3162 // because some optimizations expect AND and not UBFM
3163 Opd0 = N->getOperand(Num: 0);
3164 } else
3165 return false;
3166
3167 // Missing combines/constant folding may have left us with strange
3168 // constants.
3169 if (ShlImm >= VT.getSizeInBits()) {
3170 LLVM_DEBUG(
3171 (dbgs() << N
3172 << ": Found large shift immediate, this should not happen\n"));
3173 return false;
3174 }
3175
3176 uint64_t SrlImm = 0;
3177 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3178 return false;
3179
3180 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
3181 "bad amount in shift node!");
3182 int immr = SrlImm - ShlImm;
3183 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
3184 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
3185 // SRA requires a signed extraction
3186 if (VT == MVT::i32)
3187 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
3188 else
3189 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
3190 return true;
3191}
3192
3193static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3194 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3195 unsigned NumberOfIgnoredLowBits = 0,
3196 bool BiggerPattern = false) {
3197 if (N->getValueType(ResNo: 0) != MVT::i32 && N->getValueType(ResNo: 0) != MVT::i64)
3198 return false;
3199
3200 switch (N->getOpcode()) {
3201 default:
3202 if (!N->isMachineOpcode())
3203 return false;
3204 break;
3205 case ISD::AND:
3206 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB&: Immr, MSB&: Imms,
3207 NumberOfIgnoredLowBits, BiggerPattern);
3208 case ISD::SRL:
3209 case ISD::SRA:
3210 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3211
3212 case ISD::SIGN_EXTEND_INREG:
3213 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3214 }
3215
3216 unsigned NOpc = N->getMachineOpcode();
3217 switch (NOpc) {
3218 default:
3219 return false;
3220 case AArch64::SBFMWri:
3221 case AArch64::UBFMWri:
3222 case AArch64::SBFMXri:
3223 case AArch64::UBFMXri:
3224 Opc = NOpc;
3225 Opd0 = N->getOperand(Num: 0);
3226 Immr = N->getConstantOperandVal(Num: 1);
3227 Imms = N->getConstantOperandVal(Num: 2);
3228 return true;
3229 }
3230 // Unreachable
3231 return false;
3232}
3233
3234bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3235 unsigned Opc, Immr, Imms;
3236 SDValue Opd0;
3237 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3238 return false;
3239
3240 EVT VT = N->getValueType(ResNo: 0);
3241 SDLoc dl(N);
3242
3243 // If the bit extract operation is 64bit but the original type is 32bit, we
3244 // need to add one EXTRACT_SUBREG.
3245 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3246 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT: MVT::i64),
3247 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT: MVT::i64)};
3248
3249 SDNode *BFM = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i64, Ops: Ops64);
3250 SDValue Inner = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl,
3251 VT: MVT::i32, Operand: SDValue(BFM, 0));
3252 ReplaceNode(F: N, T: Inner.getNode());
3253 return true;
3254 }
3255
3256 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3257 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3258 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3259 return true;
3260}
3261
3262/// Does DstMask form a complementary pair with the mask provided by
3263/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3264/// this asks whether DstMask zeroes precisely those bits that will be set by
3265/// the other half.
3266static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3267 unsigned NumberOfIgnoredHighBits, EVT VT) {
3268 assert((VT == MVT::i32 || VT == MVT::i64) &&
3269 "i32 or i64 mask type expected!");
3270 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3271
3272 // Enable implicitTrunc as we're intentionally ignoring high bits.
3273 APInt SignificantDstMask =
3274 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3275 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(width: BitWidth);
3276
3277 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3278 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3279}
3280
3281// Look for bits that will be useful for later uses.
3282// A bit is consider useless as soon as it is dropped and never used
3283// before it as been dropped.
3284// E.g., looking for useful bit of x
3285// 1. y = x & 0x7
3286// 2. z = y >> 2
3287// After #1, x useful bits are 0x7, then the useful bits of x, live through
3288// y.
3289// After #2, the useful bits of x are 0x4.
3290// However, if x is used on an unpredictable instruction, then all its bits
3291// are useful.
3292// E.g.
3293// 1. y = x & 0x7
3294// 2. z = y >> 2
3295// 3. str x, [@x]
3296static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3297
3298static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
3299 unsigned Depth) {
3300 uint64_t Imm =
3301 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3302 Imm = AArch64_AM::decodeLogicalImmediate(val: Imm, regSize: UsefulBits.getBitWidth());
3303 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3304 getUsefulBits(Op, UsefulBits, Depth: Depth + 1);
3305}
3306
3307static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
3308 uint64_t Imm, uint64_t MSB,
3309 unsigned Depth) {
3310 // inherit the bitwidth value
3311 APInt OpUsefulBits(UsefulBits);
3312 OpUsefulBits = 1;
3313
3314 if (MSB >= Imm) {
3315 OpUsefulBits <<= MSB - Imm + 1;
3316 --OpUsefulBits;
3317 // The interesting part will be in the lower part of the result
3318 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3319 // The interesting part was starting at Imm in the argument
3320 OpUsefulBits <<= Imm;
3321 } else {
3322 OpUsefulBits <<= MSB + 1;
3323 --OpUsefulBits;
3324 // The interesting part will be shifted in the result
3325 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3326 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3327 // The interesting part was at zero in the argument
3328 OpUsefulBits.lshrInPlace(ShiftAmt: OpUsefulBits.getBitWidth() - Imm);
3329 }
3330
3331 UsefulBits &= OpUsefulBits;
3332}
3333
3334static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3335 unsigned Depth) {
3336 uint64_t Imm =
3337 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3338 uint64_t MSB =
3339 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3340
3341 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3342}
3343
3344static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
3345 unsigned Depth) {
3346 uint64_t ShiftTypeAndValue =
3347 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3348 APInt Mask(UsefulBits);
3349 Mask.clearAllBits();
3350 Mask.flipAllBits();
3351
3352 if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSL) {
3353 // Shift Left
3354 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3355 Mask <<= ShiftAmt;
3356 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3357 Mask.lshrInPlace(ShiftAmt);
3358 } else if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSR) {
3359 // Shift Right
3360 // We do not handle AArch64_AM::ASR, because the sign will change the
3361 // number of useful bits
3362 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3363 Mask.lshrInPlace(ShiftAmt);
3364 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3365 Mask <<= ShiftAmt;
3366 } else
3367 return;
3368
3369 UsefulBits &= Mask;
3370}
3371
3372static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3373 unsigned Depth) {
3374 uint64_t Imm =
3375 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3376 uint64_t MSB =
3377 cast<const ConstantSDNode>(Val: Op.getOperand(i: 3).getNode())->getZExtValue();
3378
3379 APInt OpUsefulBits(UsefulBits);
3380 OpUsefulBits = 1;
3381
3382 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3383 ResultUsefulBits.flipAllBits();
3384 APInt Mask(UsefulBits.getBitWidth(), 0);
3385
3386 getUsefulBits(Op, UsefulBits&: ResultUsefulBits, Depth: Depth + 1);
3387
3388 if (MSB >= Imm) {
3389 // The instruction is a BFXIL.
3390 uint64_t Width = MSB - Imm + 1;
3391 uint64_t LSB = Imm;
3392
3393 OpUsefulBits <<= Width;
3394 --OpUsefulBits;
3395
3396 if (Op.getOperand(i: 1) == Orig) {
3397 // Copy the low bits from the result to bits starting from LSB.
3398 Mask = ResultUsefulBits & OpUsefulBits;
3399 Mask <<= LSB;
3400 }
3401
3402 if (Op.getOperand(i: 0) == Orig)
3403 // Bits starting from LSB in the input contribute to the result.
3404 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3405 } else {
3406 // The instruction is a BFI.
3407 uint64_t Width = MSB + 1;
3408 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3409
3410 OpUsefulBits <<= Width;
3411 --OpUsefulBits;
3412 OpUsefulBits <<= LSB;
3413
3414 if (Op.getOperand(i: 1) == Orig) {
3415 // Copy the bits from the result to the zero bits.
3416 Mask = ResultUsefulBits & OpUsefulBits;
3417 Mask.lshrInPlace(ShiftAmt: LSB);
3418 }
3419
3420 if (Op.getOperand(i: 0) == Orig)
3421 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3422 }
3423
3424 UsefulBits &= Mask;
3425}
3426
3427static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3428 SDValue Orig, unsigned Depth) {
3429
3430 // Users of this node should have already been instruction selected
3431 // FIXME: Can we turn that into an assert?
3432 if (!UserNode->isMachineOpcode())
3433 return;
3434
3435 switch (UserNode->getMachineOpcode()) {
3436 default:
3437 return;
3438 case AArch64::ANDSWri:
3439 case AArch64::ANDSXri:
3440 case AArch64::ANDWri:
3441 case AArch64::ANDXri:
3442 // We increment Depth only when we call the getUsefulBits
3443 return getUsefulBitsFromAndWithImmediate(Op: SDValue(UserNode, 0), UsefulBits,
3444 Depth);
3445 case AArch64::UBFMWri:
3446 case AArch64::UBFMXri:
3447 return getUsefulBitsFromUBFM(Op: SDValue(UserNode, 0), UsefulBits, Depth);
3448
3449 case AArch64::ORRWrs:
3450 case AArch64::ORRXrs:
3451 if (UserNode->getOperand(Num: 0) != Orig && UserNode->getOperand(Num: 1) == Orig)
3452 getUsefulBitsFromOrWithShiftedReg(Op: SDValue(UserNode, 0), UsefulBits,
3453 Depth);
3454 return;
3455 case AArch64::BFMWri:
3456 case AArch64::BFMXri:
3457 return getUsefulBitsFromBFM(Op: SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3458
3459 case AArch64::STRBBui:
3460 case AArch64::STURBBi:
3461 if (UserNode->getOperand(Num: 0) != Orig)
3462 return;
3463 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3464 return;
3465
3466 case AArch64::STRHHui:
3467 case AArch64::STURHHi:
3468 if (UserNode->getOperand(Num: 0) != Orig)
3469 return;
3470 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3471 return;
3472 }
3473}
3474
3475static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3476 if (Depth >= SelectionDAG::MaxRecursionDepth)
3477 return;
3478 // Initialize UsefulBits
3479 if (!Depth) {
3480 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3481 // At the beginning, assume every produced bits is useful
3482 UsefulBits = APInt(Bitwidth, 0);
3483 UsefulBits.flipAllBits();
3484 }
3485 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3486
3487 for (SDNode *Node : Op.getNode()->users()) {
3488 // A use cannot produce useful bits
3489 APInt UsefulBitsForUse = APInt(UsefulBits);
3490 getUsefulBitsForUse(UserNode: Node, UsefulBits&: UsefulBitsForUse, Orig: Op, Depth);
3491 UsersUsefulBits |= UsefulBitsForUse;
3492 }
3493 // UsefulBits contains the produced bits that are meaningful for the
3494 // current definition, thus a user cannot make a bit meaningful at
3495 // this point
3496 UsefulBits &= UsersUsefulBits;
3497}
3498
3499/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3500/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3501/// 0, return Op unchanged.
3502static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3503 if (ShlAmount == 0)
3504 return Op;
3505
3506 EVT VT = Op.getValueType();
3507 SDLoc dl(Op);
3508 unsigned BitWidth = VT.getSizeInBits();
3509 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3510
3511 SDNode *ShiftNode;
3512 if (ShlAmount > 0) {
3513 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3514 ShiftNode = CurDAG->getMachineNode(
3515 Opcode: UBFMOpc, dl, VT, Op1: Op,
3516 Op2: CurDAG->getTargetConstant(Val: BitWidth - ShlAmount, DL: dl, VT),
3517 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1 - ShlAmount, DL: dl, VT));
3518 } else {
3519 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3520 assert(ShlAmount < 0 && "expected right shift");
3521 int ShrAmount = -ShlAmount;
3522 ShiftNode = CurDAG->getMachineNode(
3523 Opcode: UBFMOpc, dl, VT, Op1: Op, Op2: CurDAG->getTargetConstant(Val: ShrAmount, DL: dl, VT),
3524 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL: dl, VT));
3525 }
3526
3527 return SDValue(ShiftNode, 0);
3528}
3529
3530// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3531static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3532 bool BiggerPattern,
3533 const uint64_t NonZeroBits,
3534 SDValue &Src, int &DstLSB,
3535 int &Width);
3536
3537// For bit-field-positioning pattern "shl VAL, N)".
3538static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3539 bool BiggerPattern,
3540 const uint64_t NonZeroBits,
3541 SDValue &Src, int &DstLSB,
3542 int &Width);
3543
3544/// Does this tree qualify as an attempt to move a bitfield into position,
3545/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3546static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
3547 bool BiggerPattern, SDValue &Src,
3548 int &DstLSB, int &Width) {
3549 EVT VT = Op.getValueType();
3550 unsigned BitWidth = VT.getSizeInBits();
3551 (void)BitWidth;
3552 assert(BitWidth == 32 || BitWidth == 64);
3553
3554 KnownBits Known = CurDAG->computeKnownBits(Op);
3555
3556 // Non-zero in the sense that they're not provably zero, which is the key
3557 // point if we want to use this value
3558 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3559 if (!isShiftedMask_64(Value: NonZeroBits))
3560 return false;
3561
3562 switch (Op.getOpcode()) {
3563 default:
3564 break;
3565 case ISD::AND:
3566 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3567 NonZeroBits, Src, DstLSB, Width);
3568 case ISD::SHL:
3569 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3570 NonZeroBits, Src, DstLSB, Width);
3571 }
3572
3573 return false;
3574}
3575
3576static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3577 bool BiggerPattern,
3578 const uint64_t NonZeroBits,
3579 SDValue &Src, int &DstLSB,
3580 int &Width) {
3581 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3582
3583 EVT VT = Op.getValueType();
3584 assert((VT == MVT::i32 || VT == MVT::i64) &&
3585 "Caller guarantees VT is one of i32 or i64");
3586 (void)VT;
3587
3588 uint64_t AndImm;
3589 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::AND, Imm&: AndImm))
3590 return false;
3591
3592 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3593 // 1) (AndImm & (1 << POS) == 0)
3594 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3595 //
3596 // 1) and 2) don't agree so something must be wrong (e.g., in
3597 // 'SelectionDAG::computeKnownBits')
3598 assert((~AndImm & NonZeroBits) == 0 &&
3599 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3600
3601 SDValue AndOp0 = Op.getOperand(i: 0);
3602
3603 uint64_t ShlImm;
3604 SDValue ShlOp0;
3605 if (isOpcWithIntImmediate(N: AndOp0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3606 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3607 ShlOp0 = AndOp0.getOperand(i: 0);
3608 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3609 isOpcWithIntImmediate(N: AndOp0.getOperand(i: 0).getNode(), Opc: ISD::SHL,
3610 Imm&: ShlImm)) {
3611 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3612
3613 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3614 SDValue ShlVal = AndOp0.getOperand(i: 0);
3615
3616 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3617 // expect VT to be MVT::i32.
3618 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3619
3620 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3621 ShlOp0 = Widen(CurDAG, N: ShlVal.getOperand(i: 0));
3622 } else
3623 return false;
3624
3625 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3626 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3627 // AndOp0+AND.
3628 if (!BiggerPattern && !AndOp0.hasOneUse())
3629 return false;
3630
3631 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3632 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3633
3634 // Bail out on large Width. This happens when no proper combining / constant
3635 // folding was performed.
3636 if (Width >= (int)VT.getSizeInBits()) {
3637 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3638 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3639 // "val".
3640 // If VT is i32, what Width >= 32 means:
3641 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3642 // demands at least 'Width' bits (after dag-combiner). This together with
3643 // `any_extend` Op (undefined higher bits) indicates missed combination
3644 // when lowering the 'and' IR instruction to an machine IR instruction.
3645 LLVM_DEBUG(
3646 dbgs()
3647 << "Found large Width in bit-field-positioning -- this indicates no "
3648 "proper combining / constant folding was performed\n");
3649 return false;
3650 }
3651
3652 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3653 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3654 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3655 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3656 // which case it is not profitable to insert an extra shift.
3657 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3658 return false;
3659
3660 Src = getLeftShift(CurDAG, Op: ShlOp0, ShlAmount: ShlImm - DstLSB);
3661 return true;
3662}
3663
3664// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3665// UBFIZ.
3666static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op,
3667 SDValue &Src, int &DstLSB,
3668 int &Width) {
3669 // Caller should have verified that N is a left shift with constant shift
3670 // amount; asserts that.
3671 assert(Op.getOpcode() == ISD::SHL &&
3672 "Op.getNode() should be a SHL node to call this function");
3673 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3674 "Op.getNode() should shift ShlImm to call this function");
3675
3676 uint64_t AndImm = 0;
3677 SDValue Op0 = Op.getOperand(i: 0);
3678 if (!isOpcWithIntImmediate(N: Op0.getNode(), Opc: ISD::AND, Imm&: AndImm))
3679 return false;
3680
3681 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3682 if (isMask_64(Value: ShiftedAndImm)) {
3683 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3684 // should end with Mask, and could be prefixed with random bits if those
3685 // bits are shifted out.
3686 //
3687 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3688 // the AND result corresponding to those bits are shifted out, so it's fine
3689 // to not extract them.
3690 Width = llvm::countr_one(Value: ShiftedAndImm);
3691 DstLSB = ShlImm;
3692 Src = Op0.getOperand(i: 0);
3693 return true;
3694 }
3695 return false;
3696}
3697
3698static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3699 bool BiggerPattern,
3700 const uint64_t NonZeroBits,
3701 SDValue &Src, int &DstLSB,
3702 int &Width) {
3703 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3704
3705 EVT VT = Op.getValueType();
3706 assert((VT == MVT::i32 || VT == MVT::i64) &&
3707 "Caller guarantees that type is i32 or i64");
3708 (void)VT;
3709
3710 uint64_t ShlImm;
3711 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SHL, Imm&: ShlImm))
3712 return false;
3713
3714 if (!BiggerPattern && !Op.hasOneUse())
3715 return false;
3716
3717 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3718 return true;
3719
3720 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3721 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3722
3723 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3724 return false;
3725
3726 Src = getLeftShift(CurDAG, Op: Op.getOperand(i: 0), ShlAmount: ShlImm - DstLSB);
3727 return true;
3728}
3729
3730static bool isShiftedMask(uint64_t Mask, EVT VT) {
3731 assert(VT == MVT::i32 || VT == MVT::i64);
3732 if (VT == MVT::i32)
3733 return isShiftedMask_32(Value: Mask);
3734 return isShiftedMask_64(Value: Mask);
3735}
3736
3737// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3738// inserted only sets known zero bits.
3739static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
3740 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3741
3742 EVT VT = N->getValueType(ResNo: 0);
3743 if (VT != MVT::i32 && VT != MVT::i64)
3744 return false;
3745
3746 unsigned BitWidth = VT.getSizeInBits();
3747
3748 uint64_t OrImm;
3749 if (!isOpcWithIntImmediate(N, Opc: ISD::OR, Imm&: OrImm))
3750 return false;
3751
3752 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3753 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3754 // performance neutral.
3755 if (AArch64_AM::isLogicalImmediate(imm: OrImm, regSize: BitWidth))
3756 return false;
3757
3758 uint64_t MaskImm;
3759 SDValue And = N->getOperand(Num: 0);
3760 // Must be a single use AND with an immediate operand.
3761 if (!And.hasOneUse() ||
3762 !isOpcWithIntImmediate(N: And.getNode(), Opc: ISD::AND, Imm&: MaskImm))
3763 return false;
3764
3765 // Compute the Known Zero for the AND as this allows us to catch more general
3766 // cases than just looking for AND with imm.
3767 KnownBits Known = CurDAG->computeKnownBits(Op: And);
3768
3769 // Non-zero in the sense that they're not provably zero, which is the key
3770 // point if we want to use this value.
3771 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3772
3773 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3774 if (!isShiftedMask(Mask: Known.Zero.getZExtValue(), VT))
3775 return false;
3776
3777 // The bits being inserted must only set those bits that are known to be zero.
3778 if ((OrImm & NotKnownZero) != 0) {
3779 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3780 // currently handle this case.
3781 return false;
3782 }
3783
3784 // BFI/BFXIL dst, src, #lsb, #width.
3785 int LSB = llvm::countr_one(Value: NotKnownZero);
3786 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3787
3788 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3789 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3790 unsigned ImmS = Width - 1;
3791
3792 // If we're creating a BFI instruction avoid cases where we need more
3793 // instructions to materialize the BFI constant as compared to the original
3794 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3795 // should be no worse in this case.
3796 bool IsBFI = LSB != 0;
3797 uint64_t BFIImm = OrImm >> LSB;
3798 if (IsBFI && !AArch64_AM::isLogicalImmediate(imm: BFIImm, regSize: BitWidth)) {
3799 // We have a BFI instruction and we know the constant can't be materialized
3800 // with a ORR-immediate with the zero register.
3801 unsigned OrChunks = 0, BFIChunks = 0;
3802 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3803 if (((OrImm >> Shift) & 0xFFFF) != 0)
3804 ++OrChunks;
3805 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3806 ++BFIChunks;
3807 }
3808 if (BFIChunks > OrChunks)
3809 return false;
3810 }
3811
3812 // Materialize the constant to be inserted.
3813 SDLoc DL(N);
3814 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3815 SDNode *MOVI = CurDAG->getMachineNode(
3816 Opcode: MOVIOpc, dl: DL, VT, Op1: CurDAG->getTargetConstant(Val: BFIImm, DL, VT));
3817
3818 // Create the BFI/BFXIL instruction.
3819 SDValue Ops[] = {And.getOperand(i: 0), SDValue(MOVI, 0),
3820 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3821 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3822 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3823 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3824 return true;
3825}
3826
3827static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG,
3828 SDValue &ShiftedOperand,
3829 uint64_t &EncodedShiftImm) {
3830 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3831 if (!Dst.hasOneUse())
3832 return false;
3833
3834 EVT VT = Dst.getValueType();
3835 assert((VT == MVT::i32 || VT == MVT::i64) &&
3836 "Caller should guarantee that VT is one of i32 or i64");
3837 const unsigned SizeInBits = VT.getSizeInBits();
3838
3839 SDLoc DL(Dst.getNode());
3840 uint64_t AndImm, ShlImm;
3841 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::AND, Imm&: AndImm) &&
3842 isShiftedMask_64(Value: AndImm)) {
3843 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3844 SDValue DstOp0 = Dst.getOperand(i: 0);
3845 if (!DstOp0.hasOneUse())
3846 return false;
3847
3848 // An example to illustrate the transformation
3849 // From:
3850 // lsr x8, x1, #1
3851 // and x8, x8, #0x3f80
3852 // bfxil x8, x1, #0, #7
3853 // To:
3854 // and x8, x23, #0x7f
3855 // ubfx x9, x23, #8, #7
3856 // orr x23, x8, x9, lsl #7
3857 //
3858 // The number of instructions remains the same, but ORR is faster than BFXIL
3859 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3860 // the dependency chain is improved after the transformation.
3861 uint64_t SrlImm;
3862 if (isOpcWithIntImmediate(N: DstOp0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3863 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(Val: AndImm);
3864 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3865 unsigned MaskWidth =
3866 llvm::countr_one(Value: AndImm >> NumTrailingZeroInShiftedMask);
3867 unsigned UBFMOpc =
3868 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3869 SDNode *UBFMNode = CurDAG->getMachineNode(
3870 Opcode: UBFMOpc, dl: DL, VT, Op1: DstOp0.getOperand(i: 0),
3871 Op2: CurDAG->getTargetConstant(Val: SrlImm + NumTrailingZeroInShiftedMask, DL,
3872 VT),
3873 Op3: CurDAG->getTargetConstant(
3874 Val: SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3875 ShiftedOperand = SDValue(UBFMNode, 0);
3876 EncodedShiftImm = AArch64_AM::getShifterImm(
3877 ST: AArch64_AM::LSL, Imm: NumTrailingZeroInShiftedMask);
3878 return true;
3879 }
3880 }
3881 return false;
3882 }
3883
3884 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3885 ShiftedOperand = Dst.getOperand(i: 0);
3886 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm);
3887 return true;
3888 }
3889
3890 uint64_t SrlImm;
3891 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3892 ShiftedOperand = Dst.getOperand(i: 0);
3893 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm);
3894 return true;
3895 }
3896 return false;
3897}
3898
3899// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3900// the operands and select it to AArch64::ORR with shifted registers if
3901// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3902static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3903 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3904 const bool BiggerPattern) {
3905 EVT VT = N->getValueType(ResNo: 0);
3906 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3907 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3908 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3909 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3910 assert((VT == MVT::i32 || VT == MVT::i64) &&
3911 "Expect result type to be i32 or i64 since N is combinable to BFM");
3912 SDLoc DL(N);
3913
3914 // Bail out if BFM simplifies away one node in BFM Dst.
3915 if (OrOpd1 != Dst)
3916 return false;
3917
3918 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3919 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3920 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3921 if (BiggerPattern) {
3922 uint64_t SrcAndImm;
3923 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::AND, Imm&: SrcAndImm) &&
3924 isMask_64(Value: SrcAndImm) && OrOpd0.getOperand(i: 0) == Src) {
3925 // OrOpd0 = AND Src, #Mask
3926 // So BFM simplifies away one AND node from Src and doesn't simplify away
3927 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3928 // one node (from Rd), ORR is better since it has higher throughput and
3929 // smaller latency than BFM on many AArch64 processors (and for the rest
3930 // ORR is at least as good as BFM).
3931 SDValue ShiftedOperand;
3932 uint64_t EncodedShiftImm;
3933 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3934 EncodedShiftImm)) {
3935 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3936 CurDAG->getTargetConstant(Val: EncodedShiftImm, DL, VT)};
3937 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3938 return true;
3939 }
3940 }
3941 return false;
3942 }
3943
3944 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3945
3946 uint64_t ShlImm;
3947 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3948 if (OrOpd0.getOperand(i: 0) == Src && OrOpd0.hasOneUse()) {
3949 SDValue Ops[] = {
3950 Dst, Src,
3951 CurDAG->getTargetConstant(
3952 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3953 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3954 return true;
3955 }
3956
3957 // Select the following pattern to left-shifted operand rather than BFI.
3958 // %val1 = op ..
3959 // %val2 = shl %val1, #imm
3960 // %res = or %val1, %val2
3961 //
3962 // If N is selected to be BFI, we know that
3963 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3964 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3965 //
3966 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3967 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3968 SDValue Ops[] = {
3969 OrOpd1, OrOpd1,
3970 CurDAG->getTargetConstant(
3971 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3972 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3973 return true;
3974 }
3975 }
3976
3977 uint64_t SrlImm;
3978 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3979 // Select the following pattern to right-shifted operand rather than BFXIL.
3980 // %val1 = op ..
3981 // %val2 = lshr %val1, #imm
3982 // %res = or %val1, %val2
3983 //
3984 // If N is selected to be BFXIL, we know that
3985 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3986 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
3987 //
3988 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
3989 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3990 SDValue Ops[] = {
3991 OrOpd1, OrOpd1,
3992 CurDAG->getTargetConstant(
3993 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm), DL, VT)};
3994 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3995 return true;
3996 }
3997 }
3998
3999 return false;
4000}
4001
4002static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
4003 SelectionDAG *CurDAG) {
4004 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
4005
4006 EVT VT = N->getValueType(ResNo: 0);
4007 if (VT != MVT::i32 && VT != MVT::i64)
4008 return false;
4009
4010 unsigned BitWidth = VT.getSizeInBits();
4011
4012 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
4013 // have the expected shape. Try to undo that.
4014
4015 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
4016 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
4017
4018 // Given a OR operation, check if we have the following pattern
4019 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
4020 // isBitfieldExtractOp)
4021 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
4022 // countTrailingZeros(mask2) == imm2 - imm + 1
4023 // f = d | c
4024 // if yes, replace the OR instruction with:
4025 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
4026
4027 // OR is commutative, check all combinations of operand order and values of
4028 // BiggerPattern, i.e.
4029 // Opd0, Opd1, BiggerPattern=false
4030 // Opd1, Opd0, BiggerPattern=false
4031 // Opd0, Opd1, BiggerPattern=true
4032 // Opd1, Opd0, BiggerPattern=true
4033 // Several of these combinations may match, so check with BiggerPattern=false
4034 // first since that will produce better results by matching more instructions
4035 // and/or inserting fewer extra instructions.
4036 for (int I = 0; I < 4; ++I) {
4037
4038 SDValue Dst, Src;
4039 unsigned ImmR, ImmS;
4040 bool BiggerPattern = I / 2;
4041 SDValue OrOpd0Val = N->getOperand(Num: I % 2);
4042 SDNode *OrOpd0 = OrOpd0Val.getNode();
4043 SDValue OrOpd1Val = N->getOperand(Num: (I + 1) % 2);
4044 SDNode *OrOpd1 = OrOpd1Val.getNode();
4045
4046 unsigned BFXOpc;
4047 int DstLSB, Width;
4048 if (isBitfieldExtractOp(CurDAG, N: OrOpd0, Opc&: BFXOpc, Opd0&: Src, Immr&: ImmR, Imms&: ImmS,
4049 NumberOfIgnoredLowBits, BiggerPattern)) {
4050 // Check that the returned opcode is compatible with the pattern,
4051 // i.e., same type and zero extended (U and not S)
4052 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
4053 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
4054 continue;
4055
4056 // Compute the width of the bitfield insertion
4057 DstLSB = 0;
4058 Width = ImmS - ImmR + 1;
4059 // FIXME: This constraint is to catch bitfield insertion we may
4060 // want to widen the pattern if we want to grab general bitfield
4061 // move case
4062 if (Width <= 0)
4063 continue;
4064
4065 // If the mask on the insertee is correct, we have a BFXIL operation. We
4066 // can share the ImmR and ImmS values from the already-computed UBFM.
4067 } else if (isBitfieldPositioningOp(CurDAG, Op: OrOpd0Val,
4068 BiggerPattern,
4069 Src, DstLSB, Width)) {
4070 ImmR = (BitWidth - DstLSB) % BitWidth;
4071 ImmS = Width - 1;
4072 } else
4073 continue;
4074
4075 // Check the second part of the pattern
4076 EVT VT = OrOpd1Val.getValueType();
4077 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
4078
4079 // Compute the Known Zero for the candidate of the first operand.
4080 // This allows to catch more general case than just looking for
4081 // AND with imm. Indeed, simplify-demanded-bits may have removed
4082 // the AND instruction because it proves it was useless.
4083 KnownBits Known = CurDAG->computeKnownBits(Op: OrOpd1Val);
4084
4085 // Check if there is enough room for the second operand to appear
4086 // in the first one
4087 APInt BitsToBeInserted =
4088 APInt::getBitsSet(numBits: Known.getBitWidth(), loBit: DstLSB, hiBit: DstLSB + Width);
4089
4090 if ((BitsToBeInserted & ~Known.Zero) != 0)
4091 continue;
4092
4093 // Set the first operand
4094 uint64_t Imm;
4095 if (isOpcWithIntImmediate(N: OrOpd1, Opc: ISD::AND, Imm) &&
4096 isBitfieldDstMask(DstMask: Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
4097 // In that case, we can eliminate the AND
4098 Dst = OrOpd1->getOperand(Num: 0);
4099 else
4100 // Maybe the AND has been removed by simplify-demanded-bits
4101 // or is useful because it discards more bits
4102 Dst = OrOpd1Val;
4103
4104 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
4105 // with shifted operand is more efficient.
4106 if (tryOrrWithShift(N, OrOpd0: OrOpd0Val, OrOpd1: OrOpd1Val, Src, Dst, CurDAG,
4107 BiggerPattern))
4108 return true;
4109
4110 // both parts match
4111 SDLoc DL(N);
4112 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4113 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4114 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4115 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4116 return true;
4117 }
4118
4119 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
4120 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
4121 // mask (e.g., 0x000ffff0).
4122 uint64_t Mask0Imm, Mask1Imm;
4123 SDValue And0 = N->getOperand(Num: 0);
4124 SDValue And1 = N->getOperand(Num: 1);
4125 if (And0.hasOneUse() && And1.hasOneUse() &&
4126 isOpcWithIntImmediate(N: And0.getNode(), Opc: ISD::AND, Imm&: Mask0Imm) &&
4127 isOpcWithIntImmediate(N: And1.getNode(), Opc: ISD::AND, Imm&: Mask1Imm) &&
4128 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
4129 (isShiftedMask(Mask: Mask0Imm, VT) || isShiftedMask(Mask: Mask1Imm, VT))) {
4130
4131 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
4132 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
4133 // bits to be inserted.
4134 if (isShiftedMask(Mask: Mask0Imm, VT)) {
4135 std::swap(a&: And0, b&: And1);
4136 std::swap(a&: Mask0Imm, b&: Mask1Imm);
4137 }
4138
4139 SDValue Src = And1->getOperand(Num: 0);
4140 SDValue Dst = And0->getOperand(Num: 0);
4141 unsigned LSB = llvm::countr_zero(Val: Mask1Imm);
4142 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
4143
4144 // The BFXIL inserts the low-order bits from a source register, so right
4145 // shift the needed bits into place.
4146 SDLoc DL(N);
4147 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4148 uint64_t LsrImm = LSB;
4149 if (Src->hasOneUse() &&
4150 isOpcWithIntImmediate(N: Src.getNode(), Opc: ISD::SRL, Imm&: LsrImm) &&
4151 (LsrImm + LSB) < BitWidth) {
4152 Src = Src->getOperand(Num: 0);
4153 LsrImm += LSB;
4154 }
4155
4156 SDNode *LSR = CurDAG->getMachineNode(
4157 Opcode: ShiftOpc, dl: DL, VT, Op1: Src, Op2: CurDAG->getTargetConstant(Val: LsrImm, DL, VT),
4158 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT));
4159
4160 // BFXIL is an alias of BFM, so translate to BFM operands.
4161 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4162 unsigned ImmS = Width - 1;
4163
4164 // Create the BFXIL instruction.
4165 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4166 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4167 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4168 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4169 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4170 return true;
4171 }
4172
4173 return false;
4174}
4175
4176bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4177 if (N->getOpcode() != ISD::OR)
4178 return false;
4179
4180 APInt NUsefulBits;
4181 getUsefulBits(Op: SDValue(N, 0), UsefulBits&: NUsefulBits);
4182
4183 // If all bits are not useful, just return UNDEF.
4184 if (!NUsefulBits) {
4185 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::IMPLICIT_DEF, VT: N->getValueType(ResNo: 0));
4186 return true;
4187 }
4188
4189 if (tryBitfieldInsertOpFromOr(N, UsefulBits: NUsefulBits, CurDAG))
4190 return true;
4191
4192 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4193}
4194
4195/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4196/// equivalent of a left shift by a constant amount followed by an and masking
4197/// out a contiguous set of bits.
4198bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4199 if (N->getOpcode() != ISD::AND)
4200 return false;
4201
4202 EVT VT = N->getValueType(ResNo: 0);
4203 if (VT != MVT::i32 && VT != MVT::i64)
4204 return false;
4205
4206 SDValue Op0;
4207 int DstLSB, Width;
4208 if (!isBitfieldPositioningOp(CurDAG, Op: SDValue(N, 0), /*BiggerPattern=*/false,
4209 Src&: Op0, DstLSB, Width))
4210 return false;
4211
4212 // ImmR is the rotate right amount.
4213 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4214 // ImmS is the most significant bit of the source to be moved.
4215 unsigned ImmS = Width - 1;
4216
4217 SDLoc DL(N);
4218 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4219 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4220 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4221 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4222 return true;
4223}
4224
4225static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
4226 SDValue &FixedPos,
4227 unsigned RegWidth,
4228 bool isReciprocal) {
4229 APFloat FVal(0.0);
4230 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
4231 FVal = CN->getValueAPF();
4232 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(Val&: N)) {
4233 // Some otherwise illegal constants are allowed in this case.
4234 if (LN->getOperand(Num: 1).getOpcode() != AArch64ISD::ADDlow ||
4235 !isa<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1)))
4236 return false;
4237
4238 ConstantPoolSDNode *CN =
4239 dyn_cast<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1));
4240 FVal = cast<ConstantFP>(Val: CN->getConstVal())->getValueAPF();
4241 } else
4242 return false;
4243
4244 if (unsigned FBits =
4245 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4246 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4247 return true;
4248 }
4249
4250 return false;
4251}
4252
4253static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
4254 SDValue N,
4255 SDValue &FixedPos,
4256 unsigned RegWidth,
4257 bool isReciprocal) {
4258 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4259 N.getValueType().getScalarSizeInBits() ==
4260 N.getOperand(i: 0).getValueType().getScalarSizeInBits())
4261 N = N.getOperand(i: 0);
4262
4263 auto ImmToFloat = [RegWidth](APInt Imm) {
4264 switch (RegWidth) {
4265 case 16:
4266 return APFloat(APFloat::IEEEhalf(), Imm);
4267 case 32:
4268 return APFloat(APFloat::IEEEsingle(), Imm);
4269 case 64:
4270 return APFloat(APFloat::IEEEdouble(), Imm);
4271 default:
4272 llvm_unreachable("Unexpected RegWidth!");
4273 };
4274 };
4275
4276 APFloat FVal(0.0);
4277 switch (N->getOpcode()) {
4278 case AArch64ISD::MOVIshift:
4279 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(i: 0)
4280 << N.getConstantOperandVal(i: 1)));
4281 break;
4282 case AArch64ISD::FMOV:
4283 FVal = ImmToFloat(DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth));
4284 break;
4285 case AArch64ISD::DUP:
4286 if (isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
4287 FVal = ImmToFloat(N.getConstantOperandAPInt(i: 0).trunc(width: RegWidth));
4288 else
4289 return false;
4290 break;
4291 default:
4292 return false;
4293 }
4294
4295 if (unsigned FBits =
4296 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4297 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4298 return true;
4299 }
4300
4301 return false;
4302}
4303
4304bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4305 unsigned RegWidth) {
4306 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4307 /*isReciprocal*/ false);
4308}
4309
4310bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4311 unsigned RegWidth) {
4312 return checkCVTFixedPointOperandWithFBitsForVectors(
4313 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4314}
4315
4316bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4317 SDValue &FixedPos,
4318 unsigned RegWidth) {
4319 return checkCVTFixedPointOperandWithFBitsForVectors(
4320 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4321}
4322
4323bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4324 SDValue &FixedPos,
4325 unsigned RegWidth) {
4326 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4327 /*isReciprocal*/ true);
4328}
4329
4330// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4331// of the string and obtains the integer values from them and combines these
4332// into a single value to be used in the MRS/MSR instruction.
4333static int getIntOperandFromRegisterString(StringRef RegString) {
4334 SmallVector<StringRef, 5> Fields;
4335 RegString.split(A&: Fields, Separator: ':');
4336
4337 if (Fields.size() == 1)
4338 return -1;
4339
4340 assert(Fields.size() == 5
4341 && "Invalid number of fields in read register string");
4342
4343 SmallVector<int, 5> Ops;
4344 bool AllIntFields = true;
4345
4346 for (StringRef Field : Fields) {
4347 unsigned IntField;
4348 AllIntFields &= !Field.getAsInteger(Radix: 10, Result&: IntField);
4349 Ops.push_back(Elt: IntField);
4350 }
4351
4352 assert(AllIntFields &&
4353 "Unexpected non-integer value in special register string.");
4354 (void)AllIntFields;
4355
4356 // Need to combine the integer fields of the string into a single value
4357 // based on the bit encoding of MRS/MSR instruction.
4358 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4359 (Ops[4]);
4360}
4361
4362// Lower the read_register intrinsic to an MRS instruction node if the special
4363// register string argument is either of the form detailed in the ALCE (the
4364// form described in getIntOperandsFromRegisterString) or is a named register
4365// known by the MRS SysReg mapper.
4366bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4367 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4368 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4369 SDLoc DL(N);
4370
4371 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4372
4373 unsigned Opcode64Bit = AArch64::MRS;
4374 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4375 if (Imm == -1) {
4376 // No match, Use the sysreg mapper to map the remaining possible strings to
4377 // the value for the register to be used for the instruction operand.
4378 const auto *TheReg =
4379 AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4380 if (TheReg && TheReg->Readable &&
4381 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4382 Imm = TheReg->Encoding;
4383 else
4384 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4385
4386 if (Imm == -1) {
4387 // Still no match, see if this is "pc" or give up.
4388 if (!ReadIs128Bit && RegString->getString() == "pc") {
4389 Opcode64Bit = AArch64::ADR;
4390 Imm = 0;
4391 } else {
4392 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4393 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4394 // carries the source register as an immediate so the read does not
4395 // reference an undefined physical register (which the machine verifier
4396 // rejects); the AsmPrinter materializes the real mov/fmov.
4397 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4398 RegName: RegString->getString());
4399 unsigned PseudoOp = 0;
4400 if (AArch64::GPR64RegClass.contains(Reg: PReg))
4401 PseudoOp = AArch64::READ_REGISTER_GPR64;
4402 else if (AArch64::FPR64RegClass.contains(Reg: PReg))
4403 PseudoOp = AArch64::READ_REGISTER_FPR64;
4404 if (!ReadIs128Bit && PseudoOp && N->getValueType(ResNo: 0) == MVT::i64) {
4405 CurDAG->SelectNodeTo(N, MachineOpc: PseudoOp, VT1: MVT::i64, VT2: MVT::Other,
4406 Ops: {CurDAG->getTargetConstant(Val: PReg, DL, VT: MVT::i32),
4407 N->getOperand(Num: 0)});
4408 return true;
4409 }
4410 return false;
4411 }
4412 }
4413 }
4414
4415 SDValue InChain = N->getOperand(Num: 0);
4416 SDValue SysRegImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
4417 if (!ReadIs128Bit) {
4418 CurDAG->SelectNodeTo(N, MachineOpc: Opcode64Bit, VT1: MVT::i64, VT2: MVT::Other /* Chain */,
4419 Ops: {SysRegImm, InChain});
4420 } else {
4421 SDNode *MRRS = CurDAG->getMachineNode(
4422 Opcode: AArch64::MRRS, dl: DL,
4423 ResultTys: {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4424 Ops: {SysRegImm, InChain});
4425
4426 // Sysregs are not endian. The even register always contains the low half
4427 // of the register.
4428 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sube64, DL, VT: MVT::i64,
4429 Operand: SDValue(MRRS, 0));
4430 SDValue Hi = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::subo64, DL, VT: MVT::i64,
4431 Operand: SDValue(MRRS, 0));
4432 SDValue OutChain = SDValue(MRRS, 1);
4433
4434 ReplaceUses(F: SDValue(N, 0), T: Lo);
4435 ReplaceUses(F: SDValue(N, 1), T: Hi);
4436 ReplaceUses(F: SDValue(N, 2), T: OutChain);
4437 };
4438 return true;
4439}
4440
4441// Lower the write_register intrinsic to an MSR instruction node if the special
4442// register string argument is either of the form detailed in the ALCE (the
4443// form described in getIntOperandsFromRegisterString) or is a named register
4444// known by the MSR SysReg mapper.
4445bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4446 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4447 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4448 SDLoc DL(N);
4449
4450 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4451
4452 if (!WriteIs128Bit) {
4453 // Check if the register was one of those allowed as the pstatefield value
4454 // in the MSR (immediate) instruction. To accept the values allowed in the
4455 // pstatefield for the MSR (immediate) instruction, we also require that an
4456 // immediate value has been provided as an argument, we know that this is
4457 // the case as it has been ensured by semantic checking.
4458 auto trySelectPState = [&](auto PMapper, unsigned State) {
4459 if (PMapper) {
4460 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4461 "Expected a constant integer expression.");
4462 unsigned Reg = PMapper->Encoding;
4463 uint64_t Immed = N->getConstantOperandVal(Num: 2);
4464 CurDAG->SelectNodeTo(
4465 N, MachineOpc: State, VT: MVT::Other, Op1: CurDAG->getTargetConstant(Val: Reg, DL, VT: MVT::i32),
4466 Op2: CurDAG->getTargetConstant(Val: Immed, DL, VT: MVT::i16), Op3: N->getOperand(Num: 0));
4467 return true;
4468 }
4469 return false;
4470 };
4471
4472 if (trySelectPState(
4473 AArch64PState::lookupPStateImm0_15ByName(Name: RegString->getString()),
4474 AArch64::MSRpstateImm4))
4475 return true;
4476 if (trySelectPState(
4477 AArch64PState::lookupPStateImm0_1ByName(Name: RegString->getString()),
4478 AArch64::MSRpstateImm1))
4479 return true;
4480 }
4481
4482 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4483 if (Imm == -1) {
4484 // Use the sysreg mapper to attempt to map the remaining possible strings
4485 // to the value for the register to be used for the MSR (register)
4486 // instruction operand.
4487 auto TheReg = AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4488 if (TheReg && TheReg->Writeable &&
4489 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4490 Imm = TheReg->Encoding;
4491 else
4492 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4493
4494 if (Imm == -1) {
4495 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4496 // the physical register and keep it live with a FAKE_USE so the write is
4497 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4498 // so the generic write path cannot handle these.)
4499 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4500 RegName: RegString->getString());
4501 bool IsGPR = AArch64::GPR64RegClass.contains(Reg: PReg);
4502 bool IsFPR = AArch64::FPR64RegClass.contains(Reg: PReg);
4503 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4504 N->getOperand(Num: 2).getValueType() == MVT::i64) {
4505 SDValue Copy =
4506 CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: DL, Reg: PReg, N: N->getOperand(Num: 2));
4507 SDValue RegOp = CurDAG->getRegister(Reg: PReg, VT: MVT::i64);
4508 SDNode *FakeUse = CurDAG->getMachineNode(Opcode: TargetOpcode::FAKE_USE, dl: DL,
4509 VT: MVT::Other, Ops: {RegOp, Copy});
4510 ReplaceUses(F: SDValue(N, 0), T: SDValue(FakeUse, 0));
4511 CurDAG->RemoveDeadNode(N);
4512 return true;
4513 }
4514 return false;
4515 }
4516 }
4517
4518 SDValue InChain = N->getOperand(Num: 0);
4519 if (!WriteIs128Bit) {
4520 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSR, VT: MVT::Other,
4521 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4522 Op2: N->getOperand(Num: 2), Op3: InChain);
4523 } else {
4524 // No endian swap. The lower half always goes into the even subreg, and the
4525 // higher half always into the odd supreg.
4526 SDNode *Pair = CurDAG->getMachineNode(
4527 Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped /* XSeqPair */,
4528 Ops: {CurDAG->getTargetConstant(Val: AArch64::XSeqPairsClassRegClass.getID(), DL,
4529 VT: MVT::i32),
4530 N->getOperand(Num: 2),
4531 CurDAG->getTargetConstant(Val: AArch64::sube64, DL, VT: MVT::i32),
4532 N->getOperand(Num: 3),
4533 CurDAG->getTargetConstant(Val: AArch64::subo64, DL, VT: MVT::i32)});
4534
4535 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSRR, VT: MVT::Other,
4536 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4537 Op2: SDValue(Pair, 0), Op3: InChain);
4538 }
4539
4540 return true;
4541}
4542
4543/// We've got special pseudo-instructions for these
4544bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4545 unsigned Opcode;
4546 EVT MemTy = cast<MemSDNode>(Val: N)->getMemoryVT();
4547
4548 // Leave IR for LSE if subtarget supports it.
4549 if (Subtarget->hasLSE()) return false;
4550
4551 if (MemTy == MVT::i8)
4552 Opcode = AArch64::CMP_SWAP_8;
4553 else if (MemTy == MVT::i16)
4554 Opcode = AArch64::CMP_SWAP_16;
4555 else if (MemTy == MVT::i32)
4556 Opcode = AArch64::CMP_SWAP_32;
4557 else if (MemTy == MVT::i64)
4558 Opcode = AArch64::CMP_SWAP_64;
4559 else
4560 llvm_unreachable("Unknown AtomicCmpSwap type");
4561
4562 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4563 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3),
4564 N->getOperand(Num: 0)};
4565 SDNode *CmpSwap = CurDAG->getMachineNode(
4566 Opcode, dl: SDLoc(N),
4567 VTs: CurDAG->getVTList(VT1: RegTy, VT2: MVT::i32, VT3: MVT::Other), Ops);
4568
4569 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
4570 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
4571
4572 ReplaceUses(F: SDValue(N, 0), T: SDValue(CmpSwap, 0));
4573 ReplaceUses(F: SDValue(N, 1), T: SDValue(CmpSwap, 2));
4574 CurDAG->RemoveDeadNode(N);
4575
4576 return true;
4577}
4578
4579AArch64MemoryHint
4580AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
4581 int MemoryHint = -1;
4582 const MDNode *MemCacheHint = MMO->getMemCacheHint();
4583 if (!MemCacheHint)
4584 return AArch64MemoryHint::NONE;
4585
4586 for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
4587 if (MemCacheHint->getOperand(I).equalsStr(Str: "aarch64.mem_hint")) {
4588 const Metadata *Val = MemCacheHint->getOperand(I: I + 1).get();
4589 MemoryHint = cast<ConstantInt>(Val: cast<ConstantAsMetadata>(Val)->getValue())
4590 ->getZExtValue();
4591 }
4592 }
4593
4594 return toAArch64MemoryHint(I: MemoryHint);
4595}
4596
4597bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N,
4598 AArch64MemoryHint Hint) const {
4599 return decodeMemoryHintFlags(MMO: cast<MemSDNode>(Val: N)->getMemOperand()) == Hint;
4600}
4601
4602bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4603 SDValue &Shift, bool Negate) {
4604 if (!isa<ConstantSDNode>(Val: N))
4605 return false;
4606
4607 APInt Val =
4608 cast<ConstantSDNode>(Val&: N)->getAPIntValue().trunc(width: VT.getFixedSizeInBits());
4609
4610 return SelectSVEAddSubImm(DL: SDLoc(N), Value: Val, VT, Imm, Shift, Negate);
4611}
4612
4613bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4614 SDValue &Imm, SDValue &Shift,
4615 bool Negate) {
4616 if (Negate)
4617 Val = -Val;
4618
4619 switch (VT.SimpleTy) {
4620 case MVT::i8:
4621 // All immediates are supported.
4622 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4623 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4624 return true;
4625 case MVT::i16:
4626 case MVT::i32:
4627 case MVT::i64:
4628 // Support 8bit unsigned immediates.
4629 if ((Val & ~0xff) == 0) {
4630 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4631 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4632 return true;
4633 }
4634 // Support 16bit unsigned immediates that are a multiple of 256.
4635 if ((Val & ~0xff00) == 0) {
4636 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4637 Imm = CurDAG->getTargetConstant(Val: Val.lshr(shiftAmt: 8).getZExtValue(), DL, VT: MVT::i32);
4638 return true;
4639 }
4640 break;
4641 default:
4642 break;
4643 }
4644
4645 return false;
4646}
4647
4648bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4649 SDValue &Imm, SDValue &Shift,
4650 bool Negate) {
4651 if (!isa<ConstantSDNode>(Val: N))
4652 return false;
4653
4654 SDLoc DL(N);
4655 int64_t Val = cast<ConstantSDNode>(Val&: N)
4656 ->getAPIntValue()
4657 .trunc(width: VT.getFixedSizeInBits())
4658 .getSExtValue();
4659
4660 if (Negate)
4661 Val = -Val;
4662
4663 // Signed saturating instructions treat their immediate operand as unsigned,
4664 // whereas the related intrinsics define their operands to be signed. This
4665 // means we can only use the immediate form when the operand is non-negative.
4666 if (Val < 0)
4667 return false;
4668
4669 switch (VT.SimpleTy) {
4670 case MVT::i8:
4671 // All positive immediates are supported.
4672 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4673 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4674 return true;
4675 case MVT::i16:
4676 case MVT::i32:
4677 case MVT::i64:
4678 // Support 8bit positive immediates.
4679 if (Val <= 255) {
4680 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4681 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4682 return true;
4683 }
4684 // Support 16bit positive immediates that are a multiple of 256.
4685 if (Val <= 65280 && Val % 256 == 0) {
4686 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4687 Imm = CurDAG->getTargetConstant(Val: Val >> 8, DL, VT: MVT::i32);
4688 return true;
4689 }
4690 break;
4691 default:
4692 break;
4693 }
4694
4695 return false;
4696}
4697
4698bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4699 SDValue &Shift) {
4700 if (!isa<ConstantSDNode>(Val: N))
4701 return false;
4702
4703 SDLoc DL(N);
4704 int64_t Val = cast<ConstantSDNode>(Val&: N)
4705 ->getAPIntValue()
4706 .trunc(width: VT.getFixedSizeInBits())
4707 .getSExtValue();
4708 int32_t ImmVal, ShiftVal;
4709 if (!AArch64_AM::isSVECpyDupImm(SizeInBits: VT.getScalarSizeInBits(), Val, Imm&: ImmVal,
4710 Shift&: ShiftVal))
4711 return false;
4712
4713 Shift = CurDAG->getTargetConstant(Val: ShiftVal, DL, VT: MVT::i32);
4714 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4715 return true;
4716}
4717
4718bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4719 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N))
4720 return SelectSVESignedArithImm(DL: SDLoc(N), Value: CNode->getAPIntValue(), Imm);
4721 return false;
4722}
4723
4724bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4725 SDValue &Imm) {
4726 int64_t ImmVal = Val.getSExtValue();
4727 if (ImmVal >= -128 && ImmVal < 128) {
4728 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4729 return true;
4730 }
4731 return false;
4732}
4733
4734bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4735 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N)) {
4736 uint64_t ImmVal = CNode->getZExtValue();
4737
4738 switch (VT.SimpleTy) {
4739 case MVT::i8:
4740 ImmVal &= 0xFF;
4741 break;
4742 case MVT::i16:
4743 ImmVal &= 0xFFFF;
4744 break;
4745 case MVT::i32:
4746 ImmVal &= 0xFFFFFFFF;
4747 break;
4748 case MVT::i64:
4749 break;
4750 default:
4751 llvm_unreachable("Unexpected type");
4752 }
4753
4754 if (ImmVal < 256) {
4755 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4756 return true;
4757 }
4758 }
4759 return false;
4760}
4761
4762bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4763 bool Invert) {
4764 uint64_t ImmVal;
4765 if (auto CI = dyn_cast<ConstantSDNode>(Val&: N))
4766 ImmVal = CI->getZExtValue();
4767 else if (auto CFP = dyn_cast<ConstantFPSDNode>(Val&: N))
4768 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4769 else
4770 return false;
4771
4772 if (Invert)
4773 ImmVal = ~ImmVal;
4774
4775 uint64_t encoding;
4776 if (!AArch64_AM::isSVELogicalImm(SizeInBits: VT.getScalarSizeInBits(), ImmVal, Encoding&: encoding))
4777 return false;
4778
4779 Imm = CurDAG->getTargetConstant(Val: encoding, DL: SDLoc(N), VT: MVT::i64);
4780 return true;
4781}
4782
4783// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4784// Rather than attempt to normalise everything we can sometimes saturate the
4785// shift amount during selection. This function also allows for consistent
4786// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4787// required by the instructions.
4788bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4789 uint64_t High, bool AllowSaturation,
4790 SDValue &Imm) {
4791 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
4792 uint64_t ImmVal = CN->getZExtValue();
4793
4794 // Reject shift amounts that are too small.
4795 if (ImmVal < Low)
4796 return false;
4797
4798 // Reject or saturate shift amounts that are too big.
4799 if (ImmVal > High) {
4800 if (!AllowSaturation)
4801 return false;
4802 ImmVal = High;
4803 }
4804
4805 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4806 return true;
4807 }
4808
4809 return false;
4810}
4811
4812bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4813 // tagp(FrameIndex, IRGstack, tag_offset):
4814 // since the offset between FrameIndex and IRGstack is a compile-time
4815 // constant, this can be lowered to a single ADDG instruction.
4816 if (!(isa<FrameIndexSDNode>(Val: N->getOperand(Num: 1)))) {
4817 return false;
4818 }
4819
4820 SDValue IRG_SP = N->getOperand(Num: 2);
4821 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4822 IRG_SP->getConstantOperandVal(Num: 1) != Intrinsic::aarch64_irg_sp) {
4823 return false;
4824 }
4825
4826 const TargetLowering *TLI = getTargetLowering();
4827 SDLoc DL(N);
4828 int FI = cast<FrameIndexSDNode>(Val: N->getOperand(Num: 1))->getIndex();
4829 SDValue FiOp = CurDAG->getTargetFrameIndex(
4830 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
4831 int TagOffset = N->getConstantOperandVal(Num: 3);
4832
4833 SDNode *Out = CurDAG->getMachineNode(
4834 Opcode: AArch64::TAGPstack, dl: DL, VT: MVT::i64,
4835 Ops: {FiOp, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), N->getOperand(Num: 2),
4836 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4837 ReplaceNode(F: N, T: Out);
4838 return true;
4839}
4840
4841void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4842 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4843 "llvm.aarch64.tagp third argument must be an immediate");
4844 if (trySelectStackSlotTagP(N))
4845 return;
4846 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4847 // compile-time constant, not just for stack allocations.
4848
4849 // General case for unrelated pointers in Op1 and Op2.
4850 SDLoc DL(N);
4851 int TagOffset = N->getConstantOperandVal(Num: 3);
4852 SDNode *N1 = CurDAG->getMachineNode(Opcode: AArch64::SUBP, dl: DL, VT: MVT::i64,
4853 Ops: {N->getOperand(Num: 1), N->getOperand(Num: 2)});
4854 SDNode *N2 = CurDAG->getMachineNode(Opcode: AArch64::ADDXrr, dl: DL, VT: MVT::i64,
4855 Ops: {SDValue(N1, 0), N->getOperand(Num: 2)});
4856 SDNode *N3 = CurDAG->getMachineNode(
4857 Opcode: AArch64::ADDG, dl: DL, VT: MVT::i64,
4858 Ops: {SDValue(N2, 0), CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64),
4859 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4860 ReplaceNode(F: N, T: N3);
4861}
4862
4863bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4864 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4865
4866 // Bail when not a "cast" like insert_subvector.
4867 if (N->getConstantOperandVal(Num: 2) != 0)
4868 return false;
4869 if (!N->getOperand(Num: 0).isUndef())
4870 return false;
4871
4872 // Bail when normal isel should do the job.
4873 EVT VT = N->getValueType(ResNo: 0);
4874 EVT InVT = N->getOperand(Num: 1).getValueType();
4875 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4876 return false;
4877 if (InVT.getSizeInBits() <= 128)
4878 return false;
4879
4880 // NOTE: We can only get here when doing fixed length SVE code generation.
4881 // We do manual selection because the types involved are not linked to real
4882 // registers (despite being legal) and must be coerced into SVE registers.
4883
4884 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4885 "Expected to insert into a packed scalable vector!");
4886
4887 SDLoc DL(N);
4888 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4889 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4890 Op1: N->getOperand(Num: 1), Op2: RC));
4891 return true;
4892}
4893
4894bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4895 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4896
4897 // Bail when not a "cast" like extract_subvector.
4898 if (N->getConstantOperandVal(Num: 1) != 0)
4899 return false;
4900
4901 // Bail when normal isel can do the job.
4902 EVT VT = N->getValueType(ResNo: 0);
4903 EVT InVT = N->getOperand(Num: 0).getValueType();
4904 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4905 return false;
4906 if (VT.getSizeInBits() <= 128)
4907 return false;
4908
4909 // NOTE: We can only get here when doing fixed length SVE code generation.
4910 // We do manual selection because the types involved are not linked to real
4911 // registers (despite being legal) and must be coerced into SVE registers.
4912
4913 assert(InVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4914 "Expected to extract from a packed scalable vector!");
4915
4916 SDLoc DL(N);
4917 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4918 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4919 Op1: N->getOperand(Num: 0), Op2: RC));
4920 return true;
4921}
4922
4923bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4924 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4925
4926 SDValue N0 = N->getOperand(Num: 0);
4927 SDValue N1 = N->getOperand(Num: 1);
4928
4929 EVT VT = N->getValueType(ResNo: 0);
4930 SDLoc DL(N);
4931
4932 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4933 // Rotate by a constant is a funnel shift in IR which is expanded to
4934 // an OR with shifted operands.
4935 // We do the following transform:
4936 // OR N0, N1 -> xar (x, y, imm)
4937 // Where:
4938 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4939 // N0 = SHL_PRED true, V, splat(bits-imm)
4940 // V = (xor x, y)
4941 if (VT.isScalableVector() &&
4942 (Subtarget->hasSVE2() ||
4943 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4944 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4945 N1.getOpcode() != AArch64ISD::SRL_PRED)
4946 std::swap(a&: N0, b&: N1);
4947 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4948 N1.getOpcode() != AArch64ISD::SRL_PRED)
4949 return false;
4950
4951 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4952 if (!TLI->isAllActivePredicate(DAG: *CurDAG, N: N0.getOperand(i: 0)) ||
4953 !TLI->isAllActivePredicate(DAG: *CurDAG, N: N1.getOperand(i: 0)))
4954 return false;
4955
4956 if (N0.getOperand(i: 1) != N1.getOperand(i: 1))
4957 return false;
4958
4959 SDValue R1, R2;
4960 bool IsXOROperand = true;
4961 if (N0.getOperand(i: 1).getOpcode() != ISD::XOR) {
4962 IsXOROperand = false;
4963 } else {
4964 R1 = N0.getOperand(i: 1).getOperand(i: 0);
4965 R2 = N1.getOperand(i: 1).getOperand(i: 1);
4966 }
4967
4968 APInt ShlAmt, ShrAmt;
4969 if (!ISD::isConstantSplatVector(N: N0.getOperand(i: 2).getNode(), SplatValue&: ShlAmt) ||
4970 !ISD::isConstantSplatVector(N: N1.getOperand(i: 2).getNode(), SplatValue&: ShrAmt))
4971 return false;
4972
4973 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4974 return false;
4975
4976 if (!IsXOROperand) {
4977 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
4978 SDNode *MOV = CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT, Op1: Zero);
4979 SDValue MOVIV = SDValue(MOV, 0);
4980
4981 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
4982 SDNode *SubRegToReg =
4983 CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT, Op1: MOVIV, Op2: ZSub);
4984
4985 R1 = N1->getOperand(Num: 1);
4986 R2 = SDValue(SubRegToReg, 0);
4987 }
4988
4989 SDValue Imm =
4990 CurDAG->getTargetConstant(Val: ShrAmt.getZExtValue(), DL, VT: MVT::i32);
4991
4992 SDValue Ops[] = {R1, R2, Imm};
4993 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
4994 VT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
4995 AArch64::XAR_ZZZI_D})) {
4996 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4997 return true;
4998 }
4999 return false;
5000 }
5001
5002 // We have Neon SHA3 XAR operation for v2i64 but for types
5003 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
5004 // is available.
5005 EVT SVT;
5006 switch (VT.getSimpleVT().SimpleTy) {
5007 case MVT::v4i32:
5008 case MVT::v2i32:
5009 SVT = MVT::nxv4i32;
5010 break;
5011 case MVT::v8i16:
5012 case MVT::v4i16:
5013 SVT = MVT::nxv8i16;
5014 break;
5015 case MVT::v16i8:
5016 case MVT::v8i8:
5017 SVT = MVT::nxv16i8;
5018 break;
5019 case MVT::v2i64:
5020 case MVT::v1i64:
5021 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5022 break;
5023 default:
5024 return false;
5025 }
5026
5027 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
5028 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
5029 return false;
5030
5031 if (N0->getOpcode() != AArch64ISD::VSHL ||
5032 N1->getOpcode() != AArch64ISD::VLSHR)
5033 return false;
5034
5035 if (N0->getOperand(Num: 0) != N1->getOperand(Num: 0))
5036 return false;
5037
5038 SDValue R1, R2;
5039 bool IsXOROperand = true;
5040 if (N1->getOperand(Num: 0)->getOpcode() != ISD::XOR) {
5041 IsXOROperand = false;
5042 } else {
5043 SDValue XOR = N0.getOperand(i: 0);
5044 R1 = XOR.getOperand(i: 0);
5045 R2 = XOR.getOperand(i: 1);
5046 }
5047
5048 unsigned HsAmt = N0.getConstantOperandVal(i: 1);
5049 unsigned ShAmt = N1.getConstantOperandVal(i: 1);
5050
5051 SDValue Imm = CurDAG->getTargetConstant(
5052 Val: ShAmt, DL, VT: N0.getOperand(i: 1).getValueType(), isOpaque: false);
5053
5054 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5055 if (ShAmt + HsAmt != VTSizeInBits)
5056 return false;
5057
5058 if (!IsXOROperand) {
5059 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
5060 SDNode *MOV =
5061 CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT: MVT::v2i64, Op1: Zero);
5062 SDValue MOVIV = SDValue(MOV, 0);
5063
5064 R1 = N1->getOperand(Num: 0);
5065 R2 = MOVIV;
5066 }
5067
5068 if (SVT != VT) {
5069 SDValue Undef =
5070 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: SVT), 0);
5071
5072 if (SVT.isScalableVector() && VT.is64BitVector()) {
5073 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5074
5075 SDValue UndefQ = SDValue(
5076 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: QVT), 0);
5077 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5078
5079 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5080 Op1: UndefQ, Op2: R1, Op3: DSub),
5081 0);
5082 if (R2.getValueType() == VT)
5083 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5084 Op1: UndefQ, Op2: R2, Op3: DSub),
5085 0);
5086 }
5087
5088 SDValue SubReg = CurDAG->getTargetConstant(
5089 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, VT: MVT::i32);
5090
5091 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT, Op1: Undef,
5092 Op2: R1, Op3: SubReg),
5093 0);
5094
5095 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5096 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT,
5097 Op1: Undef, Op2: R2, Op3: SubReg),
5098 0);
5099 }
5100
5101 SDValue Ops[] = {R1, R2, Imm};
5102 SDNode *XAR = nullptr;
5103
5104 if (SVT.isScalableVector()) {
5105 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5106 VT: SVT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5107 AArch64::XAR_ZZZI_D}))
5108 XAR = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: SVT, Ops);
5109 } else {
5110 XAR = CurDAG->getMachineNode(Opcode: AArch64::XAR, dl: DL, VT: SVT, Ops);
5111 }
5112
5113 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5114
5115 if (SVT != VT) {
5116 if (VT.is64BitVector() && SVT.isScalableVector()) {
5117 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5118
5119 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
5120 SDNode *Q = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT: QVT,
5121 Op1: SDValue(XAR, 0), Op2: ZSub);
5122
5123 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5124 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5125 Op1: SDValue(Q, 0), Op2: DSub);
5126 } else {
5127 SDValue SubReg = CurDAG->getTargetConstant(
5128 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5129 VT: MVT::i32);
5130 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5131 Op1: SDValue(XAR, 0), Op2: SubReg);
5132 }
5133 }
5134 ReplaceNode(F: N, T: XAR);
5135 return true;
5136}
5137
5138/// Returns a copy from WZR or XZR. This can be used during instruction
5139/// selection (it does not require any further selection/legalization).
5140static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
5141 assert(VT == MVT::i32 || VT == MVT::i64);
5142 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
5143 Reg: VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5144}
5145
5146void AArch64DAGToDAGISel::Select(SDNode *Node) {
5147 // If we have a custom node, we already have selected!
5148 if (Node->isMachineOpcode()) {
5149 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5150 Node->setNodeId(-1);
5151 return;
5152 }
5153
5154 // Few custom selection stuff.
5155 EVT VT = Node->getValueType(ResNo: 0);
5156
5157 switch (Node->getOpcode()) {
5158 default:
5159 break;
5160
5161 case ISD::ATOMIC_CMP_SWAP:
5162 if (SelectCMP_SWAP(N: Node))
5163 return;
5164 break;
5165
5166 case ISD::READ_REGISTER:
5167 case AArch64ISD::MRRS:
5168 if (tryReadRegister(N: Node))
5169 return;
5170 break;
5171
5172 case ISD::WRITE_REGISTER:
5173 case AArch64ISD::MSRR:
5174 if (tryWriteRegister(N: Node))
5175 return;
5176 break;
5177
5178 case ISD::LOAD: {
5179 // Try to select as an indexed load. Fall through to normal processing
5180 // if we can't.
5181 if (tryIndexedLoad(N: Node))
5182 return;
5183 break;
5184 }
5185
5186 case ISD::SRL:
5187 case ISD::AND:
5188 case ISD::SRA:
5189 case ISD::SIGN_EXTEND_INREG:
5190 if (tryBitfieldExtractOp(N: Node))
5191 return;
5192 if (tryBitfieldInsertInZeroOp(N: Node))
5193 return;
5194 [[fallthrough]];
5195 case ISD::ROTR:
5196 case ISD::SHL:
5197 break;
5198
5199 case ISD::OR:
5200 if (tryBitfieldInsertOp(N: Node))
5201 return;
5202 if (trySelectXAR(N: Node))
5203 return;
5204 break;
5205
5206 case ISD::EXTRACT_SUBVECTOR: {
5207 if (trySelectCastScalableToFixedLengthVector(N: Node))
5208 return;
5209 break;
5210 }
5211
5212 case ISD::INSERT_SUBVECTOR: {
5213 if (trySelectCastFixedLengthToScalableVector(N: Node))
5214 return;
5215 break;
5216 }
5217
5218 case AArch64ISD::CSEL:
5219 if (tryFoldCselToFMaxMin(N: Node))
5220 return;
5221 break;
5222
5223 case ISD::Constant: {
5224 // Materialize zero constants as copies from WZR/XZR. This allows
5225 // the coalescer to propagate these into other instructions.
5226 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Val: Node);
5227 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5228 ReplaceNode(F: Node, T: getZeroRegister(DAG&: *CurDAG, DL: SDLoc(Node), VT).getNode());
5229 return;
5230 }
5231 break;
5232 }
5233
5234 case ISD::FrameIndex: {
5235 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5236 int FI = cast<FrameIndexSDNode>(Val: Node)->getIndex();
5237 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
5238 const TargetLowering *TLI = getTargetLowering();
5239 SDValue TFI = CurDAG->getTargetFrameIndex(
5240 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
5241 SDLoc DL(Node);
5242 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32),
5243 CurDAG->getTargetConstant(Val: Shifter, DL, VT: MVT::i32) };
5244 CurDAG->SelectNodeTo(N: Node, MachineOpc: AArch64::ADDXri, VT: MVT::i64, Ops);
5245 return;
5246 }
5247 case ISD::INTRINSIC_W_CHAIN: {
5248 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5249 switch (IntNo) {
5250 default:
5251 break;
5252 case Intrinsic::aarch64_gcsss: {
5253 SDLoc DL(Node);
5254 SDValue Chain = Node->getOperand(Num: 0);
5255 SDValue Val = Node->getOperand(Num: 2);
5256 SDValue Zero = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::XZR, VT: MVT::i64);
5257 SDNode *SS1 =
5258 CurDAG->getMachineNode(Opcode: AArch64::GCSSS1, dl: DL, VT: MVT::Other, Op1: Val, Op2: Chain);
5259 SDNode *SS2 = CurDAG->getMachineNode(Opcode: AArch64::GCSSS2, dl: DL, VT1: MVT::i64,
5260 VT2: MVT::Other, Op1: Zero, Op2: SDValue(SS1, 0));
5261 ReplaceNode(F: Node, T: SS2);
5262 return;
5263 }
5264 case Intrinsic::aarch64_ldaxp:
5265 case Intrinsic::aarch64_ldxp: {
5266 unsigned Op =
5267 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5268 SDValue MemAddr = Node->getOperand(Num: 2);
5269 SDLoc DL(Node);
5270 SDValue Chain = Node->getOperand(Num: 0);
5271
5272 SDNode *Ld = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i64, VT2: MVT::i64,
5273 VT3: MVT::Other, Op1: MemAddr, Op2: Chain);
5274
5275 // Transfer memoperands.
5276 MachineMemOperand *MemOp =
5277 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5278 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
5279 ReplaceNode(F: Node, T: Ld);
5280 return;
5281 }
5282 case Intrinsic::aarch64_stlxp:
5283 case Intrinsic::aarch64_stxp: {
5284 unsigned Op =
5285 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5286 SDLoc DL(Node);
5287 SDValue Chain = Node->getOperand(Num: 0);
5288 SDValue ValLo = Node->getOperand(Num: 2);
5289 SDValue ValHi = Node->getOperand(Num: 3);
5290 SDValue MemAddr = Node->getOperand(Num: 4);
5291
5292 // Place arguments in the right order.
5293 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5294
5295 SDNode *St = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
5296 // Transfer memoperands.
5297 MachineMemOperand *MemOp =
5298 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5299 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
5300
5301 ReplaceNode(F: Node, T: St);
5302 return;
5303 }
5304 case Intrinsic::aarch64_neon_ld1x2:
5305 if (VT == MVT::v8i8) {
5306 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b, SubRegIdx: AArch64::dsub0);
5307 return;
5308 } else if (VT == MVT::v16i8) {
5309 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b, SubRegIdx: AArch64::qsub0);
5310 return;
5311 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5312 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h, SubRegIdx: AArch64::dsub0);
5313 return;
5314 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5315 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h, SubRegIdx: AArch64::qsub0);
5316 return;
5317 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5318 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s, SubRegIdx: AArch64::dsub0);
5319 return;
5320 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5321 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s, SubRegIdx: AArch64::qsub0);
5322 return;
5323 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5324 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5325 return;
5326 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5327 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d, SubRegIdx: AArch64::qsub0);
5328 return;
5329 }
5330 break;
5331 case Intrinsic::aarch64_neon_ld1x3:
5332 if (VT == MVT::v8i8) {
5333 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b, SubRegIdx: AArch64::dsub0);
5334 return;
5335 } else if (VT == MVT::v16i8) {
5336 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b, SubRegIdx: AArch64::qsub0);
5337 return;
5338 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5339 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h, SubRegIdx: AArch64::dsub0);
5340 return;
5341 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5342 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h, SubRegIdx: AArch64::qsub0);
5343 return;
5344 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5345 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s, SubRegIdx: AArch64::dsub0);
5346 return;
5347 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5348 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s, SubRegIdx: AArch64::qsub0);
5349 return;
5350 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5351 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5352 return;
5353 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5354 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d, SubRegIdx: AArch64::qsub0);
5355 return;
5356 }
5357 break;
5358 case Intrinsic::aarch64_neon_ld1x4:
5359 if (VT == MVT::v8i8) {
5360 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b, SubRegIdx: AArch64::dsub0);
5361 return;
5362 } else if (VT == MVT::v16i8) {
5363 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b, SubRegIdx: AArch64::qsub0);
5364 return;
5365 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5366 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h, SubRegIdx: AArch64::dsub0);
5367 return;
5368 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5369 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h, SubRegIdx: AArch64::qsub0);
5370 return;
5371 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5372 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s, SubRegIdx: AArch64::dsub0);
5373 return;
5374 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5375 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s, SubRegIdx: AArch64::qsub0);
5376 return;
5377 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5378 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5379 return;
5380 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5381 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d, SubRegIdx: AArch64::qsub0);
5382 return;
5383 }
5384 break;
5385 case Intrinsic::aarch64_neon_ld2:
5386 if (VT == MVT::v8i8) {
5387 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b, SubRegIdx: AArch64::dsub0);
5388 return;
5389 } else if (VT == MVT::v16i8) {
5390 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b, SubRegIdx: AArch64::qsub0);
5391 return;
5392 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5393 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h, SubRegIdx: AArch64::dsub0);
5394 return;
5395 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5396 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h, SubRegIdx: AArch64::qsub0);
5397 return;
5398 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5399 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s, SubRegIdx: AArch64::dsub0);
5400 return;
5401 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5402 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s, SubRegIdx: AArch64::qsub0);
5403 return;
5404 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5405 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5406 return;
5407 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5408 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d, SubRegIdx: AArch64::qsub0);
5409 return;
5410 }
5411 break;
5412 case Intrinsic::aarch64_neon_ld3:
5413 if (VT == MVT::v8i8) {
5414 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b, SubRegIdx: AArch64::dsub0);
5415 return;
5416 } else if (VT == MVT::v16i8) {
5417 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b, SubRegIdx: AArch64::qsub0);
5418 return;
5419 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5420 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h, SubRegIdx: AArch64::dsub0);
5421 return;
5422 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5423 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h, SubRegIdx: AArch64::qsub0);
5424 return;
5425 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5426 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s, SubRegIdx: AArch64::dsub0);
5427 return;
5428 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5429 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s, SubRegIdx: AArch64::qsub0);
5430 return;
5431 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5432 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5433 return;
5434 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5435 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d, SubRegIdx: AArch64::qsub0);
5436 return;
5437 }
5438 break;
5439 case Intrinsic::aarch64_neon_ld4:
5440 if (VT == MVT::v8i8) {
5441 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b, SubRegIdx: AArch64::dsub0);
5442 return;
5443 } else if (VT == MVT::v16i8) {
5444 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b, SubRegIdx: AArch64::qsub0);
5445 return;
5446 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5447 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h, SubRegIdx: AArch64::dsub0);
5448 return;
5449 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5450 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h, SubRegIdx: AArch64::qsub0);
5451 return;
5452 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5453 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s, SubRegIdx: AArch64::dsub0);
5454 return;
5455 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5456 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s, SubRegIdx: AArch64::qsub0);
5457 return;
5458 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5459 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5460 return;
5461 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5462 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d, SubRegIdx: AArch64::qsub0);
5463 return;
5464 }
5465 break;
5466 case Intrinsic::aarch64_neon_ld2r:
5467 if (VT == MVT::v8i8) {
5468 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b, SubRegIdx: AArch64::dsub0);
5469 return;
5470 } else if (VT == MVT::v16i8) {
5471 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b, SubRegIdx: AArch64::qsub0);
5472 return;
5473 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5474 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h, SubRegIdx: AArch64::dsub0);
5475 return;
5476 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5477 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h, SubRegIdx: AArch64::qsub0);
5478 return;
5479 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5480 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s, SubRegIdx: AArch64::dsub0);
5481 return;
5482 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5483 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s, SubRegIdx: AArch64::qsub0);
5484 return;
5485 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5486 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d, SubRegIdx: AArch64::dsub0);
5487 return;
5488 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5489 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d, SubRegIdx: AArch64::qsub0);
5490 return;
5491 }
5492 break;
5493 case Intrinsic::aarch64_neon_ld3r:
5494 if (VT == MVT::v8i8) {
5495 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b, SubRegIdx: AArch64::dsub0);
5496 return;
5497 } else if (VT == MVT::v16i8) {
5498 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b, SubRegIdx: AArch64::qsub0);
5499 return;
5500 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5501 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h, SubRegIdx: AArch64::dsub0);
5502 return;
5503 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5504 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h, SubRegIdx: AArch64::qsub0);
5505 return;
5506 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5507 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s, SubRegIdx: AArch64::dsub0);
5508 return;
5509 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5510 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s, SubRegIdx: AArch64::qsub0);
5511 return;
5512 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5513 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d, SubRegIdx: AArch64::dsub0);
5514 return;
5515 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5516 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d, SubRegIdx: AArch64::qsub0);
5517 return;
5518 }
5519 break;
5520 case Intrinsic::aarch64_neon_ld4r:
5521 if (VT == MVT::v8i8) {
5522 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b, SubRegIdx: AArch64::dsub0);
5523 return;
5524 } else if (VT == MVT::v16i8) {
5525 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b, SubRegIdx: AArch64::qsub0);
5526 return;
5527 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5528 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h, SubRegIdx: AArch64::dsub0);
5529 return;
5530 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5531 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h, SubRegIdx: AArch64::qsub0);
5532 return;
5533 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5534 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s, SubRegIdx: AArch64::dsub0);
5535 return;
5536 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5537 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s, SubRegIdx: AArch64::qsub0);
5538 return;
5539 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5540 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d, SubRegIdx: AArch64::dsub0);
5541 return;
5542 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5543 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d, SubRegIdx: AArch64::qsub0);
5544 return;
5545 }
5546 break;
5547 case Intrinsic::aarch64_neon_ld2lane:
5548 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5549 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8);
5550 return;
5551 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5552 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5553 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16);
5554 return;
5555 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5556 VT == MVT::v2f32) {
5557 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32);
5558 return;
5559 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5560 VT == MVT::v1f64) {
5561 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64);
5562 return;
5563 }
5564 break;
5565 case Intrinsic::aarch64_neon_ld3lane:
5566 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5567 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8);
5568 return;
5569 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5570 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5571 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16);
5572 return;
5573 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5574 VT == MVT::v2f32) {
5575 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32);
5576 return;
5577 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5578 VT == MVT::v1f64) {
5579 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64);
5580 return;
5581 }
5582 break;
5583 case Intrinsic::aarch64_neon_ld4lane:
5584 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5585 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8);
5586 return;
5587 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5588 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5589 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16);
5590 return;
5591 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5592 VT == MVT::v2f32) {
5593 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32);
5594 return;
5595 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5596 VT == MVT::v1f64) {
5597 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64);
5598 return;
5599 }
5600 break;
5601 case Intrinsic::aarch64_ld64b:
5602 SelectLoad(N: Node, NumVecs: 8, Opc: AArch64::LD64B, SubRegIdx: AArch64::x8sub_0);
5603 return;
5604 case Intrinsic::aarch64_sve_ld2q_sret: {
5605 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 4, Opc_ri: AArch64::LD2Q_IMM, Opc_rr: AArch64::LD2Q, IsIntr: true);
5606 return;
5607 }
5608 case Intrinsic::aarch64_sve_ld3q_sret: {
5609 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 4, Opc_ri: AArch64::LD3Q_IMM, Opc_rr: AArch64::LD3Q, IsIntr: true);
5610 return;
5611 }
5612 case Intrinsic::aarch64_sve_ld4q_sret: {
5613 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 4, Opc_ri: AArch64::LD4Q_IMM, Opc_rr: AArch64::LD4Q, IsIntr: true);
5614 return;
5615 }
5616 case Intrinsic::aarch64_sve_ld2_sret: {
5617 if (VT == MVT::nxv16i8) {
5618 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD2B_IMM, Opc_rr: AArch64::LD2B,
5619 IsIntr: true);
5620 return;
5621 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5622 VT == MVT::nxv8bf16) {
5623 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD2H_IMM, Opc_rr: AArch64::LD2H,
5624 IsIntr: true);
5625 return;
5626 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5627 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD2W_IMM, Opc_rr: AArch64::LD2W,
5628 IsIntr: true);
5629 return;
5630 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5631 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD2D_IMM, Opc_rr: AArch64::LD2D,
5632 IsIntr: true);
5633 return;
5634 }
5635 break;
5636 }
5637 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5638 if (VT == MVT::nxv16i8) {
5639 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5640 SelectContiguousMultiVectorLoad(
5641 N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_2Z_PSEUDO);
5642 else if (Subtarget->hasSVE2p1())
5643 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM,
5644 Opc_rr: AArch64::LD1B_2Z);
5645 else
5646 break;
5647 return;
5648 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5649 VT == MVT::nxv8bf16) {
5650 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5651 SelectContiguousMultiVectorLoad(
5652 N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_2Z_PSEUDO);
5653 else if (Subtarget->hasSVE2p1())
5654 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM,
5655 Opc_rr: AArch64::LD1H_2Z);
5656 else
5657 break;
5658 return;
5659 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5660 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5661 SelectContiguousMultiVectorLoad(
5662 N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_2Z_PSEUDO);
5663 else if (Subtarget->hasSVE2p1())
5664 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM,
5665 Opc_rr: AArch64::LD1W_2Z);
5666 else
5667 break;
5668 return;
5669 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5670 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5671 SelectContiguousMultiVectorLoad(
5672 N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_2Z_PSEUDO);
5673 else if (Subtarget->hasSVE2p1())
5674 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM,
5675 Opc_rr: AArch64::LD1D_2Z);
5676 else
5677 break;
5678 return;
5679 }
5680 break;
5681 }
5682 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5683 if (VT == MVT::nxv16i8) {
5684 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5685 SelectContiguousMultiVectorLoad(
5686 N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_4Z_PSEUDO);
5687 else if (Subtarget->hasSVE2p1())
5688 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM,
5689 Opc_rr: AArch64::LD1B_4Z);
5690 else
5691 break;
5692 return;
5693 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5694 VT == MVT::nxv8bf16) {
5695 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5696 SelectContiguousMultiVectorLoad(
5697 N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_4Z_PSEUDO);
5698 else if (Subtarget->hasSVE2p1())
5699 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM,
5700 Opc_rr: AArch64::LD1H_4Z);
5701 else
5702 break;
5703 return;
5704 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5705 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5706 SelectContiguousMultiVectorLoad(
5707 N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_4Z_PSEUDO);
5708 else if (Subtarget->hasSVE2p1())
5709 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM,
5710 Opc_rr: AArch64::LD1W_4Z);
5711 else
5712 break;
5713 return;
5714 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5715 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5716 SelectContiguousMultiVectorLoad(
5717 N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_4Z_PSEUDO);
5718 else if (Subtarget->hasSVE2p1())
5719 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM,
5720 Opc_rr: AArch64::LD1D_4Z);
5721 else
5722 break;
5723 return;
5724 }
5725 break;
5726 }
5727 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5728 if (VT == MVT::nxv16i8) {
5729 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5730 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0,
5731 Opc_ri: AArch64::LDNT1B_2Z_IMM_PSEUDO,
5732 Opc_rr: AArch64::LDNT1B_2Z_PSEUDO);
5733 else if (Subtarget->hasSVE2p1())
5734 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LDNT1B_2Z_IMM,
5735 Opc_rr: AArch64::LDNT1B_2Z);
5736 else
5737 break;
5738 return;
5739 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5740 VT == MVT::nxv8bf16) {
5741 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5742 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1,
5743 Opc_ri: AArch64::LDNT1H_2Z_IMM_PSEUDO,
5744 Opc_rr: AArch64::LDNT1H_2Z_PSEUDO);
5745 else if (Subtarget->hasSVE2p1())
5746 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LDNT1H_2Z_IMM,
5747 Opc_rr: AArch64::LDNT1H_2Z);
5748 else
5749 break;
5750 return;
5751 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5752 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5753 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2,
5754 Opc_ri: AArch64::LDNT1W_2Z_IMM_PSEUDO,
5755 Opc_rr: AArch64::LDNT1W_2Z_PSEUDO);
5756 else if (Subtarget->hasSVE2p1())
5757 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LDNT1W_2Z_IMM,
5758 Opc_rr: AArch64::LDNT1W_2Z);
5759 else
5760 break;
5761 return;
5762 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5763 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5764 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3,
5765 Opc_ri: AArch64::LDNT1D_2Z_IMM_PSEUDO,
5766 Opc_rr: AArch64::LDNT1D_2Z_PSEUDO);
5767 else if (Subtarget->hasSVE2p1())
5768 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LDNT1D_2Z_IMM,
5769 Opc_rr: AArch64::LDNT1D_2Z);
5770 else
5771 break;
5772 return;
5773 }
5774 break;
5775 }
5776 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5777 if (VT == MVT::nxv16i8) {
5778 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5779 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0,
5780 Opc_ri: AArch64::LDNT1B_4Z_IMM_PSEUDO,
5781 Opc_rr: AArch64::LDNT1B_4Z_PSEUDO);
5782 else if (Subtarget->hasSVE2p1())
5783 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LDNT1B_4Z_IMM,
5784 Opc_rr: AArch64::LDNT1B_4Z);
5785 else
5786 break;
5787 return;
5788 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5789 VT == MVT::nxv8bf16) {
5790 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5791 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1,
5792 Opc_ri: AArch64::LDNT1H_4Z_IMM_PSEUDO,
5793 Opc_rr: AArch64::LDNT1H_4Z_PSEUDO);
5794 else if (Subtarget->hasSVE2p1())
5795 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LDNT1H_4Z_IMM,
5796 Opc_rr: AArch64::LDNT1H_4Z);
5797 else
5798 break;
5799 return;
5800 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5801 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5802 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2,
5803 Opc_ri: AArch64::LDNT1W_4Z_IMM_PSEUDO,
5804 Opc_rr: AArch64::LDNT1W_4Z_PSEUDO);
5805 else if (Subtarget->hasSVE2p1())
5806 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LDNT1W_4Z_IMM,
5807 Opc_rr: AArch64::LDNT1W_4Z);
5808 else
5809 break;
5810 return;
5811 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5812 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5813 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3,
5814 Opc_ri: AArch64::LDNT1D_4Z_IMM_PSEUDO,
5815 Opc_rr: AArch64::LDNT1D_4Z_PSEUDO);
5816 else if (Subtarget->hasSVE2p1())
5817 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LDNT1D_4Z_IMM,
5818 Opc_rr: AArch64::LDNT1D_4Z);
5819 else
5820 break;
5821 return;
5822 }
5823 break;
5824 }
5825 case Intrinsic::aarch64_sve_ld3_sret: {
5826 if (VT == MVT::nxv16i8) {
5827 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 0, Opc_ri: AArch64::LD3B_IMM, Opc_rr: AArch64::LD3B,
5828 IsIntr: true);
5829 return;
5830 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5831 VT == MVT::nxv8bf16) {
5832 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 1, Opc_ri: AArch64::LD3H_IMM, Opc_rr: AArch64::LD3H,
5833 IsIntr: true);
5834 return;
5835 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5836 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 2, Opc_ri: AArch64::LD3W_IMM, Opc_rr: AArch64::LD3W,
5837 IsIntr: true);
5838 return;
5839 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5840 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 3, Opc_ri: AArch64::LD3D_IMM, Opc_rr: AArch64::LD3D,
5841 IsIntr: true);
5842 return;
5843 }
5844 break;
5845 }
5846 case Intrinsic::aarch64_sve_ld4_sret: {
5847 if (VT == MVT::nxv16i8) {
5848 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD4B_IMM, Opc_rr: AArch64::LD4B,
5849 IsIntr: true);
5850 return;
5851 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5852 VT == MVT::nxv8bf16) {
5853 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD4H_IMM, Opc_rr: AArch64::LD4H,
5854 IsIntr: true);
5855 return;
5856 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5857 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD4W_IMM, Opc_rr: AArch64::LD4W,
5858 IsIntr: true);
5859 return;
5860 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5861 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD4D_IMM, Opc_rr: AArch64::LD4D,
5862 IsIntr: true);
5863 return;
5864 }
5865 break;
5866 }
5867 case Intrinsic::aarch64_sme_read_hor_vg2: {
5868 if (VT == MVT::nxv16i8) {
5869 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5870 Op: AArch64::MOVA_2ZMXI_H_B);
5871 return;
5872 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5873 VT == MVT::nxv8bf16) {
5874 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5875 Op: AArch64::MOVA_2ZMXI_H_H);
5876 return;
5877 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5878 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5879 Op: AArch64::MOVA_2ZMXI_H_S);
5880 return;
5881 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5882 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5883 Op: AArch64::MOVA_2ZMXI_H_D);
5884 return;
5885 }
5886 break;
5887 }
5888 case Intrinsic::aarch64_sme_read_ver_vg2: {
5889 if (VT == MVT::nxv16i8) {
5890 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5891 Op: AArch64::MOVA_2ZMXI_V_B);
5892 return;
5893 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5894 VT == MVT::nxv8bf16) {
5895 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5896 Op: AArch64::MOVA_2ZMXI_V_H);
5897 return;
5898 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5899 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5900 Op: AArch64::MOVA_2ZMXI_V_S);
5901 return;
5902 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5903 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5904 Op: AArch64::MOVA_2ZMXI_V_D);
5905 return;
5906 }
5907 break;
5908 }
5909 case Intrinsic::aarch64_sme_read_hor_vg4: {
5910 if (VT == MVT::nxv16i8) {
5911 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5912 Op: AArch64::MOVA_4ZMXI_H_B);
5913 return;
5914 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5915 VT == MVT::nxv8bf16) {
5916 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5917 Op: AArch64::MOVA_4ZMXI_H_H);
5918 return;
5919 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5920 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5921 Op: AArch64::MOVA_4ZMXI_H_S);
5922 return;
5923 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5924 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5925 Op: AArch64::MOVA_4ZMXI_H_D);
5926 return;
5927 }
5928 break;
5929 }
5930 case Intrinsic::aarch64_sme_read_ver_vg4: {
5931 if (VT == MVT::nxv16i8) {
5932 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5933 Op: AArch64::MOVA_4ZMXI_V_B);
5934 return;
5935 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5936 VT == MVT::nxv8bf16) {
5937 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5938 Op: AArch64::MOVA_4ZMXI_V_H);
5939 return;
5940 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5941 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5942 Op: AArch64::MOVA_4ZMXI_V_S);
5943 return;
5944 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5945 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5946 Op: AArch64::MOVA_4ZMXI_V_D);
5947 return;
5948 }
5949 break;
5950 }
5951 case Intrinsic::aarch64_sme_read_vg1x2: {
5952 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 2, BaseReg: AArch64::ZA,
5953 Op: AArch64::MOVA_VG2_2ZMXI);
5954 return;
5955 }
5956 case Intrinsic::aarch64_sme_read_vg1x4: {
5957 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 4, BaseReg: AArch64::ZA,
5958 Op: AArch64::MOVA_VG4_4ZMXI);
5959 return;
5960 }
5961 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5962 if (VT == MVT::nxv16i8) {
5963 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_B_PSEUDO, MaxIdx: 14, Scale: 2);
5964 return;
5965 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5966 VT == MVT::nxv8bf16) {
5967 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_H_PSEUDO, MaxIdx: 6, Scale: 2);
5968 return;
5969 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5970 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_S_PSEUDO, MaxIdx: 2, Scale: 2);
5971 return;
5972 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5973 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 2);
5974 return;
5975 }
5976 break;
5977 }
5978 case Intrinsic::aarch64_sme_readz_vert_x2: {
5979 if (VT == MVT::nxv16i8) {
5980 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_B_PSEUDO, MaxIdx: 14, Scale: 2);
5981 return;
5982 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5983 VT == MVT::nxv8bf16) {
5984 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_H_PSEUDO, MaxIdx: 6, Scale: 2);
5985 return;
5986 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5987 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_S_PSEUDO, MaxIdx: 2, Scale: 2);
5988 return;
5989 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5990 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 2);
5991 return;
5992 }
5993 break;
5994 }
5995 case Intrinsic::aarch64_sme_readz_horiz_x4: {
5996 if (VT == MVT::nxv16i8) {
5997 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_B_PSEUDO, MaxIdx: 12, Scale: 4);
5998 return;
5999 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6000 VT == MVT::nxv8bf16) {
6001 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_H_PSEUDO, MaxIdx: 4, Scale: 4);
6002 return;
6003 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6004 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_S_PSEUDO, MaxIdx: 0, Scale: 4);
6005 return;
6006 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6007 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 4);
6008 return;
6009 }
6010 break;
6011 }
6012 case Intrinsic::aarch64_sme_readz_vert_x4: {
6013 if (VT == MVT::nxv16i8) {
6014 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_B_PSEUDO, MaxIdx: 12, Scale: 4);
6015 return;
6016 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6017 VT == MVT::nxv8bf16) {
6018 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_H_PSEUDO, MaxIdx: 4, Scale: 4);
6019 return;
6020 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6021 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_S_PSEUDO, MaxIdx: 0, Scale: 4);
6022 return;
6023 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6024 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 4);
6025 return;
6026 }
6027 break;
6028 }
6029 case Intrinsic::aarch64_sme_readz_x2: {
6030 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6031 BaseReg: AArch64::ZA);
6032 return;
6033 }
6034 case Intrinsic::aarch64_sme_readz_x4: {
6035 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6036 BaseReg: AArch64::ZA);
6037 return;
6038 }
6039 case Intrinsic::swift_async_context_addr: {
6040 SDLoc DL(Node);
6041 SDValue Chain = Node->getOperand(Num: 0);
6042 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::FP, VT: MVT::i64);
6043 SDValue Res = SDValue(
6044 CurDAG->getMachineNode(Opcode: AArch64::SUBXri, dl: DL, VT: MVT::i64, Op1: CopyFP,
6045 Op2: CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32),
6046 Op3: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32)),
6047 0);
6048 ReplaceUses(F: SDValue(Node, 0), T: Res);
6049 ReplaceUses(F: SDValue(Node, 1), T: CopyFP.getValue(R: 1));
6050 CurDAG->RemoveDeadNode(N: Node);
6051
6052 auto &MF = CurDAG->getMachineFunction();
6053 MF.getFrameInfo().setFrameAddressIsTaken(true);
6054 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6055 return;
6056 }
6057 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6058 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6059 VT: Node->getValueType(ResNo: 0),
6060 Opcodes: {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6061 AArch64::LUTI2_4ZTZI_S}))
6062 // Second Immediate must be <= 3:
6063 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 3);
6064 return;
6065 }
6066 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6067 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6068 VT: Node->getValueType(ResNo: 0),
6069 Opcodes: {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6070 // Second Immediate must be <= 1:
6071 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 1);
6072 return;
6073 }
6074 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6075 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6076 VT: Node->getValueType(ResNo: 0),
6077 Opcodes: {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6078 AArch64::LUTI2_2ZTZI_S}))
6079 // Second Immediate must be <= 7:
6080 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 7);
6081 return;
6082 }
6083 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6084 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6085 VT: Node->getValueType(ResNo: 0),
6086 Opcodes: {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6087 AArch64::LUTI4_2ZTZI_S}))
6088 // Second Immediate must be <= 3:
6089 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 3);
6090 return;
6091 }
6092 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6093 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI4_4ZZT2Z, NumInVecs: 2);
6094 return;
6095 }
6096 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6097 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI6_4ZT3Z, NumInVecs: 3);
6098 return;
6099 }
6100 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6101 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6102 VT: Node->getValueType(ResNo: 0),
6103 Opcodes: {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6104 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6105 return;
6106 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6107 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6108 VT: Node->getValueType(ResNo: 0),
6109 Opcodes: {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6110 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6111 return;
6112 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6113 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6114 VT: Node->getValueType(ResNo: 0),
6115 Opcodes: {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6116 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6117 return;
6118 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6119 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6120 VT: Node->getValueType(ResNo: 0),
6121 Opcodes: {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6122 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6123 return;
6124 case Intrinsic::ptrauth_resign_load_relative:
6125 SelectPtrauthResign(N: Node);
6126 return;
6127 }
6128 } break;
6129 case ISD::INTRINSIC_WO_CHAIN: {
6130 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
6131 switch (IntNo) {
6132 default:
6133 break;
6134 case Intrinsic::aarch64_tagp:
6135 SelectTagP(N: Node);
6136 return;
6137
6138 case Intrinsic::ptrauth_auth:
6139 SelectPtrauthAuth(N: Node);
6140 return;
6141
6142 case Intrinsic::ptrauth_resign:
6143 SelectPtrauthResign(N: Node);
6144 return;
6145
6146 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6147 SelectPtrauthResignWithPC(N: Node);
6148 return;
6149
6150 case Intrinsic::aarch64_neon_tbl2:
6151 SelectTable(N: Node, NumVecs: 2,
6152 Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6153 isExt: false);
6154 return;
6155 case Intrinsic::aarch64_neon_tbl3:
6156 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6157 : AArch64::TBLv16i8Three,
6158 isExt: false);
6159 return;
6160 case Intrinsic::aarch64_neon_tbl4:
6161 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6162 : AArch64::TBLv16i8Four,
6163 isExt: false);
6164 return;
6165 case Intrinsic::aarch64_neon_tbx2:
6166 SelectTable(N: Node, NumVecs: 2,
6167 Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6168 isExt: true);
6169 return;
6170 case Intrinsic::aarch64_neon_tbx3:
6171 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6172 : AArch64::TBXv16i8Three,
6173 isExt: true);
6174 return;
6175 case Intrinsic::aarch64_neon_tbx4:
6176 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6177 : AArch64::TBXv16i8Four,
6178 isExt: true);
6179 return;
6180 case Intrinsic::aarch64_sve_srshl_single_x2:
6181 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6182 VT: Node->getValueType(ResNo: 0),
6183 Opcodes: {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6184 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6185 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6186 return;
6187 case Intrinsic::aarch64_sve_srshl_single_x4:
6188 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6189 VT: Node->getValueType(ResNo: 0),
6190 Opcodes: {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6191 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6192 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6193 return;
6194 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6195 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 2);
6196 return;
6197 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6198 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 3);
6199 return;
6200 case Intrinsic::aarch64_sve_urshl_single_x2:
6201 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6202 VT: Node->getValueType(ResNo: 0),
6203 Opcodes: {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6204 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6205 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6206 return;
6207 case Intrinsic::aarch64_sve_urshl_single_x4:
6208 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6209 VT: Node->getValueType(ResNo: 0),
6210 Opcodes: {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6211 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6212 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6213 return;
6214 case Intrinsic::aarch64_sve_srshl_x2:
6215 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6216 VT: Node->getValueType(ResNo: 0),
6217 Opcodes: {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6218 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6219 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6220 return;
6221 case Intrinsic::aarch64_sve_srshl_x4:
6222 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6223 VT: Node->getValueType(ResNo: 0),
6224 Opcodes: {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6225 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6226 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6227 return;
6228 case Intrinsic::aarch64_sve_urshl_x2:
6229 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6230 VT: Node->getValueType(ResNo: 0),
6231 Opcodes: {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6232 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6233 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6234 return;
6235 case Intrinsic::aarch64_sve_urshl_x4:
6236 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6237 VT: Node->getValueType(ResNo: 0),
6238 Opcodes: {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6239 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6240 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6241 return;
6242 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6243 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6244 VT: Node->getValueType(ResNo: 0),
6245 Opcodes: {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6246 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6247 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6248 return;
6249 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6250 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6251 VT: Node->getValueType(ResNo: 0),
6252 Opcodes: {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6253 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6254 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6255 return;
6256 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6257 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6258 VT: Node->getValueType(ResNo: 0),
6259 Opcodes: {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6260 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6261 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6262 return;
6263 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6264 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6265 VT: Node->getValueType(ResNo: 0),
6266 Opcodes: {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6267 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6268 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6269 return;
6270 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6271 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6272 VT: Node->getValueType(ResNo: 0),
6273 Opcodes: {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6274 AArch64::FSCALE_2ZZ_D}))
6275 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6276 return;
6277 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6278 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6279 VT: Node->getValueType(ResNo: 0),
6280 Opcodes: {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6281 AArch64::FSCALE_4ZZ_D}))
6282 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6283 return;
6284 case Intrinsic::aarch64_sme_fp8_scale_x2:
6285 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6286 VT: Node->getValueType(ResNo: 0),
6287 Opcodes: {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6288 AArch64::FSCALE_2Z2Z_D}))
6289 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6290 return;
6291 case Intrinsic::aarch64_sme_fp8_scale_x4:
6292 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6293 VT: Node->getValueType(ResNo: 0),
6294 Opcodes: {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6295 AArch64::FSCALE_4Z4Z_D}))
6296 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6297 return;
6298 case Intrinsic::aarch64_sve_whilege_x2:
6299 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6300 VT: Node->getValueType(ResNo: 0),
6301 Opcodes: {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6302 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6303 SelectWhilePair(N: Node, Opc: Op);
6304 return;
6305 case Intrinsic::aarch64_sve_whilegt_x2:
6306 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6307 VT: Node->getValueType(ResNo: 0),
6308 Opcodes: {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6309 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6310 SelectWhilePair(N: Node, Opc: Op);
6311 return;
6312 case Intrinsic::aarch64_sve_whilehi_x2:
6313 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6314 VT: Node->getValueType(ResNo: 0),
6315 Opcodes: {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6316 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6317 SelectWhilePair(N: Node, Opc: Op);
6318 return;
6319 case Intrinsic::aarch64_sve_whilehs_x2:
6320 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6321 VT: Node->getValueType(ResNo: 0),
6322 Opcodes: {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6323 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6324 SelectWhilePair(N: Node, Opc: Op);
6325 return;
6326 case Intrinsic::aarch64_sve_whilele_x2:
6327 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6328 VT: Node->getValueType(ResNo: 0),
6329 Opcodes: {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6330 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6331 SelectWhilePair(N: Node, Opc: Op);
6332 return;
6333 case Intrinsic::aarch64_sve_whilelo_x2:
6334 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6335 VT: Node->getValueType(ResNo: 0),
6336 Opcodes: {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6337 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6338 SelectWhilePair(N: Node, Opc: Op);
6339 return;
6340 case Intrinsic::aarch64_sve_whilels_x2:
6341 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6342 VT: Node->getValueType(ResNo: 0),
6343 Opcodes: {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6344 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6345 SelectWhilePair(N: Node, Opc: Op);
6346 return;
6347 case Intrinsic::aarch64_sve_whilelt_x2:
6348 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6349 VT: Node->getValueType(ResNo: 0),
6350 Opcodes: {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6351 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6352 SelectWhilePair(N: Node, Opc: Op);
6353 return;
6354 case Intrinsic::aarch64_sve_smax_single_x2:
6355 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6356 VT: Node->getValueType(ResNo: 0),
6357 Opcodes: {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6358 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6359 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6360 return;
6361 case Intrinsic::aarch64_sve_umax_single_x2:
6362 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6363 VT: Node->getValueType(ResNo: 0),
6364 Opcodes: {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6365 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6366 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6367 return;
6368 case Intrinsic::aarch64_sve_fmax_single_x2:
6369 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6370 VT: Node->getValueType(ResNo: 0),
6371 Opcodes: {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6372 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6373 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6374 return;
6375 case Intrinsic::aarch64_sve_smax_single_x4:
6376 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6377 VT: Node->getValueType(ResNo: 0),
6378 Opcodes: {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6379 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6380 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6381 return;
6382 case Intrinsic::aarch64_sve_umax_single_x4:
6383 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6384 VT: Node->getValueType(ResNo: 0),
6385 Opcodes: {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6386 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6387 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6388 return;
6389 case Intrinsic::aarch64_sve_fmax_single_x4:
6390 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6391 VT: Node->getValueType(ResNo: 0),
6392 Opcodes: {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6393 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6394 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6395 return;
6396 case Intrinsic::aarch64_sve_smin_single_x2:
6397 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6398 VT: Node->getValueType(ResNo: 0),
6399 Opcodes: {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6400 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6401 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6402 return;
6403 case Intrinsic::aarch64_sve_umin_single_x2:
6404 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6405 VT: Node->getValueType(ResNo: 0),
6406 Opcodes: {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6407 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6408 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6409 return;
6410 case Intrinsic::aarch64_sve_fmin_single_x2:
6411 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6412 VT: Node->getValueType(ResNo: 0),
6413 Opcodes: {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6414 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6415 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6416 return;
6417 case Intrinsic::aarch64_sve_smin_single_x4:
6418 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6419 VT: Node->getValueType(ResNo: 0),
6420 Opcodes: {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6421 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6422 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6423 return;
6424 case Intrinsic::aarch64_sve_umin_single_x4:
6425 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6426 VT: Node->getValueType(ResNo: 0),
6427 Opcodes: {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6428 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6429 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6430 return;
6431 case Intrinsic::aarch64_sve_fmin_single_x4:
6432 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6433 VT: Node->getValueType(ResNo: 0),
6434 Opcodes: {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6435 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6436 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6437 return;
6438 case Intrinsic::aarch64_sve_smax_x2:
6439 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6440 VT: Node->getValueType(ResNo: 0),
6441 Opcodes: {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6442 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6443 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6444 return;
6445 case Intrinsic::aarch64_sve_umax_x2:
6446 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6447 VT: Node->getValueType(ResNo: 0),
6448 Opcodes: {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6449 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6450 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6451 return;
6452 case Intrinsic::aarch64_sve_fmax_x2:
6453 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6454 VT: Node->getValueType(ResNo: 0),
6455 Opcodes: {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6456 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6457 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6458 return;
6459 case Intrinsic::aarch64_sve_smax_x4:
6460 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6461 VT: Node->getValueType(ResNo: 0),
6462 Opcodes: {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6463 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6464 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6465 return;
6466 case Intrinsic::aarch64_sve_umax_x4:
6467 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6468 VT: Node->getValueType(ResNo: 0),
6469 Opcodes: {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6470 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6471 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6472 return;
6473 case Intrinsic::aarch64_sve_fmax_x4:
6474 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6475 VT: Node->getValueType(ResNo: 0),
6476 Opcodes: {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6477 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6478 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6479 return;
6480 case Intrinsic::aarch64_sme_famax_x2:
6481 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6482 VT: Node->getValueType(ResNo: 0),
6483 Opcodes: {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6484 AArch64::FAMAX_2Z2Z_D}))
6485 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6486 return;
6487 case Intrinsic::aarch64_sme_famax_x4:
6488 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6489 VT: Node->getValueType(ResNo: 0),
6490 Opcodes: {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6491 AArch64::FAMAX_4Z4Z_D}))
6492 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6493 return;
6494 case Intrinsic::aarch64_sme_famin_x2:
6495 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6496 VT: Node->getValueType(ResNo: 0),
6497 Opcodes: {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6498 AArch64::FAMIN_2Z2Z_D}))
6499 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6500 return;
6501 case Intrinsic::aarch64_sme_famin_x4:
6502 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6503 VT: Node->getValueType(ResNo: 0),
6504 Opcodes: {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6505 AArch64::FAMIN_4Z4Z_D}))
6506 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6507 return;
6508 case Intrinsic::aarch64_sve_smin_x2:
6509 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6510 VT: Node->getValueType(ResNo: 0),
6511 Opcodes: {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6512 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6513 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6514 return;
6515 case Intrinsic::aarch64_sve_umin_x2:
6516 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6517 VT: Node->getValueType(ResNo: 0),
6518 Opcodes: {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6519 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6520 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6521 return;
6522 case Intrinsic::aarch64_sve_fmin_x2:
6523 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6524 VT: Node->getValueType(ResNo: 0),
6525 Opcodes: {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6526 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6527 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6528 return;
6529 case Intrinsic::aarch64_sve_smin_x4:
6530 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6531 VT: Node->getValueType(ResNo: 0),
6532 Opcodes: {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6533 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6534 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6535 return;
6536 case Intrinsic::aarch64_sve_umin_x4:
6537 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6538 VT: Node->getValueType(ResNo: 0),
6539 Opcodes: {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6540 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6541 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6542 return;
6543 case Intrinsic::aarch64_sve_fmin_x4:
6544 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6545 VT: Node->getValueType(ResNo: 0),
6546 Opcodes: {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6547 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6548 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6549 return;
6550 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6551 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6552 VT: Node->getValueType(ResNo: 0),
6553 Opcodes: {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6554 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6555 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6556 return;
6557 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6558 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6559 VT: Node->getValueType(ResNo: 0),
6560 Opcodes: {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6561 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6562 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6563 return;
6564 case Intrinsic::aarch64_sve_fminnm_single_x2:
6565 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6566 VT: Node->getValueType(ResNo: 0),
6567 Opcodes: {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6568 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6569 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6570 return;
6571 case Intrinsic::aarch64_sve_fminnm_single_x4:
6572 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6573 VT: Node->getValueType(ResNo: 0),
6574 Opcodes: {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6575 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6576 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6577 return;
6578 case Intrinsic::aarch64_sve_fscale_single_x4:
6579 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::BFSCALE_4ZZ);
6580 return;
6581 case Intrinsic::aarch64_sve_fscale_single_x2:
6582 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::BFSCALE_2ZZ);
6583 return;
6584 case Intrinsic::aarch64_sve_fmul_single_x4:
6585 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6586 VT: Node->getValueType(ResNo: 0),
6587 Opcodes: {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6588 AArch64::FMUL_4ZZ_D}))
6589 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6590 return;
6591 case Intrinsic::aarch64_sve_fmul_single_x2:
6592 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6593 VT: Node->getValueType(ResNo: 0),
6594 Opcodes: {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6595 AArch64::FMUL_2ZZ_D}))
6596 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6597 return;
6598 case Intrinsic::aarch64_sve_fmaxnm_x2:
6599 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6600 VT: Node->getValueType(ResNo: 0),
6601 Opcodes: {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6602 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6603 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6604 return;
6605 case Intrinsic::aarch64_sve_fmaxnm_x4:
6606 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6607 VT: Node->getValueType(ResNo: 0),
6608 Opcodes: {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6609 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6610 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6611 return;
6612 case Intrinsic::aarch64_sve_fminnm_x2:
6613 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6614 VT: Node->getValueType(ResNo: 0),
6615 Opcodes: {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6616 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6617 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6618 return;
6619 case Intrinsic::aarch64_sve_fminnm_x4:
6620 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6621 VT: Node->getValueType(ResNo: 0),
6622 Opcodes: {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6623 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6624 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6625 return;
6626 case Intrinsic::aarch64_sve_aese_lane_x2:
6627 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESE_2ZZI_B);
6628 return;
6629 case Intrinsic::aarch64_sve_aesd_lane_x2:
6630 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESD_2ZZI_B);
6631 return;
6632 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6633 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESEMC_2ZZI_B);
6634 return;
6635 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6636 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESDIMC_2ZZI_B);
6637 return;
6638 case Intrinsic::aarch64_sve_aese_lane_x4:
6639 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESE_4ZZI_B);
6640 return;
6641 case Intrinsic::aarch64_sve_aesd_lane_x4:
6642 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESD_4ZZI_B);
6643 return;
6644 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6645 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESEMC_4ZZI_B);
6646 return;
6647 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6648 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESDIMC_4ZZI_B);
6649 return;
6650 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6651 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::PMLAL_2ZZZ_Q);
6652 return;
6653 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6654 SDLoc DL(Node);
6655 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 1, M: 2));
6656 SDNode *Res =
6657 CurDAG->getMachineNode(Opcode: AArch64::PMULL_2ZZZ_Q, dl: DL, VT: MVT::Untyped, Ops: Regs);
6658 SDValue SuperReg = SDValue(Res, 0);
6659 for (unsigned I = 0; I < 2; I++)
6660 ReplaceUses(F: SDValue(Node, I),
6661 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
6662 Operand: SuperReg));
6663 CurDAG->RemoveDeadNode(N: Node);
6664 return;
6665 }
6666 case Intrinsic::aarch64_sve_fscale_x4:
6667 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: AArch64::BFSCALE_4Z4Z);
6668 return;
6669 case Intrinsic::aarch64_sve_fscale_x2:
6670 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: AArch64::BFSCALE_2Z2Z);
6671 return;
6672 case Intrinsic::aarch64_sve_fmul_x4:
6673 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6674 VT: Node->getValueType(ResNo: 0),
6675 Opcodes: {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6676 AArch64::FMUL_4Z4Z_D}))
6677 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6678 return;
6679 case Intrinsic::aarch64_sve_fmul_x2:
6680 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6681 VT: Node->getValueType(ResNo: 0),
6682 Opcodes: {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6683 AArch64::FMUL_2Z2Z_D}))
6684 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6685 return;
6686 case Intrinsic::aarch64_sve_fcvtzs_x2:
6687 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZS_2Z2Z_StoS);
6688 return;
6689 case Intrinsic::aarch64_sve_scvtf_x2:
6690 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::SCVTF_2Z2Z_StoS);
6691 return;
6692 case Intrinsic::aarch64_sve_fcvtzu_x2:
6693 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZU_2Z2Z_StoS);
6694 return;
6695 case Intrinsic::aarch64_sve_ucvtf_x2:
6696 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::UCVTF_2Z2Z_StoS);
6697 return;
6698 case Intrinsic::aarch64_sve_fcvtzs_x4:
6699 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZS_4Z4Z_StoS);
6700 return;
6701 case Intrinsic::aarch64_sve_scvtf_x4:
6702 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::SCVTF_4Z4Z_StoS);
6703 return;
6704 case Intrinsic::aarch64_sve_fcvtzu_x4:
6705 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZU_4Z4Z_StoS);
6706 return;
6707 case Intrinsic::aarch64_sve_ucvtf_x4:
6708 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::UCVTF_4Z4Z_StoS);
6709 return;
6710 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6711 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVT_2ZZ_H_S);
6712 return;
6713 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6714 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVTL_2ZZ_H_S);
6715 return;
6716 case Intrinsic::aarch64_sve_sclamp_single_x2:
6717 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6718 VT: Node->getValueType(ResNo: 0),
6719 Opcodes: {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6720 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6721 SelectClamp(N: Node, NumVecs: 2, Op);
6722 return;
6723 case Intrinsic::aarch64_sve_uclamp_single_x2:
6724 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6725 VT: Node->getValueType(ResNo: 0),
6726 Opcodes: {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6727 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6728 SelectClamp(N: Node, NumVecs: 2, Op);
6729 return;
6730 case Intrinsic::aarch64_sve_fclamp_single_x2:
6731 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6732 VT: Node->getValueType(ResNo: 0),
6733 Opcodes: {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6734 AArch64::FCLAMP_VG2_2Z2Z_D}))
6735 SelectClamp(N: Node, NumVecs: 2, Op);
6736 return;
6737 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6738 SelectClamp(N: Node, NumVecs: 2, Op: AArch64::BFCLAMP_VG2_2ZZZ_H);
6739 return;
6740 case Intrinsic::aarch64_sve_sclamp_single_x4:
6741 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6742 VT: Node->getValueType(ResNo: 0),
6743 Opcodes: {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6744 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6745 SelectClamp(N: Node, NumVecs: 4, Op);
6746 return;
6747 case Intrinsic::aarch64_sve_uclamp_single_x4:
6748 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6749 VT: Node->getValueType(ResNo: 0),
6750 Opcodes: {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6751 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6752 SelectClamp(N: Node, NumVecs: 4, Op);
6753 return;
6754 case Intrinsic::aarch64_sve_fclamp_single_x4:
6755 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6756 VT: Node->getValueType(ResNo: 0),
6757 Opcodes: {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6758 AArch64::FCLAMP_VG4_4Z4Z_D}))
6759 SelectClamp(N: Node, NumVecs: 4, Op);
6760 return;
6761 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6762 SelectClamp(N: Node, NumVecs: 4, Op: AArch64::BFCLAMP_VG4_4ZZZ_H);
6763 return;
6764 case Intrinsic::aarch64_sve_add_single_x2:
6765 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6766 VT: Node->getValueType(ResNo: 0),
6767 Opcodes: {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6768 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6769 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6770 return;
6771 case Intrinsic::aarch64_sve_add_single_x4:
6772 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6773 VT: Node->getValueType(ResNo: 0),
6774 Opcodes: {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6775 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6776 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6777 return;
6778 case Intrinsic::aarch64_sve_zip_x2:
6779 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6780 VT: Node->getValueType(ResNo: 0),
6781 Opcodes: {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6782 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6783 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6784 return;
6785 case Intrinsic::aarch64_sve_zipq_x2:
6786 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6787 Opc: AArch64::ZIP_VG2_2ZZZ_Q);
6788 return;
6789 case Intrinsic::aarch64_sve_zip_x4:
6790 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6791 VT: Node->getValueType(ResNo: 0),
6792 Opcodes: {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6793 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6794 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6795 return;
6796 case Intrinsic::aarch64_sve_zipq_x4:
6797 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6798 Opc: AArch64::ZIP_VG4_4Z4Z_Q);
6799 return;
6800 case Intrinsic::aarch64_sve_uzp_x2:
6801 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6802 VT: Node->getValueType(ResNo: 0),
6803 Opcodes: {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6804 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6805 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6806 return;
6807 case Intrinsic::aarch64_sve_uzpq_x2:
6808 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6809 Opc: AArch64::UZP_VG2_2ZZZ_Q);
6810 return;
6811 case Intrinsic::aarch64_sve_uzp_x4:
6812 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6813 VT: Node->getValueType(ResNo: 0),
6814 Opcodes: {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6815 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6816 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6817 return;
6818 case Intrinsic::aarch64_sve_uzpq_x4:
6819 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6820 Opc: AArch64::UZP_VG4_4Z4Z_Q);
6821 return;
6822 case Intrinsic::aarch64_sve_sel_x2:
6823 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6824 VT: Node->getValueType(ResNo: 0),
6825 Opcodes: {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6826 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6827 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6828 return;
6829 case Intrinsic::aarch64_sve_sel_x4:
6830 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6831 VT: Node->getValueType(ResNo: 0),
6832 Opcodes: {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6833 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6834 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6835 return;
6836 case Intrinsic::aarch64_sve_frinta_x2:
6837 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTA_2Z2Z_S);
6838 return;
6839 case Intrinsic::aarch64_sve_frinta_x4:
6840 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTA_4Z4Z_S);
6841 return;
6842 case Intrinsic::aarch64_sve_frintm_x2:
6843 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTM_2Z2Z_S);
6844 return;
6845 case Intrinsic::aarch64_sve_frintm_x4:
6846 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTM_4Z4Z_S);
6847 return;
6848 case Intrinsic::aarch64_sve_frintn_x2:
6849 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTN_2Z2Z_S);
6850 return;
6851 case Intrinsic::aarch64_sve_frintn_x4:
6852 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTN_4Z4Z_S);
6853 return;
6854 case Intrinsic::aarch64_sve_frintp_x2:
6855 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTP_2Z2Z_S);
6856 return;
6857 case Intrinsic::aarch64_sve_frintp_x4:
6858 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTP_4Z4Z_S);
6859 return;
6860 case Intrinsic::aarch64_sve_sunpk_x2:
6861 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6862 VT: Node->getValueType(ResNo: 0),
6863 Opcodes: {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6864 AArch64::SUNPK_VG2_2ZZ_D}))
6865 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6866 return;
6867 case Intrinsic::aarch64_sve_uunpk_x2:
6868 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6869 VT: Node->getValueType(ResNo: 0),
6870 Opcodes: {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6871 AArch64::UUNPK_VG2_2ZZ_D}))
6872 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6873 return;
6874 case Intrinsic::aarch64_sve_sunpk_x4:
6875 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6876 VT: Node->getValueType(ResNo: 0),
6877 Opcodes: {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6878 AArch64::SUNPK_VG4_4Z2Z_D}))
6879 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6880 return;
6881 case Intrinsic::aarch64_sve_uunpk_x4:
6882 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6883 VT: Node->getValueType(ResNo: 0),
6884 Opcodes: {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6885 AArch64::UUNPK_VG4_4Z2Z_D}))
6886 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6887 return;
6888 case Intrinsic::aarch64_sve_pext_x2: {
6889 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6890 VT: Node->getValueType(ResNo: 0),
6891 Opcodes: {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6892 AArch64::PEXT_2PCI_D}))
6893 SelectPExtPair(N: Node, Opc: Op);
6894 return;
6895 }
6896 }
6897 break;
6898 }
6899 case ISD::INTRINSIC_VOID: {
6900 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
6901 if (Node->getNumOperands() >= 3)
6902 VT = Node->getOperand(Num: 2)->getValueType(ResNo: 0);
6903 switch (IntNo) {
6904 default:
6905 break;
6906 case Intrinsic::aarch64_neon_st1x2: {
6907 if (VT == MVT::v8i8) {
6908 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b);
6909 return;
6910 } else if (VT == MVT::v16i8) {
6911 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b);
6912 return;
6913 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6914 VT == MVT::v4bf16) {
6915 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h);
6916 return;
6917 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6918 VT == MVT::v8bf16) {
6919 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h);
6920 return;
6921 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6922 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s);
6923 return;
6924 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6925 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s);
6926 return;
6927 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6928 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d);
6929 return;
6930 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6931 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6932 return;
6933 }
6934 break;
6935 }
6936 case Intrinsic::aarch64_neon_st1x3: {
6937 if (VT == MVT::v8i8) {
6938 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b);
6939 return;
6940 } else if (VT == MVT::v16i8) {
6941 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b);
6942 return;
6943 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6944 VT == MVT::v4bf16) {
6945 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h);
6946 return;
6947 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6948 VT == MVT::v8bf16) {
6949 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h);
6950 return;
6951 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6952 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s);
6953 return;
6954 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6955 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s);
6956 return;
6957 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6958 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d);
6959 return;
6960 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6961 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
6962 return;
6963 }
6964 break;
6965 }
6966 case Intrinsic::aarch64_neon_st1x4: {
6967 if (VT == MVT::v8i8) {
6968 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b);
6969 return;
6970 } else if (VT == MVT::v16i8) {
6971 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b);
6972 return;
6973 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6974 VT == MVT::v4bf16) {
6975 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h);
6976 return;
6977 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6978 VT == MVT::v8bf16) {
6979 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h);
6980 return;
6981 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6982 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s);
6983 return;
6984 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6985 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s);
6986 return;
6987 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6988 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d);
6989 return;
6990 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6991 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
6992 return;
6993 }
6994 break;
6995 }
6996 case Intrinsic::aarch64_neon_st2: {
6997 if (VT == MVT::v8i8) {
6998 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b);
6999 return;
7000 } else if (VT == MVT::v16i8) {
7001 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b);
7002 return;
7003 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7004 VT == MVT::v4bf16) {
7005 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h);
7006 return;
7007 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7008 VT == MVT::v8bf16) {
7009 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h);
7010 return;
7011 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7012 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s);
7013 return;
7014 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7015 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s);
7016 return;
7017 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7018 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d);
7019 return;
7020 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7021 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
7022 return;
7023 }
7024 break;
7025 }
7026 case Intrinsic::aarch64_neon_st3: {
7027 if (VT == MVT::v8i8) {
7028 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b);
7029 return;
7030 } else if (VT == MVT::v16i8) {
7031 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b);
7032 return;
7033 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7034 VT == MVT::v4bf16) {
7035 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h);
7036 return;
7037 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7038 VT == MVT::v8bf16) {
7039 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h);
7040 return;
7041 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7042 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s);
7043 return;
7044 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7045 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s);
7046 return;
7047 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7048 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d);
7049 return;
7050 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7051 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
7052 return;
7053 }
7054 break;
7055 }
7056 case Intrinsic::aarch64_neon_st4: {
7057 if (VT == MVT::v8i8) {
7058 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b);
7059 return;
7060 } else if (VT == MVT::v16i8) {
7061 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b);
7062 return;
7063 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7064 VT == MVT::v4bf16) {
7065 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h);
7066 return;
7067 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7068 VT == MVT::v8bf16) {
7069 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h);
7070 return;
7071 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7072 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s);
7073 return;
7074 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7075 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s);
7076 return;
7077 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7078 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d);
7079 return;
7080 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7081 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7082 return;
7083 }
7084 break;
7085 }
7086 case Intrinsic::aarch64_neon_st2lane: {
7087 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7088 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8);
7089 return;
7090 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7091 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7092 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16);
7093 return;
7094 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7095 VT == MVT::v2f32) {
7096 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32);
7097 return;
7098 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7099 VT == MVT::v1f64) {
7100 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64);
7101 return;
7102 }
7103 break;
7104 }
7105 case Intrinsic::aarch64_neon_st3lane: {
7106 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7107 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8);
7108 return;
7109 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7110 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7111 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16);
7112 return;
7113 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7114 VT == MVT::v2f32) {
7115 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32);
7116 return;
7117 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7118 VT == MVT::v1f64) {
7119 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64);
7120 return;
7121 }
7122 break;
7123 }
7124 case Intrinsic::aarch64_neon_st4lane: {
7125 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7126 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8);
7127 return;
7128 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7129 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7130 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16);
7131 return;
7132 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7133 VT == MVT::v2f32) {
7134 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32);
7135 return;
7136 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7137 VT == MVT::v1f64) {
7138 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64);
7139 return;
7140 }
7141 break;
7142 }
7143 case Intrinsic::aarch64_sve_st2q: {
7144 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 4, Opc_rr: AArch64::ST2Q, Opc_ri: AArch64::ST2Q_IMM);
7145 return;
7146 }
7147 case Intrinsic::aarch64_sve_st3q: {
7148 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 4, Opc_rr: AArch64::ST3Q, Opc_ri: AArch64::ST3Q_IMM);
7149 return;
7150 }
7151 case Intrinsic::aarch64_sve_st4q: {
7152 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 4, Opc_rr: AArch64::ST4Q, Opc_ri: AArch64::ST4Q_IMM);
7153 return;
7154 }
7155 case Intrinsic::aarch64_sve_st2: {
7156 if (VT == MVT::nxv16i8) {
7157 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 0, Opc_rr: AArch64::ST2B, Opc_ri: AArch64::ST2B_IMM);
7158 return;
7159 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7160 VT == MVT::nxv8bf16) {
7161 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 1, Opc_rr: AArch64::ST2H, Opc_ri: AArch64::ST2H_IMM);
7162 return;
7163 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7164 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 2, Opc_rr: AArch64::ST2W, Opc_ri: AArch64::ST2W_IMM);
7165 return;
7166 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7167 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 3, Opc_rr: AArch64::ST2D, Opc_ri: AArch64::ST2D_IMM);
7168 return;
7169 }
7170 break;
7171 }
7172 case Intrinsic::aarch64_sve_st3: {
7173 if (VT == MVT::nxv16i8) {
7174 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 0, Opc_rr: AArch64::ST3B, Opc_ri: AArch64::ST3B_IMM);
7175 return;
7176 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7177 VT == MVT::nxv8bf16) {
7178 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 1, Opc_rr: AArch64::ST3H, Opc_ri: AArch64::ST3H_IMM);
7179 return;
7180 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7181 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 2, Opc_rr: AArch64::ST3W, Opc_ri: AArch64::ST3W_IMM);
7182 return;
7183 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7184 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 3, Opc_rr: AArch64::ST3D, Opc_ri: AArch64::ST3D_IMM);
7185 return;
7186 }
7187 break;
7188 }
7189 case Intrinsic::aarch64_sve_st4: {
7190 if (VT == MVT::nxv16i8) {
7191 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 0, Opc_rr: AArch64::ST4B, Opc_ri: AArch64::ST4B_IMM);
7192 return;
7193 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7194 VT == MVT::nxv8bf16) {
7195 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 1, Opc_rr: AArch64::ST4H, Opc_ri: AArch64::ST4H_IMM);
7196 return;
7197 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7198 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 2, Opc_rr: AArch64::ST4W, Opc_ri: AArch64::ST4W_IMM);
7199 return;
7200 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7201 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 3, Opc_rr: AArch64::ST4D, Opc_ri: AArch64::ST4D_IMM);
7202 return;
7203 }
7204 break;
7205 }
7206 }
7207 break;
7208 }
7209 case AArch64ISD::LD2post: {
7210 if (VT == MVT::v8i8) {
7211 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b_POST, SubRegIdx: AArch64::dsub0);
7212 return;
7213 } else if (VT == MVT::v16i8) {
7214 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b_POST, SubRegIdx: AArch64::qsub0);
7215 return;
7216 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7217 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h_POST, SubRegIdx: AArch64::dsub0);
7218 return;
7219 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7220 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h_POST, SubRegIdx: AArch64::qsub0);
7221 return;
7222 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7223 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s_POST, SubRegIdx: AArch64::dsub0);
7224 return;
7225 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7226 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s_POST, SubRegIdx: AArch64::qsub0);
7227 return;
7228 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7229 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7230 return;
7231 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7232 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d_POST, SubRegIdx: AArch64::qsub0);
7233 return;
7234 }
7235 break;
7236 }
7237 case AArch64ISD::LD3post: {
7238 if (VT == MVT::v8i8) {
7239 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b_POST, SubRegIdx: AArch64::dsub0);
7240 return;
7241 } else if (VT == MVT::v16i8) {
7242 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b_POST, SubRegIdx: AArch64::qsub0);
7243 return;
7244 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7245 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h_POST, SubRegIdx: AArch64::dsub0);
7246 return;
7247 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7248 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h_POST, SubRegIdx: AArch64::qsub0);
7249 return;
7250 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7251 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s_POST, SubRegIdx: AArch64::dsub0);
7252 return;
7253 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7254 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s_POST, SubRegIdx: AArch64::qsub0);
7255 return;
7256 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7257 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7258 return;
7259 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7260 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d_POST, SubRegIdx: AArch64::qsub0);
7261 return;
7262 }
7263 break;
7264 }
7265 case AArch64ISD::LD4post: {
7266 if (VT == MVT::v8i8) {
7267 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7268 return;
7269 } else if (VT == MVT::v16i8) {
7270 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7271 return;
7272 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7273 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7274 return;
7275 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7276 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7277 return;
7278 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7279 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7280 return;
7281 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7282 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7283 return;
7284 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7285 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7286 return;
7287 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7288 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7289 return;
7290 }
7291 break;
7292 }
7293 case AArch64ISD::LD1x2post: {
7294 if (VT == MVT::v8i8) {
7295 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b_POST, SubRegIdx: AArch64::dsub0);
7296 return;
7297 } else if (VT == MVT::v16i8) {
7298 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b_POST, SubRegIdx: AArch64::qsub0);
7299 return;
7300 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7301 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h_POST, SubRegIdx: AArch64::dsub0);
7302 return;
7303 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7304 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h_POST, SubRegIdx: AArch64::qsub0);
7305 return;
7306 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7307 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s_POST, SubRegIdx: AArch64::dsub0);
7308 return;
7309 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7310 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s_POST, SubRegIdx: AArch64::qsub0);
7311 return;
7312 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7313 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7314 return;
7315 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7316 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d_POST, SubRegIdx: AArch64::qsub0);
7317 return;
7318 }
7319 break;
7320 }
7321 case AArch64ISD::LD1x3post: {
7322 if (VT == MVT::v8i8) {
7323 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b_POST, SubRegIdx: AArch64::dsub0);
7324 return;
7325 } else if (VT == MVT::v16i8) {
7326 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b_POST, SubRegIdx: AArch64::qsub0);
7327 return;
7328 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7329 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h_POST, SubRegIdx: AArch64::dsub0);
7330 return;
7331 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7332 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h_POST, SubRegIdx: AArch64::qsub0);
7333 return;
7334 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7335 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s_POST, SubRegIdx: AArch64::dsub0);
7336 return;
7337 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7338 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s_POST, SubRegIdx: AArch64::qsub0);
7339 return;
7340 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7341 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7342 return;
7343 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7344 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d_POST, SubRegIdx: AArch64::qsub0);
7345 return;
7346 }
7347 break;
7348 }
7349 case AArch64ISD::LD1x4post: {
7350 if (VT == MVT::v8i8) {
7351 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7352 return;
7353 } else if (VT == MVT::v16i8) {
7354 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7355 return;
7356 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7357 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7358 return;
7359 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7360 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7361 return;
7362 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7363 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7364 return;
7365 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7366 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7367 return;
7368 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7369 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7370 return;
7371 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7372 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7373 return;
7374 }
7375 break;
7376 }
7377 case AArch64ISD::LD1DUPpost: {
7378 if (VT == MVT::v8i8) {
7379 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8b_POST, SubRegIdx: AArch64::dsub0);
7380 return;
7381 } else if (VT == MVT::v16i8) {
7382 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv16b_POST, SubRegIdx: AArch64::qsub0);
7383 return;
7384 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7385 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4h_POST, SubRegIdx: AArch64::dsub0);
7386 return;
7387 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7388 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8h_POST, SubRegIdx: AArch64::qsub0);
7389 return;
7390 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7391 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2s_POST, SubRegIdx: AArch64::dsub0);
7392 return;
7393 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7394 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4s_POST, SubRegIdx: AArch64::qsub0);
7395 return;
7396 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7397 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv1d_POST, SubRegIdx: AArch64::dsub0);
7398 return;
7399 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7400 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2d_POST, SubRegIdx: AArch64::qsub0);
7401 return;
7402 }
7403 break;
7404 }
7405 case AArch64ISD::LD2DUPpost: {
7406 if (VT == MVT::v8i8) {
7407 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b_POST, SubRegIdx: AArch64::dsub0);
7408 return;
7409 } else if (VT == MVT::v16i8) {
7410 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b_POST, SubRegIdx: AArch64::qsub0);
7411 return;
7412 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7413 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h_POST, SubRegIdx: AArch64::dsub0);
7414 return;
7415 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7416 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h_POST, SubRegIdx: AArch64::qsub0);
7417 return;
7418 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7419 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s_POST, SubRegIdx: AArch64::dsub0);
7420 return;
7421 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7422 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s_POST, SubRegIdx: AArch64::qsub0);
7423 return;
7424 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7425 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d_POST, SubRegIdx: AArch64::dsub0);
7426 return;
7427 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7428 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d_POST, SubRegIdx: AArch64::qsub0);
7429 return;
7430 }
7431 break;
7432 }
7433 case AArch64ISD::LD3DUPpost: {
7434 if (VT == MVT::v8i8) {
7435 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b_POST, SubRegIdx: AArch64::dsub0);
7436 return;
7437 } else if (VT == MVT::v16i8) {
7438 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b_POST, SubRegIdx: AArch64::qsub0);
7439 return;
7440 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7441 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h_POST, SubRegIdx: AArch64::dsub0);
7442 return;
7443 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7444 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h_POST, SubRegIdx: AArch64::qsub0);
7445 return;
7446 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7447 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s_POST, SubRegIdx: AArch64::dsub0);
7448 return;
7449 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7450 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s_POST, SubRegIdx: AArch64::qsub0);
7451 return;
7452 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7453 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d_POST, SubRegIdx: AArch64::dsub0);
7454 return;
7455 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7456 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d_POST, SubRegIdx: AArch64::qsub0);
7457 return;
7458 }
7459 break;
7460 }
7461 case AArch64ISD::LD4DUPpost: {
7462 if (VT == MVT::v8i8) {
7463 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b_POST, SubRegIdx: AArch64::dsub0);
7464 return;
7465 } else if (VT == MVT::v16i8) {
7466 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b_POST, SubRegIdx: AArch64::qsub0);
7467 return;
7468 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7469 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h_POST, SubRegIdx: AArch64::dsub0);
7470 return;
7471 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7472 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h_POST, SubRegIdx: AArch64::qsub0);
7473 return;
7474 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7475 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s_POST, SubRegIdx: AArch64::dsub0);
7476 return;
7477 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7478 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s_POST, SubRegIdx: AArch64::qsub0);
7479 return;
7480 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7481 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d_POST, SubRegIdx: AArch64::dsub0);
7482 return;
7483 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7484 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d_POST, SubRegIdx: AArch64::qsub0);
7485 return;
7486 }
7487 break;
7488 }
7489 case AArch64ISD::LD1LANEpost: {
7490 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7491 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i8_POST);
7492 return;
7493 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7494 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7495 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i16_POST);
7496 return;
7497 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7498 VT == MVT::v2f32) {
7499 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i32_POST);
7500 return;
7501 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7502 VT == MVT::v1f64) {
7503 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i64_POST);
7504 return;
7505 }
7506 break;
7507 }
7508 case AArch64ISD::LD2LANEpost: {
7509 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7510 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8_POST);
7511 return;
7512 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7513 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7514 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16_POST);
7515 return;
7516 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7517 VT == MVT::v2f32) {
7518 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32_POST);
7519 return;
7520 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7521 VT == MVT::v1f64) {
7522 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64_POST);
7523 return;
7524 }
7525 break;
7526 }
7527 case AArch64ISD::LD3LANEpost: {
7528 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7529 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8_POST);
7530 return;
7531 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7532 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7533 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16_POST);
7534 return;
7535 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7536 VT == MVT::v2f32) {
7537 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32_POST);
7538 return;
7539 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7540 VT == MVT::v1f64) {
7541 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64_POST);
7542 return;
7543 }
7544 break;
7545 }
7546 case AArch64ISD::LD4LANEpost: {
7547 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7548 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8_POST);
7549 return;
7550 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7551 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7552 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16_POST);
7553 return;
7554 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7555 VT == MVT::v2f32) {
7556 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32_POST);
7557 return;
7558 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7559 VT == MVT::v1f64) {
7560 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64_POST);
7561 return;
7562 }
7563 break;
7564 }
7565 case AArch64ISD::ST2post: {
7566 VT = Node->getOperand(Num: 1).getValueType();
7567 if (VT == MVT::v8i8) {
7568 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b_POST);
7569 return;
7570 } else if (VT == MVT::v16i8) {
7571 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b_POST);
7572 return;
7573 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7574 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h_POST);
7575 return;
7576 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7577 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h_POST);
7578 return;
7579 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7580 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s_POST);
7581 return;
7582 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7583 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s_POST);
7584 return;
7585 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7586 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d_POST);
7587 return;
7588 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7589 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7590 return;
7591 }
7592 break;
7593 }
7594 case AArch64ISD::ST3post: {
7595 VT = Node->getOperand(Num: 1).getValueType();
7596 if (VT == MVT::v8i8) {
7597 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b_POST);
7598 return;
7599 } else if (VT == MVT::v16i8) {
7600 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b_POST);
7601 return;
7602 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7603 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h_POST);
7604 return;
7605 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7606 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h_POST);
7607 return;
7608 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7609 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s_POST);
7610 return;
7611 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7612 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s_POST);
7613 return;
7614 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7615 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d_POST);
7616 return;
7617 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7618 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7619 return;
7620 }
7621 break;
7622 }
7623 case AArch64ISD::ST4post: {
7624 VT = Node->getOperand(Num: 1).getValueType();
7625 if (VT == MVT::v8i8) {
7626 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b_POST);
7627 return;
7628 } else if (VT == MVT::v16i8) {
7629 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b_POST);
7630 return;
7631 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7632 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h_POST);
7633 return;
7634 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7635 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h_POST);
7636 return;
7637 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7638 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s_POST);
7639 return;
7640 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7641 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s_POST);
7642 return;
7643 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7644 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d_POST);
7645 return;
7646 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7647 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7648 return;
7649 }
7650 break;
7651 }
7652 case AArch64ISD::ST1x2post: {
7653 VT = Node->getOperand(Num: 1).getValueType();
7654 if (VT == MVT::v8i8) {
7655 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b_POST);
7656 return;
7657 } else if (VT == MVT::v16i8) {
7658 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b_POST);
7659 return;
7660 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7661 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h_POST);
7662 return;
7663 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7664 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h_POST);
7665 return;
7666 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7667 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s_POST);
7668 return;
7669 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7670 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s_POST);
7671 return;
7672 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7673 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7674 return;
7675 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7676 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d_POST);
7677 return;
7678 }
7679 break;
7680 }
7681 case AArch64ISD::ST1x3post: {
7682 VT = Node->getOperand(Num: 1).getValueType();
7683 if (VT == MVT::v8i8) {
7684 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b_POST);
7685 return;
7686 } else if (VT == MVT::v16i8) {
7687 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b_POST);
7688 return;
7689 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7690 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h_POST);
7691 return;
7692 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7693 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h_POST);
7694 return;
7695 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7696 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s_POST);
7697 return;
7698 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7699 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s_POST);
7700 return;
7701 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7702 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7703 return;
7704 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7705 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d_POST);
7706 return;
7707 }
7708 break;
7709 }
7710 case AArch64ISD::ST1x4post: {
7711 VT = Node->getOperand(Num: 1).getValueType();
7712 if (VT == MVT::v8i8) {
7713 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b_POST);
7714 return;
7715 } else if (VT == MVT::v16i8) {
7716 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b_POST);
7717 return;
7718 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7719 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h_POST);
7720 return;
7721 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7722 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h_POST);
7723 return;
7724 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7725 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s_POST);
7726 return;
7727 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7728 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s_POST);
7729 return;
7730 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7731 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7732 return;
7733 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7734 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d_POST);
7735 return;
7736 }
7737 break;
7738 }
7739 case AArch64ISD::ST2LANEpost: {
7740 VT = Node->getOperand(Num: 1).getValueType();
7741 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7742 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8_POST);
7743 return;
7744 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7745 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7746 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16_POST);
7747 return;
7748 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7749 VT == MVT::v2f32) {
7750 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32_POST);
7751 return;
7752 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7753 VT == MVT::v1f64) {
7754 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64_POST);
7755 return;
7756 }
7757 break;
7758 }
7759 case AArch64ISD::ST3LANEpost: {
7760 VT = Node->getOperand(Num: 1).getValueType();
7761 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7762 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8_POST);
7763 return;
7764 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7765 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7766 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16_POST);
7767 return;
7768 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7769 VT == MVT::v2f32) {
7770 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32_POST);
7771 return;
7772 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7773 VT == MVT::v1f64) {
7774 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64_POST);
7775 return;
7776 }
7777 break;
7778 }
7779 case AArch64ISD::ST4LANEpost: {
7780 VT = Node->getOperand(Num: 1).getValueType();
7781 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7782 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8_POST);
7783 return;
7784 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7785 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7786 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16_POST);
7787 return;
7788 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7789 VT == MVT::v2f32) {
7790 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32_POST);
7791 return;
7792 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7793 VT == MVT::v1f64) {
7794 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64_POST);
7795 return;
7796 }
7797 break;
7798 }
7799 }
7800
7801 // Select the default instruction
7802 SelectCode(N: Node);
7803}
7804
7805/// createAArch64ISelDag - This pass converts a legalized DAG into a
7806/// AArch64-specific DAG, ready for instruction scheduling.
7807FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
7808 CodeGenOptLevel OptLevel) {
7809 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7810}
7811
7812/// When \p PredVT is a scalable vector predicate in the form
7813/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7814/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7815/// structured vectors (NumVec >1), the output data type is
7816/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7817/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7818/// EVT.
7819static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
7820 unsigned NumVec) {
7821 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7822 if (!PredVT.isScalableVectorOf(EltVT: MVT::i1))
7823 return EVT();
7824
7825 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7826 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7827 return EVT();
7828
7829 ElementCount EC = PredVT.getVectorElementCount();
7830 EVT ScalarVT =
7831 EVT::getIntegerVT(Context&: Ctx, BitWidth: AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7832 EVT MemVT = EVT::getVectorVT(Context&: Ctx, VT: ScalarVT, EC: EC * NumVec);
7833
7834 return MemVT;
7835}
7836
7837/// Builds an integer vector type large enough to hold \p NumVec instances
7838/// of \p VecVT.
7839static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7840 return EVT::getVectorVT(Context&: Ctx, VT: VecVT.getScalarType().changeTypeToInteger(),
7841 EC: VecVT.getVectorElementCount() * NumVec);
7842}
7843
7844/// Return the EVT of the data associated to a memory operation in \p
7845/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7846static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
7847 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: Root))
7848 return MemIntr->getMemoryVT();
7849
7850 if (isa<MemSDNode>(Val: Root)) {
7851 EVT MemVT = cast<MemSDNode>(Val: Root)->getMemoryVT();
7852
7853 EVT DataVT;
7854 if (auto *Load = dyn_cast<LoadSDNode>(Val: Root))
7855 DataVT = Load->getValueType(ResNo: 0);
7856 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Val: Root))
7857 DataVT = Load->getValueType(ResNo: 0);
7858 else if (auto *Store = dyn_cast<StoreSDNode>(Val: Root))
7859 DataVT = Store->getValue().getValueType();
7860 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Val: Root))
7861 DataVT = Store->getValue().getValueType();
7862 else
7863 llvm_unreachable("Unexpected MemSDNode!");
7864
7865 return DataVT.changeVectorElementType(Context&: Ctx, EltVT: MemVT.getVectorElementType());
7866 }
7867
7868 const unsigned Opcode = Root->getOpcode();
7869 // For custom ISD nodes, we have to look at them individually to extract the
7870 // type of the data moved to/from memory.
7871 switch (Opcode) {
7872 case AArch64ISD::LD1_MERGE_ZERO:
7873 case AArch64ISD::LD1S_MERGE_ZERO:
7874 case AArch64ISD::LDNF1_MERGE_ZERO:
7875 case AArch64ISD::LDNF1S_MERGE_ZERO:
7876 return cast<VTSDNode>(Val: Root->getOperand(Num: 3))->getVT();
7877 case AArch64ISD::ST1_PRED:
7878 return cast<VTSDNode>(Val: Root->getOperand(Num: 4))->getVT();
7879 default:
7880 break;
7881 }
7882
7883 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7884 return EVT();
7885
7886 switch (Root->getConstantOperandVal(Num: 1)) {
7887 default:
7888 return EVT();
7889 case Intrinsic::aarch64_sme_ldr:
7890 case Intrinsic::aarch64_sme_str:
7891 return MVT::nxv16i8;
7892 case Intrinsic::aarch64_sve_prf:
7893 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7894 // width of the predicate.
7895 return getPackedVectorTypeFromPredicateType(
7896 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/1);
7897 case Intrinsic::aarch64_sve_ld2_sret:
7898 case Intrinsic::aarch64_sve_ld2q_sret:
7899 return getPackedVectorTypeFromPredicateType(
7900 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/2);
7901 case Intrinsic::aarch64_sve_st2q:
7902 return getPackedVectorTypeFromPredicateType(
7903 Ctx, PredVT: Root->getOperand(Num: 4)->getValueType(ResNo: 0), /*NumVec=*/2);
7904 case Intrinsic::aarch64_sve_ld3_sret:
7905 case Intrinsic::aarch64_sve_ld3q_sret:
7906 return getPackedVectorTypeFromPredicateType(
7907 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/3);
7908 case Intrinsic::aarch64_sve_st3q:
7909 return getPackedVectorTypeFromPredicateType(
7910 Ctx, PredVT: Root->getOperand(Num: 5)->getValueType(ResNo: 0), /*NumVec=*/3);
7911 case Intrinsic::aarch64_sve_ld4_sret:
7912 case Intrinsic::aarch64_sve_ld4q_sret:
7913 return getPackedVectorTypeFromPredicateType(
7914 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/4);
7915 case Intrinsic::aarch64_sve_st4q:
7916 return getPackedVectorTypeFromPredicateType(
7917 Ctx, PredVT: Root->getOperand(Num: 6)->getValueType(ResNo: 0), /*NumVec=*/4);
7918 case Intrinsic::aarch64_sve_ld1_pn_x2:
7919 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7920 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7921 /*NumVec=*/2);
7922 case Intrinsic::aarch64_sve_ld1_pn_x4:
7923 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7924 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7925 /*NumVec=*/4);
7926 case Intrinsic::aarch64_sve_st1_pn_x2:
7927 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7928 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7929 /*NumVec=*/2);
7930 case Intrinsic::aarch64_sve_st1_pn_x4:
7931 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7932 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7933 /*NumVec=*/4);
7934 case Intrinsic::aarch64_sve_ld1udq:
7935 case Intrinsic::aarch64_sve_st1dq:
7936 return EVT(MVT::nxv1i64);
7937 case Intrinsic::aarch64_sve_ld1uwq:
7938 case Intrinsic::aarch64_sve_st1wq:
7939 return EVT(MVT::nxv1i32);
7940 }
7941}
7942
7943/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7944/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7945/// where Root is the memory access using N for its address.
7946template <int64_t Min, int64_t Max>
7947bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7948 SDValue &Base,
7949 SDValue &OffImm) {
7950 const EVT MemVT = getMemVTFromNode(Ctx&: *(CurDAG->getContext()), Root);
7951 const DataLayout &DL = CurDAG->getDataLayout();
7952 const MachineFrameInfo &MFI = MF->getFrameInfo();
7953
7954 if (N.getOpcode() == ISD::FrameIndex) {
7955 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
7956 // We can only encode VL scaled offsets, so only fold in frame indexes
7957 // referencing SVE objects.
7958 if (MFI.hasScalableStackID(ObjectIdx: FI)) {
7959 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7960 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
7961 return true;
7962 }
7963
7964 return false;
7965 }
7966
7967 if (MemVT == EVT())
7968 return false;
7969
7970 if (N.getOpcode() != ISD::ADD)
7971 return false;
7972
7973 SDValue VScale = N.getOperand(i: 1);
7974 int64_t MulImm = std::numeric_limits<int64_t>::max();
7975 if (VScale.getOpcode() == ISD::VSCALE) {
7976 MulImm = cast<ConstantSDNode>(Val: VScale.getOperand(i: 0))->getSExtValue();
7977 } else if (auto C = dyn_cast<ConstantSDNode>(Val&: VScale)) {
7978 int64_t ByteOffset = C->getSExtValue();
7979 const auto KnownVScale =
7980 Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
7981
7982 if (!KnownVScale || ByteOffset % KnownVScale != 0)
7983 return false;
7984
7985 MulImm = ByteOffset / KnownVScale;
7986 } else
7987 return false;
7988
7989 TypeSize TS = MemVT.getSizeInBits();
7990 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
7991
7992 if ((MulImm % MemWidthBytes) != 0)
7993 return false;
7994
7995 int64_t Offset = MulImm / MemWidthBytes;
7996 if (Offset < Min || Offset > Max)
7997 return false;
7998
7999 Base = N.getOperand(i: 0);
8000 if (Base.getOpcode() == ISD::FrameIndex) {
8001 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
8002 // We can only encode VL scaled offsets, so only fold in frame indexes
8003 // referencing SVE objects.
8004 if (MFI.hasScalableStackID(ObjectIdx: FI))
8005 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
8006 }
8007
8008 OffImm = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(N), VT: MVT::i64);
8009 return true;
8010}
8011
8012/// Select register plus register addressing mode for SVE, with scaled
8013/// offset.
8014bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
8015 SDValue &Base,
8016 SDValue &Offset) {
8017 if (N.getOpcode() != ISD::ADD)
8018 return false;
8019
8020 // Process an ADD node.
8021 const SDValue LHS = N.getOperand(i: 0);
8022 const SDValue RHS = N.getOperand(i: 1);
8023
8024 // 8 bit data does not come with the SHL node, so it is treated
8025 // separately.
8026 if (Scale == 0) {
8027 Base = LHS;
8028 Offset = RHS;
8029 return true;
8030 }
8031
8032 if (auto C = dyn_cast<ConstantSDNode>(Val: RHS)) {
8033 int64_t ImmOff = C->getSExtValue();
8034 unsigned Size = 1 << Scale;
8035
8036 // To use the reg+reg addressing mode, the immediate must be a multiple of
8037 // the vector element's byte size.
8038 if (ImmOff % Size)
8039 return false;
8040
8041 SDLoc DL(N);
8042 Base = LHS;
8043 Offset = CurDAG->getTargetConstant(Val: ImmOff >> Scale, DL, VT: MVT::i64);
8044 SDValue Ops[] = {Offset};
8045 SDNode *MI = CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
8046 Offset = SDValue(MI, 0);
8047 return true;
8048 }
8049
8050 // Check if the RHS is a shift node with a constant.
8051 if (RHS.getOpcode() != ISD::SHL)
8052 return false;
8053
8054 const SDValue ShiftRHS = RHS.getOperand(i: 1);
8055 if (auto *C = dyn_cast<ConstantSDNode>(Val: ShiftRHS))
8056 if (C->getZExtValue() == Scale) {
8057 Base = LHS;
8058 Offset = RHS.getOperand(i: 0);
8059 return true;
8060 }
8061
8062 return false;
8063}
8064
8065bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8066 const AArch64TargetLowering *TLI =
8067 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8068
8069 return TLI->isAllActivePredicate(DAG: *CurDAG, N);
8070}
8071
8072bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8073 return N.getValueType().isScalableVectorOf(EltVT: MVT::i1);
8074}
8075
8076bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8077 SDValue &Base, SDValue &Offset,
8078 unsigned Scale) {
8079 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8080 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CN)) {
8081 int64_t ImmOff = C->getSExtValue();
8082 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8083 return CurDAG->getTargetConstant(Val: ImmOff / Scale, DL: SDLoc(N), VT: MVT::i64);
8084 }
8085 return SDValue();
8086 };
8087
8088 if (SDValue C = MatchConstantOffset(N)) {
8089 Base = getZeroRegister(DAG&: *CurDAG, DL: SDLoc(N), VT: MVT::i32);
8090 Offset = C;
8091 return true;
8092 }
8093
8094 // Try to untangle an ADD node into a 'reg + offset'
8095 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
8096 if (SDValue C = MatchConstantOffset(N.getOperand(i: 1))) {
8097 Base = N.getOperand(i: 0);
8098 Offset = C;
8099 return true;
8100 }
8101 }
8102
8103 // By default, just match reg + 0.
8104 Base = N;
8105 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
8106 return true;
8107}
8108
8109bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8110 SDValue &Imm) {
8111 AArch64CC::CondCode CC =
8112 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(Num: 1));
8113 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
8114 // Check conservatively if the immediate fits the valid range [0, 64).
8115 // Immediate variants for GE and HS definitely need to be decremented
8116 // when lowering the pseudos later, so an immediate of 1 would become 0.
8117 // For the inverse conditions LT and LO we don't know for sure if they
8118 // will need a decrement but should the decision be made to reverse the
8119 // branch condition, we again end up with the need to decrement.
8120 // The same argument holds for LE, LS, GT and HI and possibly
8121 // incremented immediates. This can lead to slightly less optimal
8122 // codegen, e.g. we never codegen the legal case
8123 // cblt w0, #63, A
8124 // because we could end up with the illegal case
8125 // cbge w0, #64, B
8126 // should the decision to reverse the branch direction be made. For the
8127 // lower bound cases this is no problem since we can express comparisons
8128 // against 0 with either tbz/tnbz or using wzr/xzr.
8129 uint64_t LowerBound = 0, UpperBound = 64;
8130 switch (CC) {
8131 case AArch64CC::GE:
8132 case AArch64CC::HS:
8133 case AArch64CC::LT:
8134 case AArch64CC::LO:
8135 LowerBound = 1;
8136 break;
8137 case AArch64CC::LE:
8138 case AArch64CC::LS:
8139 case AArch64CC::GT:
8140 case AArch64CC::HI:
8141 UpperBound = 63;
8142 break;
8143 default:
8144 break;
8145 }
8146
8147 if (CN->getAPIntValue().uge(RHS: LowerBound) &&
8148 CN->getAPIntValue().ult(RHS: UpperBound)) {
8149 SDLoc DL(N);
8150 Imm = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: N.getValueType());
8151 return true;
8152 }
8153 }
8154
8155 return false;
8156}
8157
8158template <bool MatchCBB>
8159bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8160 SDValue &ExtType) {
8161
8162 // Use an invalid shift-extend value to indicate we don't need to extend later
8163 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8164 EVT Ty = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
8165 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8166 return false;
8167 Reg = N.getOperand(i: 0);
8168 ExtType = CurDAG->getSignedTargetConstant(Val: AArch64_AM::InvalidShiftExtend,
8169 DL: SDLoc(N), VT: MVT::i32);
8170 return true;
8171 }
8172
8173 AArch64_AM::ShiftExtendType ET = getExtendTypeForNode(N);
8174
8175 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8176 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8177 Reg = N.getOperand(i: 0);
8178 ExtType =
8179 CurDAG->getTargetConstant(Val: getExtendEncoding(ET), DL: SDLoc(N), VT: MVT::i32);
8180 return true;
8181 }
8182
8183 return false;
8184}
8185
8186/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8187///
8188/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8189/// doing this earlier based on the defining operation of X can be invalidated
8190/// by later DAG combines. At this point the DAG is being prepared for
8191/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8192/// final SDValue being selected.
8193/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8194/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8195/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8196/// differ from fcmp+fcsel for signaling NaN inputs.
8197bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *N) {
8198 EVT VT = N->getValueType(ResNo: 0);
8199
8200 // Scalar FP only.
8201 if (!VT.isFloatingPoint() || VT.isVector())
8202 return false;
8203
8204 SDValue TVal = N->getOperand(Num: 0);
8205 SDValue FVal = N->getOperand(Num: 1);
8206 SDValue CCVal = N->getOperand(Num: 2);
8207 SDValue Cmp = N->getOperand(Num: 3);
8208
8209 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8210 return false;
8211
8212 auto *CC = dyn_cast<ConstantSDNode>(Val&: CCVal);
8213 if (!CC)
8214 return false;
8215
8216 SDValue CmpLHS = Cmp.getOperand(i: 0);
8217 SDValue CmpRHS = Cmp.getOperand(i: 1);
8218 unsigned CondCode = CC->getZExtValue();
8219
8220 // Map VT and operation (max/min) to machine opcode.
8221 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8222 if (VT == MVT::f16)
8223 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8224 else if (VT == MVT::f32)
8225 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8226 else if (VT == MVT::f64)
8227 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8228 else
8229 return 0; // unsupported
8230 };
8231
8232 // Determine whether to use max or min based on condition code and operands.
8233 bool isMax;
8234 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8235 if (TVal == CmpLHS && FVal == CmpRHS)
8236 isMax = true;
8237 else
8238 return false;
8239 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8240 if (TVal == CmpLHS && FVal == CmpRHS)
8241 isMax = false;
8242 else
8243 return false;
8244 } else {
8245 return false;
8246 }
8247
8248 // Get the machine opcode for this VT and operation.
8249 unsigned Opc = getOpc(VT, isMax);
8250 if (!Opc)
8251 return false;
8252
8253 // Constant must be non-NaN.
8254 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: CmpRHS);
8255 if (!CFP || CFP->getValueAPF().isNaN())
8256 return false;
8257
8258 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8259 // fcmp+select's -0. For non-zero constants, semantics are identical.
8260 if (CFP->isZero() && !N->getFlags().hasNoSignedZeros())
8261 return false;
8262
8263 // Only fold if variable operand is never sNaN.
8264 // This runs after DAG combines, so later combines cannot remove a defining
8265 // operation used by isKnownNeverSNaN().
8266 if (!CurDAG->isKnownNeverSNaN(Op: CmpLHS))
8267 return false;
8268
8269 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Op1: CmpLHS, Op2: CmpRHS);
8270 return true;
8271}
8272
8273void AArch64DAGToDAGISel::PreprocessISelDAG() {
8274 bool MadeChange = false;
8275 for (SDNode &N : llvm::make_early_inc_range(Range: CurDAG->allnodes())) {
8276 if (N.use_empty())
8277 continue;
8278
8279 SDValue Result;
8280 switch (N.getOpcode()) {
8281 case ISD::SCALAR_TO_VECTOR: {
8282 EVT ScalarTy = N.getValueType(ResNo: 0).getVectorElementType();
8283 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8284 ScalarTy == N.getOperand(Num: 0).getValueType())
8285 Result = addBitcastHints(DAG&: *CurDAG, N);
8286
8287 break;
8288 }
8289 case AArch64ISD::VSHL: {
8290 // Undo mul(shl(A,C),B) -> shl(mul(A,B),C) canonicalisation when A is an
8291 // extend that can be folded into the shift.
8292 EVT VT = N.getValueType(ResNo: 0);
8293 SDValue A, B, C = N.getOperand(Num: 1);
8294 if (sd_match(N: N.getOperand(Num: 0),
8295 P: m_OneUse(P: m_Mul(L: m_Value(N&: A, P: m_AnyOf(preds: m_ZExt(Op: m_Value()),
8296 preds: m_SExt(Op: m_Value()))),
8297 R: m_Value(N&: B))))) {
8298 // If both mul operands are extended, preserve the smull/umull idiom.
8299 if (B.getOpcode() == A.getOpcode())
8300 break;
8301 SDLoc DL(&N);
8302 SDValue SHL = CurDAG->getNode(Opcode: AArch64ISD::VSHL, DL, VT, N1: A, N2: C);
8303 Result = CurDAG->getNode(Opcode: ISD::MUL, DL, VT, N1: SHL, N2: B);
8304 }
8305 break;
8306 }
8307 default:
8308 break;
8309 }
8310
8311 if (Result) {
8312 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8313 LLVM_DEBUG(N.dump(CurDAG));
8314 LLVM_DEBUG(dbgs() << "\nNew: ");
8315 LLVM_DEBUG(Result.dump(CurDAG));
8316 LLVM_DEBUG(dbgs() << "\n");
8317
8318 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(&N, 0), To: Result);
8319 MadeChange = true;
8320 }
8321 }
8322
8323 if (MadeChange)
8324 CurDAG->RemoveDeadNodes();
8325
8326 SelectionDAGISel::PreprocessISelDAG();
8327}
8328