1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64TargetMachine.h"
17#include "MCTargetDesc/AArch64AddressingModes.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/CodeGen/ISDOpcodes.h"
20#include "llvm/CodeGen/SDPatternMatch.h"
21#include "llvm/CodeGen/SelectionDAGISel.h"
22#include "llvm/IR/Function.h" // To access function attributes.
23#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/Support/ErrorHandling.h"
28#include "llvm/Support/KnownBits.h"
29#include "llvm/Support/MathExtras.h"
30#include "llvm/Support/raw_ostream.h"
31
32using namespace llvm;
33using namespace llvm::SDPatternMatch;
34
35#define DEBUG_TYPE "aarch64-isel"
36#define PASS_NAME "AArch64 Instruction Selection"
37
38// https://github.com/llvm/llvm-project/issues/114425
39#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
40#pragma inline_depth(0)
41#endif
42
43//===--------------------------------------------------------------------===//
44/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
45/// instructions for SelectionDAG operations.
46///
47namespace {
48
49class AArch64DAGToDAGISel : public SelectionDAGISel {
50
51 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
52 /// make the right decision when generating code for different targets.
53 const AArch64Subtarget *Subtarget;
54
55public:
56 AArch64DAGToDAGISel() = delete;
57
58 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
59 CodeGenOptLevel OptLevel)
60 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
61
62 bool runOnMachineFunction(MachineFunction &MF) override {
63 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
64 return SelectionDAGISel::runOnMachineFunction(mf&: MF);
65 }
66
67 void Select(SDNode *Node) override;
68 void PreprocessISelDAG() override;
69
70 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
71 /// inline asm expressions.
72 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
73 InlineAsm::ConstraintCode ConstraintID,
74 std::vector<SDValue> &OutOps) override;
75
76 template <signed Low, signed High, signed Scale>
77 bool SelectRDVLImm(SDValue N, SDValue &Imm);
78
79 template <signed Low, signed High>
80 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
81
82 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
83 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
84 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
85 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
86 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
87 return SelectShiftedRegister(N, AllowROR: false, Reg, Shift);
88 }
89 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
90 return SelectShiftedRegister(N, AllowROR: true, Reg, Shift);
91 }
92 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
93 return SelectAddrModeIndexed7S(N, Size: 1, Base, OffImm);
94 }
95 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
96 return SelectAddrModeIndexed7S(N, Size: 2, Base, OffImm);
97 }
98 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
99 return SelectAddrModeIndexed7S(N, Size: 4, Base, OffImm);
100 }
101 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
102 return SelectAddrModeIndexed7S(N, Size: 8, Base, OffImm);
103 }
104 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
105 return SelectAddrModeIndexed7S(N, Size: 16, Base, OffImm);
106 }
107 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
108 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 9, Size: 16, Base, OffImm);
109 }
110 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
111 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: false, BW: 6, Size: 16, Base, OffImm);
112 }
113 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
114 return SelectAddrModeIndexed(N, Size: 1, Base, OffImm);
115 }
116 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
117 return SelectAddrModeIndexed(N, Size: 2, Base, OffImm);
118 }
119 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
120 return SelectAddrModeIndexed(N, Size: 4, Base, OffImm);
121 }
122 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
123 return SelectAddrModeIndexed(N, Size: 8, Base, OffImm);
124 }
125 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
126 return SelectAddrModeIndexed(N, Size: 16, Base, OffImm);
127 }
128 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
129 return SelectAddrModeUnscaled(N, Size: 1, Base, OffImm);
130 }
131 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
132 return SelectAddrModeUnscaled(N, Size: 2, Base, OffImm);
133 }
134 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
135 return SelectAddrModeUnscaled(N, Size: 4, Base, OffImm);
136 }
137 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
138 return SelectAddrModeUnscaled(N, Size: 8, Base, OffImm);
139 }
140 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
141 return SelectAddrModeUnscaled(N, Size: 16, Base, OffImm);
142 }
143 template <unsigned Size, unsigned Max>
144 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
145 // Test if there is an appropriate addressing mode and check if the
146 // immediate fits.
147 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
148 if (Found) {
149 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: OffImm)) {
150 int64_t C = CI->getSExtValue();
151 if (C <= Max)
152 return true;
153 }
154 }
155
156 // Otherwise, base only, materialize address in register.
157 Base = N;
158 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
159 return true;
160 }
161
162 template<int Width>
163 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
164 SDValue &SignExtend, SDValue &DoShift) {
165 return SelectAddrModeWRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
166 }
167
168 template<int Width>
169 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
170 SDValue &SignExtend, SDValue &DoShift) {
171 return SelectAddrModeXRO(N, Size: Width / 8, Base, Offset, SignExtend, DoShift);
172 }
173
174 bool SelectExtractHigh(SDValue N, SDValue &Res) {
175 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
176 N = N->getOperand(Num: 0);
177 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
178 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
179 return false;
180 EVT VT = N->getValueType(ResNo: 0);
181 EVT LVT = N->getOperand(Num: 0).getValueType();
182 unsigned Index = N->getConstantOperandVal(Num: 1);
183 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
184 Index != VT.getVectorNumElements())
185 return false;
186 Res = N->getOperand(Num: 0);
187 return true;
188 }
189
190 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
191 if (N.getOpcode() != AArch64ISD::VLSHR)
192 return false;
193 SDValue Op = N->getOperand(Num: 0);
194 EVT VT = Op.getValueType();
195 unsigned ShtAmt = N->getConstantOperandVal(Num: 1);
196 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
197 return false;
198
199 APInt Imm;
200 if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::MOVIshift)
201 Imm = APInt(VT.getScalarSizeInBits(),
202 Op.getOperand(i: 1).getConstantOperandVal(i: 0)
203 << Op.getOperand(i: 1).getConstantOperandVal(i: 1));
204 else if (Op.getOperand(i: 1).getOpcode() == AArch64ISD::DUP &&
205 isa<ConstantSDNode>(Val: Op.getOperand(i: 1).getOperand(i: 0)))
206 Imm = APInt(VT.getScalarSizeInBits(),
207 Op.getOperand(i: 1).getConstantOperandVal(i: 0));
208 else
209 return false;
210
211 if (Imm != 1ULL << (ShtAmt - 1))
212 return false;
213
214 Res1 = Op.getOperand(i: 0);
215 Res2 = CurDAG->getTargetConstant(Val: ShtAmt, DL: SDLoc(N), VT: MVT::i32);
216 return true;
217 }
218
219 bool SelectDupZeroOrUndef(SDValue N) {
220 switch(N->getOpcode()) {
221 case ISD::UNDEF:
222 case ISD::POISON:
223 return true;
224 case AArch64ISD::DUP:
225 case ISD::SPLAT_VECTOR: {
226 auto Opnd0 = N->getOperand(Num: 0);
227 if (isNullConstant(V: Opnd0))
228 return true;
229 if (isNullFPConstant(V: Opnd0))
230 return true;
231 break;
232 }
233 default:
234 break;
235 }
236
237 return false;
238 }
239
240 bool SelectAny(SDValue) { return true; }
241
242 bool SelectDupZero(SDValue N) {
243 switch(N->getOpcode()) {
244 case AArch64ISD::DUP:
245 case ISD::SPLAT_VECTOR: {
246 auto Opnd0 = N->getOperand(Num: 0);
247 if (isNullConstant(V: Opnd0))
248 return true;
249 if (isNullFPConstant(V: Opnd0))
250 return true;
251 break;
252 }
253 }
254
255 return false;
256 }
257
258 template <MVT::SimpleValueType VT, bool Negate>
259 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
260 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
261 }
262
263 template <MVT::SimpleValueType VT, bool Negate>
264 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
265 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
266 }
267
268 template <MVT::SimpleValueType VT>
269 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
270 return SelectSVECpyDupImm(N, VT, Imm, Shift);
271 }
272
273 template <MVT::SimpleValueType VT, bool Invert = false>
274 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
275 return SelectSVELogicalImm(N, VT, Imm, Invert);
276 }
277
278 template <MVT::SimpleValueType VT>
279 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
280 return SelectSVEArithImm(N, VT, Imm);
281 }
282
283 template <unsigned Low, unsigned High, bool AllowSaturation = false>
284 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
285 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
286 }
287
288 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
289 if (N->getOpcode() != ISD::SPLAT_VECTOR)
290 return false;
291
292 EVT EltVT = N->getValueType(ResNo: 0).getVectorElementType();
293 return SelectSVEShiftImm(N: N->getOperand(Num: 0), /* Low */ 1,
294 /* High */ EltVT.getFixedSizeInBits(),
295 /* AllowSaturation */ true, Imm);
296 }
297
298 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
299 template<signed Min, signed Max, signed Scale, bool Shift>
300 bool SelectCntImm(SDValue N, SDValue &Imm) {
301 if (!isa<ConstantSDNode>(Val: N))
302 return false;
303
304 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
305 if (Shift)
306 MulImm = 1LL << MulImm;
307
308 if ((MulImm % std::abs(x: Scale)) != 0)
309 return false;
310
311 MulImm /= Scale;
312 if ((MulImm >= Min) && (MulImm <= Max)) {
313 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
314 return true;
315 }
316
317 return false;
318 }
319
320 template <signed Max, signed Scale>
321 bool SelectEXTImm(SDValue N, SDValue &Imm) {
322 if (!isa<ConstantSDNode>(Val: N))
323 return false;
324
325 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
326
327 if (MulImm >= 0 && MulImm <= Max) {
328 MulImm *= Scale;
329 Imm = CurDAG->getTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
330 return true;
331 }
332
333 return false;
334 }
335
336 template <unsigned BaseReg, unsigned Max>
337 bool ImmToReg(SDValue N, SDValue &Imm) {
338 if (auto *CI = dyn_cast<ConstantSDNode>(Val&: N)) {
339 uint64_t C = CI->getZExtValue();
340
341 if (C > Max)
342 return false;
343
344 Imm = CurDAG->getRegister(Reg: BaseReg + C, VT: MVT::Other);
345 return true;
346 }
347 return false;
348 }
349
350 /// Form sequences of consecutive 64/128-bit registers for use in NEON
351 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
352 /// between 1 and 4 elements. If it contains a single element that is returned
353 /// unchanged; otherwise a REG_SEQUENCE value is returned.
354 SDValue createDTuple(ArrayRef<SDValue> Vecs);
355 SDValue createQTuple(ArrayRef<SDValue> Vecs);
356 // Form a sequence of SVE registers for instructions using list of vectors,
357 // e.g. structured loads and stores (ldN, stN).
358 SDValue createZTuple(ArrayRef<SDValue> Vecs);
359
360 // Similar to above, except the register must start at a multiple of the
361 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
362 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
363
364 /// Generic helper for the createDTuple/createQTuple
365 /// functions. Those should almost always be called instead.
366 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
367 const unsigned SubRegs[]);
368
369 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
370
371 bool tryIndexedLoad(SDNode *N);
372
373 void SelectPtrauthAuth(SDNode *N);
374 void SelectPtrauthResign(SDNode *N);
375 void SelectPtrauthResignWithPC(SDNode *N);
376
377 bool trySelectStackSlotTagP(SDNode *N);
378 void SelectTagP(SDNode *N);
379
380 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
381 unsigned SubRegIdx);
382 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
383 unsigned SubRegIdx);
384 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
385 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
386 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
387 unsigned Opc_rr, unsigned Opc_ri,
388 bool IsIntr = false);
389 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
390 unsigned Scale, unsigned Opc_ri,
391 unsigned Opc_rr);
392 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
393 bool IsZmMulti, unsigned Opcode,
394 bool HasPred = false);
395 void SelectPExtPair(SDNode *N, unsigned Opc);
396 void SelectWhilePair(SDNode *N, unsigned Opc);
397 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
398 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
399 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
400 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
401 bool IsTupleInput, unsigned Opc);
402 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
403
404 template <unsigned MaxIdx, unsigned Scale>
405 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
406 unsigned Op);
407 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
408 unsigned Op, unsigned MaxIdx, unsigned Scale,
409 unsigned BaseReg = 0);
410 /// SVE Reg+Imm addressing mode.
411 template <int64_t Min, int64_t Max>
412 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
413 SDValue &OffImm);
414 /// SVE Reg+Reg address mode.
415 template <unsigned Scale>
416 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
417 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
418 }
419
420 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
421 unsigned Opc, uint32_t MaxImm);
422 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
423
424 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
425 unsigned NumInVecs);
426
427 template <unsigned MaxIdx, unsigned Scale>
428 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
429 return SelectSMETileSlice(N, MaxSize: MaxIdx, Vector, Offset, Scale);
430 }
431
432 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
433 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
434 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
435 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
436 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
437 unsigned Opc_rr, unsigned Opc_ri);
438 std::tuple<unsigned, SDValue, SDValue>
439 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
440 const SDValue &OldBase, const SDValue &OldOffset,
441 unsigned Scale);
442
443 bool tryBitfieldExtractOp(SDNode *N);
444 bool tryBitfieldExtractOpFromSExt(SDNode *N);
445 bool tryBitfieldInsertOp(SDNode *N);
446 bool tryBitfieldInsertInZeroOp(SDNode *N);
447 bool tryShiftAmountMod(SDNode *N);
448
449 bool tryReadRegister(SDNode *N);
450 bool tryWriteRegister(SDNode *N);
451
452 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
453 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
454
455 bool trySelectXAR(SDNode *N);
456
457 bool tryFoldCselToFMaxMin(SDNode *N);
458
459// Include the pieces autogenerated from the target description.
460#include "AArch64GenDAGISel.inc"
461
462private:
463 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
464 SDValue &Shift);
465 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
466 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
467 SDValue &OffImm) {
468 return SelectAddrModeIndexedBitWidth(N, IsSignedImm: true, BW: 7, Size, Base, OffImm);
469 }
470 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
471 unsigned Size, SDValue &Base,
472 SDValue &OffImm);
473 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
474 SDValue &OffImm);
475 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
476 SDValue &OffImm);
477 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
478 SDValue &Offset, SDValue &SignExtend,
479 SDValue &DoShift);
480 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
481 SDValue &Offset, SDValue &SignExtend,
482 SDValue &DoShift);
483 bool isWorthNegatingImm(SDValue V) const;
484 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
485 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
486 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
487 SDValue &Offset, SDValue &SignExtend);
488
489 template<unsigned RegWidth>
490 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
491 return SelectCVTFixedPosOperand(N, FixedPos, Width: RegWidth);
492 }
493 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
494
495 template <unsigned RegWidth>
496 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
497 return SelectCVTFixedPointVec(N, FixedPos, Width: RegWidth);
498 }
499 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
500
501 template<unsigned RegWidth>
502 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
503 return SelectCVTFixedPosRecipOperand(N, FixedPos, Width: RegWidth);
504 }
505
506 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
507 unsigned Width);
508
509 template <unsigned FloatWidth>
510 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
511 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, Width: FloatWidth);
512 }
513
514 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
515 unsigned Width);
516
517 bool SelectCMP_SWAP(SDNode *N);
518
519 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
520 bool Negate);
521 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
522 SDValue &Shift, bool Negate);
523 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
524 bool Negate);
525 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
526 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
527
528 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
529 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
530 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
531 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
532 bool SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm);
533 bool SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm);
534
535 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
536 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
537 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
538 bool AllowSaturation, SDValue &Imm);
539
540 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
541 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
542 SDValue &Offset);
543 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
544 SDValue &Offset, unsigned Scale = 1);
545
546 bool SelectAllActivePredicate(SDValue N);
547 bool SelectAnyPredicate(SDValue N);
548
549 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
550
551 template <bool MatchCBB>
552 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
553};
554
555class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
556public:
557 static char ID;
558 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
559 CodeGenOptLevel OptLevel)
560 : SelectionDAGISelLegacy(
561 ID, std::make_unique<AArch64DAGToDAGISel>(args&: tm, args&: OptLevel)) {}
562};
563} // end anonymous namespace
564
565char AArch64DAGToDAGISelLegacy::ID = 0;
566
567INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
568
569AArch64DAGToDAGISelPass::AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
570 : SelectionDAGISelPass(
571 std::make_unique<AArch64DAGToDAGISel>(args&: TM, args: TM.getOptLevel())) {}
572
573/// addBitcastHints - This method adds bitcast hints to the operands of a node
574/// to help instruction selector determine which operands are in Neon registers.
575static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N) {
576 SDLoc DL(&N);
577 auto getFloatVT = [&](EVT VT) {
578 EVT ScalarVT = VT.getScalarType();
579 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
580 return VT.changeElementType(Context&: *(DAG.getContext()),
581 EltVT: ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
582 };
583 SmallVector<SDValue, 2> NewOps;
584 NewOps.reserve(N: N.getNumOperands());
585
586 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
587 auto bitcasted = DAG.getBitcast(VT: getFloatVT(N.getOperand(Num: I).getValueType()),
588 V: N.getOperand(Num: I));
589 NewOps.push_back(Elt: bitcasted);
590 }
591 EVT OrigVT = N.getValueType(ResNo: 0);
592 SDValue OpNode = DAG.getNode(Opcode: N.getOpcode(), DL, VT: getFloatVT(OrigVT), Ops: NewOps);
593 return DAG.getBitcast(VT: OrigVT, V: OpNode);
594}
595
596/// isIntImmediate - This method tests to see if the node is a constant
597/// operand. If so Imm will receive the 64-bit value.
598static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
599 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(Val: N)) {
600 Imm = C->getZExtValue();
601 return true;
602 }
603 return false;
604}
605
606// isIntImmediate - This method tests to see if a constant operand.
607// If so Imm will receive the value.
608static bool isIntImmediate(SDValue N, uint64_t &Imm) {
609 return isIntImmediate(N: N.getNode(), Imm);
610}
611
612// isOpcWithIntImmediate - This method tests to see if the node is a specific
613// opcode and that it has a immediate integer right operand.
614// If so Imm will receive the 32 bit value.
615static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
616 uint64_t &Imm) {
617 return N->getOpcode() == Opc &&
618 isIntImmediate(N: N->getOperand(Num: 1).getNode(), Imm);
619}
620
621// isIntImmediateEq - This method tests to see if N is a constant operand that
622// is equivalent to 'ImmExpected'.
623#ifndef NDEBUG
624static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
625 uint64_t Imm;
626 if (!isIntImmediate(N.getNode(), Imm))
627 return false;
628 return Imm == ImmExpected;
629}
630#endif
631
632static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
633 assert(RegWidth == 32 || RegWidth == 64);
634 if (RegWidth == 32)
635 return APInt(RegWidth,
636 uint32_t(AArch64_AM::decodeAdvSIMDModImmType11(Imm)));
637 return APInt(RegWidth, AArch64_AM::decodeAdvSIMDModImmType12(Imm));
638}
639
640// Decodes the raw integer splat value from a NEON splat operation.
641static std::optional<APInt> DecodeNEONSplat(SDValue N) {
642 assert(N.getValueType().isInteger() && "Only integers are supported");
643 if (N->getOpcode() == AArch64ISD::NVCAST)
644 N = N->getOperand(Num: 0);
645 unsigned SplatWidth = N.getScalarValueSizeInBits();
646 if (N.getOpcode() == AArch64ISD::FMOV)
647 return DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth: SplatWidth);
648 if (N->getOpcode() == AArch64ISD::MOVI)
649 return APInt(SplatWidth, N.getConstantOperandVal(i: 0));
650 if (N->getOpcode() == AArch64ISD::MOVIshift)
651 return APInt(SplatWidth, N.getConstantOperandVal(i: 0)
652 << N.getConstantOperandVal(i: 1));
653 if (N->getOpcode() == AArch64ISD::MVNIshift)
654 return ~APInt(SplatWidth, N.getConstantOperandVal(i: 0)
655 << N.getConstantOperandVal(i: 1));
656 if (N->getOpcode() == AArch64ISD::MOVIedit)
657 return APInt(SplatWidth, AArch64_AM::decodeAdvSIMDModImmType10(
658 Imm: N.getConstantOperandVal(i: 0)));
659 if (N->getOpcode() == AArch64ISD::DUP)
660 if (auto *Const = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0)))
661 return Const->getAPIntValue().trunc(width: SplatWidth);
662 APInt SplatVal;
663 if (ISD::isConstantSplatVector(N: N.getNode(), SplatValue&: SplatVal))
664 return SplatVal.trunc(width: SplatWidth);
665 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
666 // in AArch64ISelLowering.
667 return std::nullopt;
668}
669
670// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
671// matching the element size of N.
672static std::optional<APInt> GetNEONSplatValue(SDValue N) {
673 unsigned SplatWidth = N.getScalarValueSizeInBits();
674 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N)) {
675 if (SplatVal->getBitWidth() <= SplatWidth)
676 return APInt::getSplat(NewLen: SplatWidth, V: *SplatVal);
677 if (SplatVal->isSplat(SplatSizeInBits: SplatWidth))
678 return SplatVal->trunc(width: SplatWidth);
679 }
680 return std::nullopt;
681}
682
683bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
684 SDValue &Imm) {
685 std::optional<APInt> ImmVal = GetNEONSplatValue(N);
686 if (!ImmVal)
687 return false;
688 uint64_t Encoding;
689 if (!AArch64_AM::isSVELogicalImm(SizeInBits: N.getScalarValueSizeInBits(),
690 ImmVal: ImmVal->getZExtValue(), Encoding))
691 return false;
692
693 Imm = CurDAG->getTargetConstant(Val: Encoding, DL: SDLoc(N), VT: MVT::i64);
694 return true;
695}
696
697bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
698 SDValue &Shift) {
699 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N))
700 return SelectSVEAddSubImm(DL: SDLoc(N), Value: *ImmVal,
701 VT: N.getValueType().getScalarType().getSimpleVT(),
702 Imm, Shift,
703 /*Negate=*/false);
704 return false;
705}
706
707bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
708 SDValue &Imm) {
709 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N))
710 return SelectSVESignedArithImm(DL: SDLoc(N), Value: *ImmVal, Imm);
711 return false;
712}
713
714bool AArch64DAGToDAGISel::SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm) {
715 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N);
716 if (!ImmAPIntVal)
717 return false;
718
719 int64_t ImmVal = ImmAPIntVal->getSExtValue();
720 if (ImmVal < -128 || ImmVal > 127)
721 return false;
722
723 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
724 return true;
725}
726
727bool AArch64DAGToDAGISel::SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm) {
728 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N);
729 if (!ImmAPIntVal)
730 return false;
731
732 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
733 if (ImmVal > 255)
734 return false;
735
736 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
737 return true;
738}
739
740bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
741 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
742 std::vector<SDValue> &OutOps) {
743 switch(ConstraintID) {
744 default:
745 llvm_unreachable("Unexpected asm memory constraint");
746 case InlineAsm::ConstraintCode::m:
747 case InlineAsm::ConstraintCode::o:
748 case InlineAsm::ConstraintCode::Q:
749 // We need to make sure that this one operand does not end up in XZR, thus
750 // require the address to be in a PointerRegClass register.
751 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
752 const TargetRegisterClass *TRC = TRI->getPointerRegClass();
753 SDLoc dl(Op);
754 SDValue RC = CurDAG->getTargetConstant(Val: TRC->getID(), DL: dl, VT: MVT::i64);
755 SDValue NewOp =
756 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS,
757 dl, VT: Op.getValueType(),
758 Op1: Op, Op2: RC), 0);
759 OutOps.push_back(x: NewOp);
760 return false;
761 }
762 return true;
763}
764
765/// SelectArithImmed - Select an immediate value that can be represented as
766/// a 12-bit value shifted left by either 0 or 12. If so, return true with
767/// Val set to the 12-bit value and Shift set to the shifter operand.
768bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
769 SDValue &Shift) {
770 // This function is called from the addsub_shifted_imm ComplexPattern,
771 // which lists [imm] as the list of opcode it's interested in, however
772 // we still need to check whether the operand is actually an immediate
773 // here because the ComplexPattern opcode list is only used in
774 // root-level opcode matching.
775 if (!isa<ConstantSDNode>(Val: N.getNode()))
776 return false;
777
778 uint64_t Immed = N.getNode()->getAsZExtVal();
779
780 if (!AArch64_AM::isLegalArithImmed(C: Immed))
781 return false;
782
783 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(C: Immed);
784 Immed >>= ShiftAmt;
785
786 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShiftAmt);
787 SDLoc dl(N);
788 Val = CurDAG->getTargetConstant(Val: Immed, DL: dl, VT: MVT::i32);
789 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: dl, VT: MVT::i32);
790 return true;
791}
792
793/// SelectNegArithImmed - As above, but negates the value before trying to
794/// select it.
795bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
796 SDValue &Shift) {
797 // This function is called from the addsub_shifted_imm ComplexPattern,
798 // which lists [imm] as the list of opcode it's interested in, however
799 // we still need to check whether the operand is actually an immediate
800 // here because the ComplexPattern opcode list is only used in
801 // root-level opcode matching.
802 if (!isa<ConstantSDNode>(Val: N.getNode()))
803 return false;
804
805 // The immediate operand must be a 24-bit zero-extended immediate.
806 uint64_t Immed = N.getNode()->getAsZExtVal();
807
808 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
809 // have the opposite effect on the C flag, so this pattern mustn't match under
810 // those circumstances.
811 if (Immed == 0)
812 return false;
813
814 if (N.getValueType() == MVT::i32)
815 Immed = ~((uint32_t)Immed) + 1;
816 else
817 Immed = ~Immed + 1ULL;
818 if (Immed & 0xFFFFFFFFFF000000ULL)
819 return false;
820
821 Immed &= 0xFFFFFFULL;
822 return SelectArithImmed(N: CurDAG->getConstant(Val: Immed, DL: SDLoc(N), VT: MVT::i32), Val,
823 Shift);
824}
825
826/// getShiftTypeForNode - Translate a shift node to the corresponding
827/// ShiftType value.
828static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
829 switch (N.getOpcode()) {
830 default:
831 return AArch64_AM::InvalidShiftExtend;
832 case ISD::SHL:
833 return AArch64_AM::LSL;
834 case ISD::SRL:
835 return AArch64_AM::LSR;
836 case ISD::SRA:
837 return AArch64_AM::ASR;
838 case ISD::ROTR:
839 return AArch64_AM::ROR;
840 }
841}
842
843static bool isMemOpOrPrefetch(SDNode *N) {
844 return isa<MemSDNode>(Val: *N) || N->getOpcode() == AArch64ISD::PREFETCH;
845}
846
847/// Determine whether it is worth it to fold SHL into the addressing
848/// mode.
849static bool isWorthFoldingSHL(SDValue V) {
850 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
851 // It is worth folding logical shift of up to three places.
852 auto *CSD = dyn_cast<ConstantSDNode>(Val: V.getOperand(i: 1));
853 if (!CSD)
854 return false;
855 unsigned ShiftVal = CSD->getZExtValue();
856 if (ShiftVal > 3)
857 return false;
858
859 // Check if this particular node is reused in any non-memory related
860 // operation. If yes, do not try to fold this node into the address
861 // computation, since the computation will be kept.
862 const SDNode *Node = V.getNode();
863 for (SDNode *UI : Node->users())
864 if (!isMemOpOrPrefetch(N: UI))
865 for (SDNode *UII : UI->users())
866 if (!isMemOpOrPrefetch(N: UII))
867 return false;
868 return true;
869}
870
871/// Determine whether it is worth to fold V into an extended register addressing
872/// mode.
873bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
874 // Trivial if we are optimizing for code size or if there is only
875 // one use of the value.
876 if (CurDAG->shouldOptForSize() || V.hasOneUse())
877 return true;
878
879 // If a subtarget has a slow shift, folding a shift into multiple loads
880 // costs additional micro-ops.
881 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
882 return false;
883
884 // Check whether we're going to emit the address arithmetic anyway because
885 // it's used by a non-address operation.
886 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
887 return true;
888 if (V.getOpcode() == ISD::ADD) {
889 const SDValue LHS = V.getOperand(i: 0);
890 const SDValue RHS = V.getOperand(i: 1);
891 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: LHS))
892 return true;
893 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(V: RHS))
894 return true;
895 }
896
897 // It hurts otherwise, since the value will be reused.
898 return false;
899}
900
901/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
902/// to select more shifted register
903bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
904 SDValue &Shift) {
905 EVT VT = N.getValueType();
906 if (VT != MVT::i32 && VT != MVT::i64)
907 return false;
908
909 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
910 return false;
911 SDValue LHS = N.getOperand(i: 0);
912 if (!LHS->hasOneUse())
913 return false;
914
915 unsigned LHSOpcode = LHS->getOpcode();
916 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
917 return false;
918
919 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
920 if (!ShiftAmtNode)
921 return false;
922
923 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
924 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
925 if (!RHSC)
926 return false;
927
928 APInt AndMask = RHSC->getAPIntValue();
929 unsigned LowZBits, MaskLen;
930 if (!AndMask.isShiftedMask(MaskIdx&: LowZBits, MaskLen))
931 return false;
932
933 unsigned BitWidth = N.getValueSizeInBits();
934 SDLoc DL(LHS);
935 uint64_t NewShiftC;
936 unsigned NewShiftOp;
937 if (LHSOpcode == ISD::SHL) {
938 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
939 // BitWidth != LowZBits + MaskLen doesn't match the pattern
940 if (LowZBits <= ShiftAmtC || (BitWidth != LowZBits + MaskLen))
941 return false;
942
943 NewShiftC = LowZBits - ShiftAmtC;
944 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
945 } else {
946 if (LowZBits == 0)
947 return false;
948
949 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
950 NewShiftC = LowZBits + ShiftAmtC;
951 if (NewShiftC >= BitWidth)
952 return false;
953
954 // SRA need all high bits
955 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
956 return false;
957
958 // SRL high bits can be 0 or 1
959 if (LHSOpcode == ISD::SRL && (BitWidth > (NewShiftC + MaskLen)))
960 return false;
961
962 if (LHSOpcode == ISD::SRL)
963 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
964 else
965 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
966 }
967
968 assert(NewShiftC < BitWidth && "Invalid shift amount");
969 SDValue NewShiftAmt = CurDAG->getTargetConstant(Val: NewShiftC, DL, VT);
970 SDValue BitWidthMinus1 = CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT);
971 Reg = SDValue(CurDAG->getMachineNode(Opcode: NewShiftOp, dl: DL, VT, Op1: LHS->getOperand(Num: 0),
972 Op2: NewShiftAmt, Op3: BitWidthMinus1),
973 0);
974 unsigned ShVal = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LowZBits);
975 Shift = CurDAG->getTargetConstant(Val: ShVal, DL, VT: MVT::i32);
976 return true;
977}
978
979/// getExtendTypeForNode - Translate an extend node to the corresponding
980/// ExtendType value.
981static AArch64_AM::ShiftExtendType
982getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
983 if (N.getOpcode() == ISD::SIGN_EXTEND ||
984 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
985 EVT SrcVT;
986 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
987 SrcVT = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
988 else
989 SrcVT = N.getOperand(i: 0).getValueType();
990
991 if (!IsLoadStore && SrcVT == MVT::i8)
992 return AArch64_AM::SXTB;
993 else if (!IsLoadStore && SrcVT == MVT::i16)
994 return AArch64_AM::SXTH;
995 else if (SrcVT == MVT::i32)
996 return AArch64_AM::SXTW;
997 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
998
999 return AArch64_AM::InvalidShiftExtend;
1000 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
1001 N.getOpcode() == ISD::ANY_EXTEND) {
1002 EVT SrcVT = N.getOperand(i: 0).getValueType();
1003 if (!IsLoadStore && SrcVT == MVT::i8)
1004 return AArch64_AM::UXTB;
1005 else if (!IsLoadStore && SrcVT == MVT::i16)
1006 return AArch64_AM::UXTH;
1007 else if (SrcVT == MVT::i32)
1008 return AArch64_AM::UXTW;
1009 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1010
1011 return AArch64_AM::InvalidShiftExtend;
1012 } else if (N.getOpcode() == ISD::AND) {
1013 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1014 if (!CSD)
1015 return AArch64_AM::InvalidShiftExtend;
1016 uint64_t AndMask = CSD->getZExtValue();
1017
1018 switch (AndMask) {
1019 default:
1020 return AArch64_AM::InvalidShiftExtend;
1021 case 0xFF:
1022 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
1023 case 0xFFFF:
1024 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
1025 case 0xFFFFFFFF:
1026 return AArch64_AM::UXTW;
1027 }
1028 }
1029
1030 return AArch64_AM::InvalidShiftExtend;
1031}
1032
1033/// Determine whether constant -V is cheaper to materialise than V.
1034bool AArch64DAGToDAGISel::isWorthNegatingImm(SDValue V) const {
1035 assert(isa<ConstantSDNode>(V) && "invalid node");
1036
1037 EVT VT = V.getValueType();
1038 assert((VT == MVT::i32 || VT == MVT::i64) && "invalid type");
1039
1040 // It's only worth negating the constant if it doesn't have other uses.
1041 if (!V.hasOneUse())
1042 return false;
1043
1044 uint64_t Imm = cast<ConstantSDNode>(Val&: V)->getZExtValue();
1045 unsigned BitSize = VT.getSizeInBits();
1046 SmallVector<AArch64_IMM::ImmInsnModel, 4> OrigCost, NewCost;
1047 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: OrigCost);
1048 AArch64_IMM::expandMOVImm(Imm: -Imm, BitSize, Insn&: NewCost);
1049 return NewCost.size() < OrigCost.size();
1050}
1051
1052/// Determine whether it is worth to fold V into an extended register of an
1053/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
1054/// instruction, and the shift should be treated as worth folding even if has
1055/// multiple uses.
1056bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1057 // Trivial if we are optimizing for code size or if there is only
1058 // one use of the value.
1059 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1060 return true;
1061
1062 // If a subtarget has a fastpath LSL we can fold a logical shift into
1063 // the add/sub and save a cycle.
1064 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1065 V.getConstantOperandVal(i: 1) <= 4 &&
1066 getExtendTypeForNode(N: V.getOperand(i: 0)) == AArch64_AM::InvalidShiftExtend)
1067 return true;
1068
1069 // It hurts otherwise, since the value will be reused.
1070 return false;
1071}
1072
1073/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1074/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1075/// instructions allow the shifted register to be rotated, but the arithmetic
1076/// instructions do not. The AllowROR parameter specifies whether ROR is
1077/// supported.
1078bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1079 SDValue &Reg, SDValue &Shift) {
1080 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1081 return true;
1082
1083 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
1084 if (ShType == AArch64_AM::InvalidShiftExtend)
1085 return false;
1086 if (!AllowROR && ShType == AArch64_AM::ROR)
1087 return false;
1088
1089 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1090 unsigned BitSize = N.getValueSizeInBits();
1091 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1092 unsigned ShVal = AArch64_AM::getShifterImm(ST: ShType, Imm: Val);
1093
1094 Reg = N.getOperand(i: 0);
1095 Shift = CurDAG->getTargetConstant(Val: ShVal, DL: SDLoc(N), VT: MVT::i32);
1096 return isWorthFoldingALU(V: N, LSL: true);
1097 }
1098
1099 return false;
1100}
1101
1102/// Instructions that accept extend modifiers like UXTW expect the register
1103/// being extended to be a GPR32, but the incoming DAG might be acting on a
1104/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1105/// this is the case.
1106static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
1107 if (N.getValueType() == MVT::i32)
1108 return N;
1109
1110 SDLoc dl(N);
1111 return CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i32, Operand: N);
1112}
1113
1114// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1115template<signed Low, signed High, signed Scale>
1116bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1117 if (!isa<ConstantSDNode>(Val: N))
1118 return false;
1119
1120 int64_t MulImm = cast<ConstantSDNode>(Val&: N)->getSExtValue();
1121 if ((MulImm % std::abs(x: Scale)) == 0) {
1122 int64_t RDVLImm = MulImm / Scale;
1123 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1124 Imm = CurDAG->getSignedTargetConstant(Val: RDVLImm, DL: SDLoc(N), VT: MVT::i32);
1125 return true;
1126 }
1127 }
1128
1129 return false;
1130}
1131
1132// Returns a suitable RDSVL multiplier from a left shift.
1133template <signed Low, signed High>
1134bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1135 if (!isa<ConstantSDNode>(Val: N))
1136 return false;
1137
1138 int64_t MulImm = 1LL << cast<ConstantSDNode>(Val&: N)->getSExtValue();
1139 if (MulImm >= Low && MulImm <= High) {
1140 Imm = CurDAG->getSignedTargetConstant(Val: MulImm, DL: SDLoc(N), VT: MVT::i32);
1141 return true;
1142 }
1143
1144 return false;
1145}
1146
1147/// SelectArithExtendedRegister - Select a "extended register" operand. This
1148/// operand folds in an extend followed by an optional left shift.
1149bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1150 SDValue &Shift) {
1151 unsigned ShiftVal = 0;
1152 AArch64_AM::ShiftExtendType Ext;
1153
1154 if (N.getOpcode() == ISD::SHL) {
1155 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1156 if (!CSD)
1157 return false;
1158 ShiftVal = CSD->getZExtValue();
1159 if (ShiftVal > 4)
1160 return false;
1161
1162 Ext = getExtendTypeForNode(N: N.getOperand(i: 0));
1163 if (Ext == AArch64_AM::InvalidShiftExtend)
1164 return false;
1165
1166 Reg = N.getOperand(i: 0).getOperand(i: 0);
1167 } else {
1168 Ext = getExtendTypeForNode(N);
1169 if (Ext == AArch64_AM::InvalidShiftExtend)
1170 return false;
1171
1172 // Don't match sext of vector extracts. These can use SMOV, but if we match
1173 // this as an extended register, we'll always fold the extend into an ALU op
1174 // user of the extend (which results in a UMOV).
1175 if (AArch64_AM::isSignExtendShiftType(Type: Ext)) {
1176 SDValue Op = N.getOperand(i: 0);
1177 if (Op->getOpcode() == ISD::ANY_EXTEND)
1178 Op = Op->getOperand(Num: 0);
1179 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1180 Op.getOperand(i: 0).getValueType().isFixedLengthVector())
1181 return false;
1182 }
1183
1184 Reg = N.getOperand(i: 0);
1185
1186 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1187 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1188 auto isDef32 = [](SDValue N) {
1189 unsigned Opc = N.getOpcode();
1190 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1191 Opc != ISD::CopyFromReg && Opc != ISD::AssertSext &&
1192 Opc != ISD::AssertZext && Opc != ISD::AssertAlign &&
1193 Opc != ISD::FREEZE;
1194 };
1195 if (Ext == AArch64_AM::UXTW && Reg->getValueType(ResNo: 0).getSizeInBits() == 32 &&
1196 isDef32(Reg))
1197 return false;
1198 }
1199
1200 // AArch64 mandates that the RHS of the operation must use the smallest
1201 // register class that could contain the size being extended from. Thus,
1202 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1203 // there might not be an actual 32-bit value in the program. We can
1204 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1205 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1206 Reg = narrowIfNeeded(CurDAG, N: Reg);
1207 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1208 VT: MVT::i32);
1209 return isWorthFoldingALU(V: N);
1210}
1211
1212/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1213/// operand is referred by the instructions have SP operand
1214bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1215 SDValue &Shift) {
1216 unsigned ShiftVal = 0;
1217 AArch64_AM::ShiftExtendType Ext;
1218
1219 if (N.getOpcode() != ISD::SHL)
1220 return false;
1221
1222 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1223 if (!CSD)
1224 return false;
1225 ShiftVal = CSD->getZExtValue();
1226 if (ShiftVal > 4)
1227 return false;
1228
1229 Ext = AArch64_AM::UXTX;
1230 Reg = N.getOperand(i: 0);
1231 Shift = CurDAG->getTargetConstant(Val: getArithExtendImm(ET: Ext, Imm: ShiftVal), DL: SDLoc(N),
1232 VT: MVT::i32);
1233 return isWorthFoldingALU(V: N);
1234}
1235
1236/// If there's a use of this ADDlow that's not itself a load/store then we'll
1237/// need to create a real ADD instruction from it anyway and there's no point in
1238/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1239/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1240/// leads to duplicated ADRP instructions.
1241static bool isWorthFoldingADDlow(SDValue N) {
1242 for (auto *User : N->users()) {
1243 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1244 User->getOpcode() != ISD::ATOMIC_LOAD &&
1245 User->getOpcode() != ISD::ATOMIC_STORE)
1246 return false;
1247
1248 // ldar and stlr have much more restrictive addressing modes (just a
1249 // register).
1250 if (isStrongerThanMonotonic(AO: cast<MemSDNode>(Val: User)->getSuccessOrdering()))
1251 return false;
1252 }
1253
1254 return true;
1255}
1256
1257/// Check if the immediate offset is valid as a scaled immediate.
1258static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1259 unsigned Size) {
1260 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1261 Offset < (Range << Log2_32(Value: Size)))
1262 return true;
1263 return false;
1264}
1265
1266/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1267/// immediate" address. The "Size" argument is the size in bytes of the memory
1268/// reference, which determines the scale.
1269bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1270 unsigned BW, unsigned Size,
1271 SDValue &Base,
1272 SDValue &OffImm) {
1273 SDLoc dl(N);
1274 const DataLayout &DL = CurDAG->getDataLayout();
1275 const TargetLowering *TLI = getTargetLowering();
1276 if (N.getOpcode() == ISD::FrameIndex) {
1277 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1278 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1279 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1280 return true;
1281 }
1282
1283 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1284 // selected here doesn't support labels/immediates, only base+offset.
1285 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1286 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1287 if (IsSignedImm) {
1288 int64_t RHSC = RHS->getSExtValue();
1289 unsigned Scale = Log2_32(Value: Size);
1290 int64_t Range = 0x1LL << (BW - 1);
1291
1292 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1293 RHSC < (Range << Scale)) {
1294 Base = N.getOperand(i: 0);
1295 if (Base.getOpcode() == ISD::FrameIndex) {
1296 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1297 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1298 }
1299 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1300 return true;
1301 }
1302 } else {
1303 // unsigned Immediate
1304 uint64_t RHSC = RHS->getZExtValue();
1305 unsigned Scale = Log2_32(Value: Size);
1306 uint64_t Range = 0x1ULL << BW;
1307
1308 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1309 Base = N.getOperand(i: 0);
1310 if (Base.getOpcode() == ISD::FrameIndex) {
1311 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1312 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1313 }
1314 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1315 return true;
1316 }
1317 }
1318 }
1319 }
1320 // Base only. The address will be materialized into a register before
1321 // the memory is accessed.
1322 // add x0, Xbase, #offset
1323 // stp x1, x2, [x0]
1324 Base = N;
1325 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1326 return true;
1327}
1328
1329/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1330/// immediate" address. The "Size" argument is the size in bytes of the memory
1331/// reference, which determines the scale.
1332bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1333 SDValue &Base, SDValue &OffImm) {
1334 SDLoc dl(N);
1335 const DataLayout &DL = CurDAG->getDataLayout();
1336 const TargetLowering *TLI = getTargetLowering();
1337 if (N.getOpcode() == ISD::FrameIndex) {
1338 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
1339 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1340 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1341 return true;
1342 }
1343
1344 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1345 GlobalAddressSDNode *GAN =
1346 dyn_cast<GlobalAddressSDNode>(Val: N.getOperand(i: 1).getNode());
1347 Base = N.getOperand(i: 0);
1348 OffImm = N.getOperand(i: 1);
1349 if (!GAN)
1350 return true;
1351
1352 if (GAN->getOffset() % Size == 0 &&
1353 GAN->getGlobal()->getPointerAlignment(DL) >= Size)
1354 return true;
1355 }
1356
1357 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
1358 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1359 int64_t RHSC = (int64_t)RHS->getZExtValue();
1360 unsigned Scale = Log2_32(Value: Size);
1361 if (isValidAsScaledImmediate(Offset: RHSC, Range: 0x1000, Size)) {
1362 Base = N.getOperand(i: 0);
1363 if (Base.getOpcode() == ISD::FrameIndex) {
1364 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1365 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
1366 }
1367 OffImm = CurDAG->getTargetConstant(Val: RHSC >> Scale, DL: dl, VT: MVT::i64);
1368 return true;
1369 }
1370 }
1371 }
1372
1373 // Before falling back to our general case, check if the unscaled
1374 // instructions can handle this. If so, that's preferable.
1375 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1376 return false;
1377
1378 // Base only. The address will be materialized into a register before
1379 // the memory is accessed.
1380 // add x0, Xbase, #offset
1381 // ldr x0, [x0]
1382 Base = N;
1383 OffImm = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64);
1384 return true;
1385}
1386
1387/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1388/// immediate" address. This should only match when there is an offset that
1389/// is not valid for a scaled immediate addressing mode. The "Size" argument
1390/// is the size in bytes of the memory reference, which is needed here to know
1391/// what is valid for a scaled immediate.
1392bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1393 SDValue &Base,
1394 SDValue &OffImm) {
1395 if (!CurDAG->isBaseWithConstantOffset(Op: N))
1396 return false;
1397 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1))) {
1398 int64_t RHSC = RHS->getSExtValue();
1399 if (RHSC >= -256 && RHSC < 256) {
1400 Base = N.getOperand(i: 0);
1401 if (Base.getOpcode() == ISD::FrameIndex) {
1402 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
1403 const TargetLowering *TLI = getTargetLowering();
1404 Base = CurDAG->getTargetFrameIndex(
1405 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
1406 }
1407 OffImm = CurDAG->getTargetConstant(Val: RHSC, DL: SDLoc(N), VT: MVT::i64);
1408 return true;
1409 }
1410 }
1411 return false;
1412}
1413
1414static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
1415 SDLoc dl(N);
1416 SDValue ImpDef = SDValue(
1417 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl, VT: MVT::i64), 0);
1418 return CurDAG->getTargetInsertSubreg(SRIdx: AArch64::sub_32, DL: dl, VT: MVT::i64, Operand: ImpDef,
1419 Subreg: N);
1420}
1421
1422/// Check if the given SHL node (\p N), can be used to form an
1423/// extended register for an addressing mode.
1424bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1425 bool WantExtend, SDValue &Offset,
1426 SDValue &SignExtend) {
1427 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1428 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(Val: N.getOperand(i: 1));
1429 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1430 return false;
1431
1432 SDLoc dl(N);
1433 if (WantExtend) {
1434 AArch64_AM::ShiftExtendType Ext =
1435 getExtendTypeForNode(N: N.getOperand(i: 0), IsLoadStore: true);
1436 if (Ext == AArch64_AM::InvalidShiftExtend)
1437 return false;
1438
1439 Offset = narrowIfNeeded(CurDAG, N: N.getOperand(i: 0).getOperand(i: 0));
1440 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1441 VT: MVT::i32);
1442 } else {
1443 Offset = N.getOperand(i: 0);
1444 SignExtend = CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i32);
1445 }
1446
1447 unsigned LegalShiftVal = Log2_32(Value: Size);
1448 unsigned ShiftVal = CSD->getZExtValue();
1449
1450 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1451 return false;
1452
1453 return isWorthFoldingAddr(V: N, Size);
1454}
1455
1456bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1457 SDValue &Base, SDValue &Offset,
1458 SDValue &SignExtend,
1459 SDValue &DoShift) {
1460 if (N.getOpcode() != ISD::ADD)
1461 return false;
1462 SDValue LHS = N.getOperand(i: 0);
1463 SDValue RHS = N.getOperand(i: 1);
1464 SDLoc dl(N);
1465
1466 // We don't want to match immediate adds here, because they are better lowered
1467 // to the register-immediate addressing modes.
1468 if (isa<ConstantSDNode>(Val: LHS) || isa<ConstantSDNode>(Val: RHS))
1469 return false;
1470
1471 // Check if this particular node is reused in any non-memory related
1472 // operation. If yes, do not try to fold this node into the address
1473 // computation, since the computation will be kept.
1474 const SDNode *Node = N.getNode();
1475 for (SDNode *UI : Node->users()) {
1476 if (!isMemOpOrPrefetch(N: UI))
1477 return false;
1478 }
1479
1480 // Remember if it is worth folding N when it produces extended register.
1481 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1482
1483 // Try to match a shifted extend on the RHS.
1484 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1485 SelectExtendedSHL(N: RHS, Size, WantExtend: true, Offset, SignExtend)) {
1486 Base = LHS;
1487 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1488 return true;
1489 }
1490
1491 // Try to match a shifted extend on the LHS.
1492 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1493 SelectExtendedSHL(N: LHS, Size, WantExtend: true, Offset, SignExtend)) {
1494 Base = RHS;
1495 DoShift = CurDAG->getTargetConstant(Val: true, DL: dl, VT: MVT::i32);
1496 return true;
1497 }
1498
1499 // There was no shift, whatever else we find.
1500 DoShift = CurDAG->getTargetConstant(Val: false, DL: dl, VT: MVT::i32);
1501
1502 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1503 // Try to match an unshifted extend on the LHS.
1504 if (IsExtendedRegisterWorthFolding &&
1505 (Ext = getExtendTypeForNode(N: LHS, IsLoadStore: true)) !=
1506 AArch64_AM::InvalidShiftExtend) {
1507 Base = RHS;
1508 Offset = narrowIfNeeded(CurDAG, N: LHS.getOperand(i: 0));
1509 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1510 VT: MVT::i32);
1511 if (isWorthFoldingAddr(V: LHS, Size))
1512 return true;
1513 }
1514
1515 // Try to match an unshifted extend on the RHS.
1516 if (IsExtendedRegisterWorthFolding &&
1517 (Ext = getExtendTypeForNode(N: RHS, IsLoadStore: true)) !=
1518 AArch64_AM::InvalidShiftExtend) {
1519 Base = LHS;
1520 Offset = narrowIfNeeded(CurDAG, N: RHS.getOperand(i: 0));
1521 SignExtend = CurDAG->getTargetConstant(Val: Ext == AArch64_AM::SXTW, DL: dl,
1522 VT: MVT::i32);
1523 if (isWorthFoldingAddr(V: RHS, Size))
1524 return true;
1525 }
1526
1527 return false;
1528}
1529
1530// Check if the given immediate is preferred by ADD. If an immediate can be
1531// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1532// encoded by one MOVZ, return true.
1533static bool isPreferredADD(int64_t ImmOff) {
1534 // Constant in [0x0, 0xfff] can be encoded in ADD.
1535 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1536 return true;
1537 // Check if it can be encoded in an "ADD LSL #12".
1538 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1539 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1540 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1541 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1542 return false;
1543}
1544
1545bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1546 SDValue &Base, SDValue &Offset,
1547 SDValue &SignExtend,
1548 SDValue &DoShift) {
1549 if (N.getOpcode() != ISD::ADD)
1550 return false;
1551 SDValue LHS = N.getOperand(i: 0);
1552 SDValue RHS = N.getOperand(i: 1);
1553 SDLoc DL(N);
1554
1555 // Check if this particular node is reused in any non-memory related
1556 // operation. If yes, do not try to fold this node into the address
1557 // computation, since the computation will be kept.
1558 const SDNode *Node = N.getNode();
1559 for (SDNode *UI : Node->users()) {
1560 if (!isMemOpOrPrefetch(N: UI))
1561 return false;
1562 }
1563
1564 // Watch out if RHS is a wide immediate, it can not be selected into
1565 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1566 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1567 // instructions like:
1568 // MOV X0, WideImmediate
1569 // ADD X1, BaseReg, X0
1570 // LDR X2, [X1, 0]
1571 // For such situation, using [BaseReg, XReg] addressing mode can save one
1572 // ADD/SUB:
1573 // MOV X0, WideImmediate
1574 // LDR X2, [BaseReg, X0]
1575 if (isa<ConstantSDNode>(Val: RHS)) {
1576 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1577 // Skip the immediate can be selected by load/store addressing mode.
1578 // Also skip the immediate can be encoded by a single ADD (SUB is also
1579 // checked by using -ImmOff).
1580 if (isValidAsScaledImmediate(Offset: ImmOff, Range: 0x1000, Size) ||
1581 isPreferredADD(ImmOff) || isPreferredADD(ImmOff: -ImmOff))
1582 return false;
1583
1584 SDValue Ops[] = { RHS };
1585 SDNode *MOVI =
1586 CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
1587 SDValue MOVIV = SDValue(MOVI, 0);
1588 // This ADD of two X register will be selected into [Reg+Reg] mode.
1589 N = CurDAG->getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: LHS, N2: MOVIV);
1590 }
1591
1592 // Remember if it is worth folding N when it produces extended register.
1593 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(V: N, Size);
1594
1595 // Try to match a shifted extend on the RHS.
1596 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1597 SelectExtendedSHL(N: RHS, Size, WantExtend: false, Offset, SignExtend)) {
1598 Base = LHS;
1599 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1600 return true;
1601 }
1602
1603 // Try to match a shifted extend on the LHS.
1604 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1605 SelectExtendedSHL(N: LHS, Size, WantExtend: false, Offset, SignExtend)) {
1606 Base = RHS;
1607 DoShift = CurDAG->getTargetConstant(Val: true, DL, VT: MVT::i32);
1608 return true;
1609 }
1610
1611 // Match any non-shifted, non-extend, non-immediate add expression.
1612 Base = LHS;
1613 Offset = RHS;
1614 SignExtend = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1615 DoShift = CurDAG->getTargetConstant(Val: false, DL, VT: MVT::i32);
1616 // Reg1 + Reg2 is free: no check needed.
1617 return true;
1618}
1619
1620SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1621 static const unsigned RegClassIDs[] = {
1622 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1623 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1624 AArch64::dsub2, AArch64::dsub3};
1625
1626 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1627}
1628
1629SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1630 static const unsigned RegClassIDs[] = {
1631 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1632 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1633 AArch64::qsub2, AArch64::qsub3};
1634
1635 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1636}
1637
1638SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1639 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1640 AArch64::ZPR3RegClassID,
1641 AArch64::ZPR4RegClassID};
1642 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1643 AArch64::zsub2, AArch64::zsub3};
1644
1645 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1646}
1647
1648SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1649 assert(Regs.size() == 2 || Regs.size() == 4);
1650
1651 // The createTuple interface requires 3 RegClassIDs for each possible
1652 // tuple type even though we only have them for ZPR2 and ZPR4.
1653 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1654 AArch64::ZPR4Mul4RegClassID};
1655 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1656 AArch64::zsub2, AArch64::zsub3};
1657 return createTuple(Vecs: Regs, RegClassIDs, SubRegs);
1658}
1659
1660SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1661 const unsigned RegClassIDs[],
1662 const unsigned SubRegs[]) {
1663 // There's no special register-class for a vector-list of 1 element: it's just
1664 // a vector.
1665 if (Regs.size() == 1)
1666 return Regs[0];
1667
1668 assert(Regs.size() >= 2 && Regs.size() <= 4);
1669
1670 SDLoc DL(Regs[0]);
1671
1672 SmallVector<SDValue, 4> Ops;
1673
1674 // First operand of REG_SEQUENCE is the desired RegClass.
1675 Ops.push_back(
1676 Elt: CurDAG->getTargetConstant(Val: RegClassIDs[Regs.size() - 2], DL, VT: MVT::i32));
1677
1678 // Then we get pairs of source & subregister-position for the components.
1679 for (unsigned i = 0; i < Regs.size(); ++i) {
1680 Ops.push_back(Elt: Regs[i]);
1681 Ops.push_back(Elt: CurDAG->getTargetConstant(Val: SubRegs[i], DL, VT: MVT::i32));
1682 }
1683
1684 SDNode *N =
1685 CurDAG->getMachineNode(Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped, Ops);
1686 return SDValue(N, 0);
1687}
1688
1689void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1690 bool isExt) {
1691 SDLoc dl(N);
1692 EVT VT = N->getValueType(ResNo: 0);
1693
1694 unsigned ExtOff = isExt;
1695
1696 // Form a REG_SEQUENCE to force register allocation.
1697 unsigned Vec0Off = ExtOff + 1;
1698 SmallVector<SDValue, 4> Regs(N->ops().slice(N: Vec0Off, M: NumVecs));
1699 SDValue RegSeq = createQTuple(Regs);
1700
1701 SmallVector<SDValue, 6> Ops;
1702 if (isExt)
1703 Ops.push_back(Elt: N->getOperand(Num: 1));
1704 Ops.push_back(Elt: RegSeq);
1705 Ops.push_back(Elt: N->getOperand(Num: NumVecs + ExtOff + 1));
1706 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: Opc, dl, VT, Ops));
1707}
1708
1709static std::tuple<SDValue, SDValue>
1710extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
1711 SDLoc DL(Disc);
1712 SDValue AddrDisc;
1713 SDValue ConstDisc;
1714
1715 // If this is a blend, remember the constant and address discriminators.
1716 // Otherwise, it's either a constant discriminator, or a non-blended
1717 // address discriminator.
1718 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1719 Disc->getConstantOperandVal(Num: 0) == Intrinsic::ptrauth_blend) {
1720 AddrDisc = Disc->getOperand(Num: 1);
1721 ConstDisc = Disc->getOperand(Num: 2);
1722 } else {
1723 ConstDisc = Disc;
1724 }
1725
1726 // If the constant discriminator (either the blend RHS, or the entire
1727 // discriminator value) isn't a 16-bit constant, bail out, and let the
1728 // discriminator be computed separately.
1729 auto *ConstDiscN = dyn_cast<ConstantSDNode>(Val&: ConstDisc);
1730 if (!ConstDiscN || !isUInt<16>(x: ConstDiscN->getZExtValue()))
1731 return std::make_tuple(args: DAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), args&: Disc);
1732
1733 // If there's no address discriminator, use XZR directly.
1734 if (!AddrDisc)
1735 AddrDisc = DAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64);
1736
1737 return std::make_tuple(
1738 args: DAG->getTargetConstant(Val: ConstDiscN->getZExtValue(), DL, VT: MVT::i64),
1739 args&: AddrDisc);
1740}
1741
1742void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1743 SDLoc DL(N);
1744 // IntrinsicID is operand #0
1745 SDValue Val = N->getOperand(Num: 1);
1746 SDValue AUTKey = N->getOperand(Num: 2);
1747 SDValue AUTDisc = N->getOperand(Num: 3);
1748
1749 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1750 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1751
1752 SDValue AUTAddrDisc, AUTConstDisc;
1753 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1754 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1755
1756 if (!Subtarget->isX16X17Safer()) {
1757 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1758 // Copy deactivation symbol if present.
1759 if (N->getNumOperands() > 4)
1760 Ops.push_back(x: N->getOperand(Num: 4));
1761
1762 SDNode *AUT =
1763 CurDAG->getMachineNode(Opcode: AArch64::AUTxMxN, dl: DL, VT1: MVT::i64, VT2: MVT::i64, Ops);
1764 ReplaceNode(F: N, T: AUT);
1765 } else {
1766 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1767 Reg: AArch64::X16, N: Val, Glue: SDValue());
1768 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(R: 1)};
1769
1770 SDNode *AUT = CurDAG->getMachineNode(Opcode: AArch64::AUTx16x17, dl: DL, VT: MVT::i64, Ops);
1771 ReplaceNode(F: N, T: AUT);
1772 }
1773}
1774
1775void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
1776 SDLoc DL(N);
1777 // IntrinsicID is operand #0, if W_CHAIN it is #1
1778 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
1779 SDValue Val = N->getOperand(Num: OffsetBase + 1);
1780 SDValue AUTKey = N->getOperand(Num: OffsetBase + 2);
1781 SDValue AUTDisc = N->getOperand(Num: OffsetBase + 3);
1782 SDValue PACKey = N->getOperand(Num: OffsetBase + 4);
1783 SDValue PACDisc = N->getOperand(Num: OffsetBase + 5);
1784 uint32_t IntNum = N->getConstantOperandVal(Num: OffsetBase + 0);
1785 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
1786
1787 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1788 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1789
1790 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1791 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1792
1793 SDValue AUTAddrDisc, AUTConstDisc;
1794 std::tie(args&: AUTConstDisc, args&: AUTAddrDisc) =
1795 extractPtrauthBlendDiscriminators(Disc: AUTDisc, DAG: CurDAG);
1796
1797 SDValue PACAddrDisc, PACConstDisc;
1798 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1799 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1800
1801 SDValue X16Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1802 Reg: AArch64::X16, N: Val, Glue: SDValue());
1803
1804 if (HasLoad) {
1805 SDValue Addend = N->getOperand(Num: OffsetBase + 6);
1806 SDValue IncomingChain = N->getOperand(Num: 0);
1807 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
1808 PACKey, PACConstDisc, PACAddrDisc,
1809 Addend, IncomingChain, X16Copy.getValue(R: 1)};
1810
1811 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTRELLOADPAC, dl: DL,
1812 VT1: MVT::i64, VT2: MVT::Other, Ops);
1813 ReplaceNode(F: N, T: AUTRELLOADPAC);
1814 } else {
1815 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
1816 PACConstDisc, PACAddrDisc, X16Copy.getValue(R: 1)};
1817
1818 SDNode *AUTPAC = CurDAG->getMachineNode(Opcode: AArch64::AUTPAC, dl: DL, VT: MVT::i64, Ops);
1819 ReplaceNode(F: N, T: AUTPAC);
1820 }
1821}
1822
1823void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
1824 SDLoc DL(N);
1825 SDValue Val = N->getOperand(Num: 1);
1826 SDValue AUTKey = N->getOperand(Num: 2);
1827 SDValue AUTDisc = N->getOperand(Num: 3);
1828 SDValue AUTPC = N->getOperand(Num: 4);
1829 SDValue PACKey = N->getOperand(Num: 5);
1830 SDValue PACDisc = N->getOperand(Num: 6);
1831
1832 unsigned AUTKeyC = cast<ConstantSDNode>(Val&: AUTKey)->getZExtValue();
1833 unsigned PACKeyC = cast<ConstantSDNode>(Val&: PACKey)->getZExtValue();
1834
1835 AUTKey = CurDAG->getTargetConstant(Val: AUTKeyC, DL, VT: MVT::i64);
1836 PACKey = CurDAG->getTargetConstant(Val: PACKeyC, DL, VT: MVT::i64);
1837
1838 SDValue PACAddrDisc, PACConstDisc;
1839 std::tie(args&: PACConstDisc, args&: PACAddrDisc) =
1840 extractPtrauthBlendDiscriminators(Disc: PACDisc, DAG: CurDAG);
1841
1842 SDValue X17Copy = CurDAG->getCopyToReg(Chain: CurDAG->getEntryNode(), dl: DL,
1843 Reg: AArch64::X17, N: Val, Glue: SDValue());
1844 SDValue X16Copy = CurDAG->getCopyToReg(
1845 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X16, N: AUTDisc, Glue: X17Copy.getValue(R: 1));
1846 SDValue X15Copy = CurDAG->getCopyToReg(
1847 Chain: CurDAG->getEntryNode(), dl: DL, Reg: AArch64::X15, N: AUTPC, Glue: X16Copy.getValue(R: 1));
1848
1849 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
1850 X15Copy.getValue(R: 1)};
1851 SDNode *AUTPCPAC =
1852 CurDAG->getMachineNode(Opcode: AArch64::AUTPCPAC, dl: DL, VT: MVT::i64, Ops);
1853 ReplaceNode(F: N, T: AUTPCPAC);
1854}
1855
1856bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1857 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
1858 if (LD->isUnindexed())
1859 return false;
1860 EVT VT = LD->getMemoryVT();
1861 EVT DstVT = N->getValueType(ResNo: 0);
1862 ISD::MemIndexedMode AM = LD->getAddressingMode();
1863 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1864 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(Val: LD->getOffset());
1865 int OffsetVal = (int)OffsetOp->getZExtValue();
1866
1867 // We're not doing validity checking here. That was done when checking
1868 // if we should mark the load as indexed or not. We're just selecting
1869 // the right instruction.
1870 unsigned Opcode = 0;
1871
1872 ISD::LoadExtType ExtType = LD->getExtensionType();
1873 bool InsertTo64 = false;
1874 bool UseLd1 =
1875 (VT.is64BitVector() || VT.is128BitVector()) &&
1876 (!Subtarget->isLittleEndian() || (Subtarget->requiresStrictAlign() &&
1877 LD->getAlign() < VT.getStoreSize()));
1878 if (VT == MVT::i64)
1879 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1880 else if (VT == MVT::i32) {
1881 if (ExtType == ISD::NON_EXTLOAD)
1882 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1883 else if (ExtType == ISD::SEXTLOAD)
1884 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1885 else {
1886 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1887 InsertTo64 = true;
1888 // The result of the load is only i32. It's the subreg_to_reg that makes
1889 // it into an i64.
1890 DstVT = MVT::i32;
1891 }
1892 } else if (VT == MVT::i16) {
1893 if (ExtType == ISD::SEXTLOAD) {
1894 if (DstVT == MVT::i64)
1895 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1896 else
1897 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1898 } else {
1899 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1900 InsertTo64 = DstVT == MVT::i64;
1901 // The result of the load is only i32. It's the subreg_to_reg that makes
1902 // it into an i64.
1903 DstVT = MVT::i32;
1904 }
1905 } else if (VT == MVT::i8) {
1906 if (ExtType == ISD::SEXTLOAD) {
1907 if (DstVT == MVT::i64)
1908 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1909 else
1910 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1911 } else {
1912 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1913 InsertTo64 = DstVT == MVT::i64;
1914 // The result of the load is only i32. It's the subreg_to_reg that makes
1915 // it into an i64.
1916 DstVT = MVT::i32;
1917 }
1918 } else if (VT == MVT::f16) {
1919 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1920 } else if (VT == MVT::bf16) {
1921 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1922 } else if (VT == MVT::f32) {
1923 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1924 } else if (VT == MVT::f64 || (VT.is64BitVector() && !UseLd1)) {
1925 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1926 } else if (VT.is128BitVector() && !UseLd1) {
1927 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1928 } else if (VT.is64BitVector() && UseLd1) {
1929 if (IsPre || OffsetVal != 8)
1930 return false;
1931 switch (VT.getScalarSizeInBits()) {
1932 case 8:
1933 Opcode = AArch64::LD1Onev8b_POST;
1934 break;
1935 case 16:
1936 Opcode = AArch64::LD1Onev4h_POST;
1937 break;
1938 case 32:
1939 Opcode = AArch64::LD1Onev2s_POST;
1940 break;
1941 case 64:
1942 Opcode = AArch64::LD1Onev1d_POST;
1943 break;
1944 default:
1945 llvm_unreachable("Expected vector element to be a power of 2");
1946 }
1947 } else if (VT.is128BitVector() && UseLd1) {
1948 if (IsPre || OffsetVal != 16)
1949 return false;
1950 switch (VT.getScalarSizeInBits()) {
1951 case 8:
1952 Opcode = AArch64::LD1Onev16b_POST;
1953 break;
1954 case 16:
1955 Opcode = AArch64::LD1Onev8h_POST;
1956 break;
1957 case 32:
1958 Opcode = AArch64::LD1Onev4s_POST;
1959 break;
1960 case 64:
1961 Opcode = AArch64::LD1Onev2d_POST;
1962 break;
1963 default:
1964 llvm_unreachable("Expected vector element to be a power of 2");
1965 }
1966 } else
1967 return false;
1968 SDValue Chain = LD->getChain();
1969 SDValue Base = LD->getBasePtr();
1970 SDLoc dl(N);
1971 // LD1 encodes an immediate offset by using XZR as the offset register.
1972 SDValue Offset = UseLd1 ? CurDAG->getRegister(Reg: AArch64::XZR, VT: MVT::i64)
1973 : CurDAG->getTargetConstant(Val: OffsetVal, DL: dl, VT: MVT::i64);
1974 SDValue Ops[] = { Base, Offset, Chain };
1975 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, VT1: MVT::i64, VT2: DstVT,
1976 VT3: MVT::Other, Ops);
1977
1978 // Transfer memoperands.
1979 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
1980 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Res), NewMemRefs: {MemOp});
1981
1982 // Either way, we're replacing the node, so tell the caller that.
1983 SDValue LoadedVal = SDValue(Res, 1);
1984 if (InsertTo64) {
1985 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL: dl, VT: MVT::i32);
1986 LoadedVal = SDValue(CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl,
1987 VT: MVT::i64, Op1: LoadedVal, Op2: SubReg),
1988 0);
1989 }
1990
1991 ReplaceUses(F: SDValue(N, 0), T: LoadedVal);
1992 ReplaceUses(F: SDValue(N, 1), T: SDValue(Res, 0));
1993 ReplaceUses(F: SDValue(N, 2), T: SDValue(Res, 2));
1994 CurDAG->RemoveDeadNode(N);
1995 return true;
1996}
1997
1998void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
1999 unsigned SubRegIdx) {
2000 SDLoc dl(N);
2001 EVT VT = N->getValueType(ResNo: 0);
2002 SDValue Chain = N->getOperand(Num: 0);
2003
2004 SDValue Ops[] = {N->getOperand(Num: 2), // Mem operand;
2005 Chain};
2006
2007 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2008
2009 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2010 SDValue SuperReg = SDValue(Ld, 0);
2011 for (unsigned i = 0; i < NumVecs; ++i)
2012 ReplaceUses(F: SDValue(N, i),
2013 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2014
2015 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2016
2017 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
2018 // because it's too simple to have needed special treatment during lowering.
2019 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: N)) {
2020 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2021 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2022 }
2023
2024 CurDAG->RemoveDeadNode(N);
2025}
2026
2027void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
2028 unsigned Opc, unsigned SubRegIdx) {
2029 SDLoc dl(N);
2030 EVT VT = N->getValueType(ResNo: 0);
2031 SDValue Chain = N->getOperand(Num: 0);
2032
2033 SDValue Ops[] = {N->getOperand(Num: 1), // Mem operand
2034 N->getOperand(Num: 2), // Incremental
2035 Chain};
2036
2037 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2038 MVT::Untyped, MVT::Other};
2039
2040 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2041
2042 // Update uses of write back register
2043 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2044
2045 // Update uses of vector list
2046 SDValue SuperReg = SDValue(Ld, 1);
2047 if (NumVecs == 1)
2048 ReplaceUses(F: SDValue(N, 0), T: SuperReg);
2049 else
2050 for (unsigned i = 0; i < NumVecs; ++i)
2051 ReplaceUses(F: SDValue(N, i),
2052 T: CurDAG->getTargetExtractSubreg(SRIdx: SubRegIdx + i, DL: dl, VT, Operand: SuperReg));
2053
2054 // Transfer memoperands.
2055 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2056 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
2057
2058 // Update the chain
2059 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2060 CurDAG->RemoveDeadNode(N);
2061}
2062
2063/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2064/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2065/// new Base and an SDValue representing the new offset.
2066std::tuple<unsigned, SDValue, SDValue>
2067AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2068 unsigned Opc_ri,
2069 const SDValue &OldBase,
2070 const SDValue &OldOffset,
2071 unsigned Scale) {
2072 SDValue NewBase = OldBase;
2073 SDValue NewOffset = OldOffset;
2074 // Detect a possible Reg+Imm addressing mode.
2075 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2076 Root: N, N: OldBase, Base&: NewBase, OffImm&: NewOffset);
2077
2078 // Detect a possible reg+reg addressing mode, but only if we haven't already
2079 // detected a Reg+Imm one.
2080 const bool IsRegReg =
2081 !IsRegImm && SelectSVERegRegAddrMode(N: OldBase, Scale, Base&: NewBase, Offset&: NewOffset);
2082
2083 // Select the instruction.
2084 return std::make_tuple(args&: IsRegReg ? Opc_rr : Opc_ri, args&: NewBase, args&: NewOffset);
2085}
2086
2087enum class SelectTypeKind {
2088 Int1 = 0,
2089 Int = 1,
2090 FP = 2,
2091 AnyType = 3,
2092};
2093
2094/// This function selects an opcode from a list of opcodes, which is
2095/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2096/// element types, in this order.
2097template <SelectTypeKind Kind>
2098static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2099 // Only match scalable vector VTs
2100 if (!VT.isScalableVector())
2101 return 0;
2102
2103 EVT EltVT = VT.getVectorElementType();
2104 unsigned Key = VT.getVectorMinNumElements();
2105 switch (Kind) {
2106 case SelectTypeKind::AnyType:
2107 break;
2108 case SelectTypeKind::Int:
2109 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2110 EltVT != MVT::i64)
2111 return 0;
2112 break;
2113 case SelectTypeKind::Int1:
2114 if (EltVT != MVT::i1)
2115 return 0;
2116 break;
2117 case SelectTypeKind::FP:
2118 if (EltVT == MVT::bf16)
2119 Key = 16;
2120 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2121 EltVT != MVT::f64)
2122 return 0;
2123 break;
2124 }
2125
2126 unsigned Offset;
2127 switch (Key) {
2128 case 16: // 8-bit or bf16
2129 Offset = 0;
2130 break;
2131 case 8: // 16-bit
2132 Offset = 1;
2133 break;
2134 case 4: // 32-bit
2135 Offset = 2;
2136 break;
2137 case 2: // 64-bit
2138 Offset = 3;
2139 break;
2140 default:
2141 return 0;
2142 }
2143
2144 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2145}
2146
2147// This function is almost identical to SelectWhilePair, but has an
2148// extra check on the range of the immediate operand.
2149// TODO: Merge these two functions together at some point?
2150void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2151 // Immediate can be either 0 or 1.
2152 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 2)))
2153 if (Imm->getZExtValue() > 1)
2154 return;
2155
2156 SDLoc DL(N);
2157 EVT VT = N->getValueType(ResNo: 0);
2158 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2159 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2160 SDValue SuperReg = SDValue(WhilePair, 0);
2161
2162 for (unsigned I = 0; I < 2; ++I)
2163 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2164 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2165
2166 CurDAG->RemoveDeadNode(N);
2167}
2168
2169void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2170 SDLoc DL(N);
2171 EVT VT = N->getValueType(ResNo: 0);
2172
2173 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2)};
2174
2175 SDNode *WhilePair = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2176 SDValue SuperReg = SDValue(WhilePair, 0);
2177
2178 for (unsigned I = 0; I < 2; ++I)
2179 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2180 SRIdx: AArch64::psub0 + I, DL, VT, Operand: SuperReg));
2181
2182 CurDAG->RemoveDeadNode(N);
2183}
2184
2185void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2186 unsigned Opcode) {
2187 EVT VT = N->getValueType(ResNo: 0);
2188 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2189 SDValue Ops = createZTuple(Regs);
2190 SDLoc DL(N);
2191 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Op1: Ops);
2192 SDValue SuperReg = SDValue(Intrinsic, 0);
2193 for (unsigned i = 0; i < NumVecs; ++i)
2194 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2195 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2196
2197 CurDAG->RemoveDeadNode(N);
2198}
2199
2200void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2201 unsigned Opcode) {
2202 SDLoc DL(N);
2203 EVT VT = N->getValueType(ResNo: 0);
2204 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2205 Ops.push_back(/*Chain*/ Elt: N->getOperand(Num: 0));
2206
2207 SDNode *Instruction =
2208 CurDAG->getMachineNode(Opcode, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2209 SDValue SuperReg = SDValue(Instruction, 0);
2210
2211 for (unsigned i = 0; i < NumVecs; ++i)
2212 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2213 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2214
2215 // Copy chain
2216 unsigned ChainIdx = NumVecs;
2217 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Instruction, 1));
2218 CurDAG->RemoveDeadNode(N);
2219}
2220
2221void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2222 unsigned NumVecs,
2223 bool IsZmMulti,
2224 unsigned Opcode,
2225 bool HasPred) {
2226 assert(Opcode != 0 && "Unexpected opcode");
2227
2228 SDLoc DL(N);
2229 EVT VT = N->getValueType(ResNo: 0);
2230 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2231 SmallVector<SDValue, 4> Ops;
2232
2233 auto GetMultiVecOperand = [&]() {
2234 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2235 OpsIter += NumVecs;
2236 return createZMulTuple(Regs);
2237 };
2238
2239 if (HasPred)
2240 Ops.push_back(Elt: *OpsIter++);
2241
2242 Ops.push_back(Elt: GetMultiVecOperand());
2243 if (IsZmMulti)
2244 Ops.push_back(Elt: GetMultiVecOperand());
2245 else
2246 Ops.push_back(Elt: *OpsIter++);
2247
2248 // Append any remaining operands.
2249 Ops.append(in_start: OpsIter, in_end: N->op_end());
2250 SDNode *Intrinsic;
2251 Intrinsic = CurDAG->getMachineNode(Opcode, dl: DL, VT: MVT::Untyped, Ops);
2252 SDValue SuperReg = SDValue(Intrinsic, 0);
2253 for (unsigned i = 0; i < NumVecs; ++i)
2254 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2255 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2256
2257 CurDAG->RemoveDeadNode(N);
2258}
2259
2260void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2261 unsigned Scale, unsigned Opc_ri,
2262 unsigned Opc_rr, bool IsIntr) {
2263 assert(Scale < 5 && "Invalid scaling value.");
2264 SDLoc DL(N);
2265 EVT VT = N->getValueType(ResNo: 0);
2266 SDValue Chain = N->getOperand(Num: 0);
2267
2268 // Optimize addressing mode.
2269 SDValue Base, Offset;
2270 unsigned Opc;
2271 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2272 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: IsIntr ? 3 : 2),
2273 OldOffset: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), Scale);
2274
2275 SDValue Ops[] = {N->getOperand(Num: IsIntr ? 2 : 1), // Predicate
2276 Base, // Memory operand
2277 Offset, Chain};
2278
2279 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2280
2281 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2282 SDValue SuperReg = SDValue(Load, 0);
2283 for (unsigned i = 0; i < NumVecs; ++i)
2284 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2285 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2286
2287 // Copy chain
2288 unsigned ChainIdx = NumVecs;
2289 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2290 CurDAG->RemoveDeadNode(N);
2291}
2292
2293void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2294 unsigned NumVecs,
2295 unsigned Scale,
2296 unsigned Opc_ri,
2297 unsigned Opc_rr) {
2298 assert(Scale < 4 && "Invalid scaling value.");
2299 SDLoc DL(N);
2300 EVT VT = N->getValueType(ResNo: 0);
2301 SDValue Chain = N->getOperand(Num: 0);
2302
2303 SDValue PNg = N->getOperand(Num: 2);
2304 SDValue Base = N->getOperand(Num: 3);
2305 SDValue Offset = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
2306 unsigned Opc;
2307 std::tie(args&: Opc, args&: Base, args&: Offset) =
2308 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, OldBase: Base, OldOffset: Offset, Scale);
2309
2310 SDValue Ops[] = {PNg, // Predicate-as-counter
2311 Base, // Memory operand
2312 Offset, Chain};
2313
2314 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2315
2316 SDNode *Load = CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: ResTys, Ops);
2317 SDValue SuperReg = SDValue(Load, 0);
2318 for (unsigned i = 0; i < NumVecs; ++i)
2319 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2320 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2321
2322 // Copy chain
2323 unsigned ChainIdx = NumVecs;
2324 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Load, 1));
2325 CurDAG->RemoveDeadNode(N);
2326}
2327
2328void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2329 unsigned Opcode) {
2330 if (N->getValueType(ResNo: 0) != MVT::nxv4f32)
2331 return;
2332 SelectUnaryMultiIntrinsic(N, NumOutVecs: NumVecs, IsTupleInput: true, Opc: Opcode);
2333}
2334
2335void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2336 unsigned NumOutVecs,
2337 unsigned Opc,
2338 uint32_t MaxImm) {
2339 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: 4)))
2340 if (Imm->getZExtValue() > MaxImm)
2341 return;
2342
2343 SDValue ZtValue;
2344 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2345 return;
2346
2347 SDValue Chain = Node->getOperand(Num: 0);
2348 SDValue Ops[] = {ZtValue, Node->getOperand(Num: 3), Node->getOperand(Num: 4), Chain};
2349 SDLoc DL(Node);
2350 EVT VT = Node->getValueType(ResNo: 0);
2351
2352 SDNode *Instruction =
2353 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2354 SDValue SuperReg = SDValue(Instruction, 0);
2355
2356 for (unsigned I = 0; I < NumOutVecs; ++I)
2357 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2358 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2359
2360 // Copy chain
2361 unsigned ChainIdx = NumOutVecs;
2362 ReplaceUses(F: SDValue(Node, ChainIdx), T: SDValue(Instruction, 1));
2363 CurDAG->RemoveDeadNode(N: Node);
2364}
2365
2366void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2367 unsigned NumIndexVecs) {
2368 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2369 "unexpected number of index vectors");
2370
2371 constexpr unsigned FirstIndexOp = 3;
2372 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2373 auto *Imm = dyn_cast<ConstantSDNode>(Val: Node->getOperand(Num: ImmOp));
2374 if (!Imm || Imm->getZExtValue() > 1)
2375 return;
2376
2377 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2378 // ACLE form provides three index vectors, so the lane selects which adjacent
2379 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2380 unsigned Lane = Imm->getZExtValue();
2381 unsigned IndexOp = FirstIndexOp;
2382 if (NumIndexVecs == 3)
2383 IndexOp += Lane;
2384
2385 SDValue TableTuple = createZTuple(Regs: {Node->getOperand(Num: 1), Node->getOperand(Num: 2)});
2386 SDValue IndexTuple =
2387 createZTuple(Regs: {Node->getOperand(Num: IndexOp), Node->getOperand(Num: IndexOp + 1)});
2388 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(Num: ImmOp)};
2389
2390 SDLoc DL(Node);
2391 EVT VT = Node->getValueType(ResNo: 0);
2392 SDNode *Instruction =
2393 CurDAG->getMachineNode(Opcode: AArch64::LUTI6_4Z2Z2ZI, dl: DL, VT: MVT::Untyped, Ops);
2394 SDValue SuperReg = SDValue(Instruction, 0);
2395
2396 for (unsigned I = 0; I < 4; ++I)
2397 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2398 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2399
2400 CurDAG->RemoveDeadNode(N: Node);
2401}
2402
2403void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2404 unsigned NumOutVecs,
2405 unsigned Opc,
2406 unsigned NumInVecs) {
2407 assert((NumInVecs == 2 || NumInVecs == 3) &&
2408 "unexpected number of input vectors");
2409
2410 SDValue ZtValue;
2411 if (!ImmToReg<AArch64::ZT0, 0>(N: Node->getOperand(Num: 2), Imm&: ZtValue))
2412 return;
2413
2414 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 3, M: NumInVecs));
2415 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2416 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(Num: 0)};
2417
2418 SDLoc DL(Node);
2419 EVT VT = Node->getValueType(ResNo: 0);
2420
2421 SDNode *Instruction =
2422 CurDAG->getMachineNode(Opcode: Opc, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2423 SDValue SuperReg = SDValue(Instruction, 0);
2424
2425 for (unsigned I = 0; I < NumOutVecs; ++I)
2426 ReplaceUses(F: SDValue(Node, I), T: CurDAG->getTargetExtractSubreg(
2427 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2428
2429 ReplaceUses(F: SDValue(Node, NumOutVecs), T: SDValue(Instruction, 1));
2430 CurDAG->RemoveDeadNode(N: Node);
2431}
2432
2433void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2434 unsigned Op) {
2435 SDLoc DL(N);
2436 EVT VT = N->getValueType(ResNo: 0);
2437
2438 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2439 SDValue Zd = createZMulTuple(Regs);
2440 SDValue Zn = N->getOperand(Num: 1 + NumVecs);
2441 SDValue Zm = N->getOperand(Num: 2 + NumVecs);
2442
2443 SDValue Ops[] = {Zd, Zn, Zm};
2444
2445 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT: MVT::Untyped, Ops);
2446 SDValue SuperReg = SDValue(Intrinsic, 0);
2447 for (unsigned i = 0; i < NumVecs; ++i)
2448 ReplaceUses(F: SDValue(N, i), T: CurDAG->getTargetExtractSubreg(
2449 SRIdx: AArch64::zsub0 + i, DL, VT, Operand: SuperReg));
2450
2451 CurDAG->RemoveDeadNode(N);
2452}
2453
2454bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2455 switch (BaseReg) {
2456 default:
2457 return false;
2458 case AArch64::ZA:
2459 case AArch64::ZAB0:
2460 if (TileNum == 0)
2461 break;
2462 return false;
2463 case AArch64::ZAH0:
2464 if (TileNum <= 1)
2465 break;
2466 return false;
2467 case AArch64::ZAS0:
2468 if (TileNum <= 3)
2469 break;
2470 return false;
2471 case AArch64::ZAD0:
2472 if (TileNum <= 7)
2473 break;
2474 return false;
2475 }
2476
2477 BaseReg += TileNum;
2478 return true;
2479}
2480
2481template <unsigned MaxIdx, unsigned Scale>
2482void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2483 unsigned BaseReg, unsigned Op) {
2484 unsigned TileNum = 0;
2485 if (BaseReg != AArch64::ZA)
2486 TileNum = N->getConstantOperandVal(Num: 2);
2487
2488 if (!SelectSMETile(BaseReg, TileNum))
2489 return;
2490
2491 SDValue SliceBase, Base, Offset;
2492 if (BaseReg == AArch64::ZA)
2493 SliceBase = N->getOperand(Num: 2);
2494 else
2495 SliceBase = N->getOperand(Num: 3);
2496
2497 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2498 return;
2499
2500 SDLoc DL(N);
2501 SDValue SubReg = CurDAG->getRegister(Reg: BaseReg, VT: MVT::Other);
2502 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(Num: 0)};
2503 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2504
2505 EVT VT = N->getValueType(ResNo: 0);
2506 for (unsigned I = 0; I < NumVecs; ++I)
2507 ReplaceUses(F: SDValue(N, I),
2508 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2509 Operand: SDValue(Mov, 0)));
2510 // Copy chain
2511 unsigned ChainIdx = NumVecs;
2512 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2513 CurDAG->RemoveDeadNode(N);
2514}
2515
2516void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2517 unsigned Op, unsigned MaxIdx,
2518 unsigned Scale, unsigned BaseReg) {
2519 // Slice can be in different positions
2520 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2521 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2522 SDValue SliceBase = N->getOperand(Num: 2);
2523 if (BaseReg != AArch64::ZA)
2524 SliceBase = N->getOperand(Num: 3);
2525
2526 SDValue Base, Offset;
2527 if (!SelectSMETileSlice(N: SliceBase, MaxSize: MaxIdx, Vector&: Base, Offset, Scale))
2528 return;
2529 // The correct Za tile number is computed in Machine Instruction
2530 // See EmitZAInstr
2531 // DAG cannot select Za tile as an output register with ZReg
2532 SDLoc DL(N);
2533 SmallVector<SDValue, 6> Ops;
2534 if (BaseReg != AArch64::ZA )
2535 Ops.push_back(Elt: N->getOperand(Num: 2));
2536 Ops.push_back(Elt: Base);
2537 Ops.push_back(Elt: Offset);
2538 Ops.push_back(Elt: N->getOperand(Num: 0)); //Chain
2539 SDNode *Mov = CurDAG->getMachineNode(Opcode: Op, dl: DL, ResultTys: {MVT::Untyped, MVT::Other}, Ops);
2540
2541 EVT VT = N->getValueType(ResNo: 0);
2542 for (unsigned I = 0; I < NumVecs; ++I)
2543 ReplaceUses(F: SDValue(N, I),
2544 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
2545 Operand: SDValue(Mov, 0)));
2546
2547 // Copy chain
2548 unsigned ChainIdx = NumVecs;
2549 ReplaceUses(F: SDValue(N, ChainIdx), T: SDValue(Mov, 1));
2550 CurDAG->RemoveDeadNode(N);
2551}
2552
2553void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2554 unsigned NumOutVecs,
2555 bool IsTupleInput,
2556 unsigned Opc) {
2557 SDLoc DL(N);
2558 EVT VT = N->getValueType(ResNo: 0);
2559 unsigned NumInVecs = N->getNumOperands() - 1;
2560
2561 SmallVector<SDValue, 6> Ops;
2562 if (IsTupleInput) {
2563 assert((NumInVecs == 2 || NumInVecs == 4) &&
2564 "Don't know how to handle multi-register input!");
2565 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumInVecs));
2566 Ops.push_back(Elt: createZMulTuple(Regs));
2567 } else {
2568 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2569 for (unsigned I = 0; I < NumInVecs; I++)
2570 Ops.push_back(Elt: N->getOperand(Num: 1 + I));
2571 }
2572
2573 SDNode *Res = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: MVT::Untyped, Ops);
2574 SDValue SuperReg = SDValue(Res, 0);
2575
2576 for (unsigned I = 0; I < NumOutVecs; I++)
2577 ReplaceUses(F: SDValue(N, I), T: CurDAG->getTargetExtractSubreg(
2578 SRIdx: AArch64::zsub0 + I, DL, VT, Operand: SuperReg));
2579 CurDAG->RemoveDeadNode(N);
2580}
2581
2582void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2583 unsigned Opc) {
2584 SDLoc dl(N);
2585 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2586
2587 // Form a REG_SEQUENCE to force register allocation.
2588 bool Is128Bit = VT.getSizeInBits() == 128;
2589 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2590 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2591
2592 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), N->getOperand(Num: 0)};
2593 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2594
2595 // Transfer memoperands.
2596 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2597 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2598
2599 ReplaceNode(F: N, T: St);
2600}
2601
2602void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2603 unsigned Scale, unsigned Opc_rr,
2604 unsigned Opc_ri) {
2605 SDLoc dl(N);
2606
2607 // Form a REG_SEQUENCE to force register allocation.
2608 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2609 SDValue RegSeq = createZTuple(Regs);
2610
2611 // Optimize addressing mode.
2612 unsigned Opc;
2613 SDValue Offset, Base;
2614 std::tie(args&: Opc, args&: Base, args&: Offset) = findAddrModeSVELoadStore(
2615 N, Opc_rr, Opc_ri, OldBase: N->getOperand(Num: NumVecs + 3),
2616 OldOffset: CurDAG->getTargetConstant(Val: 0, DL: dl, VT: MVT::i64), Scale);
2617
2618 SDValue Ops[] = {RegSeq, N->getOperand(Num: NumVecs + 2), // predicate
2619 Base, // address
2620 Offset, // offset
2621 N->getOperand(Num: 0)}; // chain
2622 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: N->getValueType(ResNo: 0), Ops);
2623
2624 // Transfer memoperands.
2625 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2626 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2627
2628 ReplaceNode(F: N, T: St);
2629}
2630
2631void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2632 unsigned Opc) {
2633 SDLoc dl(N);
2634 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2635 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2636 MVT::Other}; // Type for the Chain
2637
2638 // Form a REG_SEQUENCE to force register allocation.
2639 bool Is128Bit = VT.getSizeInBits() == 128;
2640 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2641 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2642
2643 SDValue Ops[] = {RegSeq,
2644 N->getOperand(Num: NumVecs + 1), // base register
2645 N->getOperand(Num: NumVecs + 2), // Incremental
2646 N->getOperand(Num: 0)}; // Chain
2647 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2648
2649 // Transfer memoperands.
2650 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2651 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2652
2653 ReplaceNode(F: N, T: St);
2654}
2655
2656namespace {
2657/// WidenVector - Given a value in the V64 register class, produce the
2658/// equivalent value in the V128 register class.
2659class WidenVector {
2660 SelectionDAG &DAG;
2661
2662public:
2663 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2664
2665 SDValue operator()(SDValue V64Reg) {
2666 EVT VT = V64Reg.getValueType();
2667 unsigned NarrowSize = VT.getVectorNumElements();
2668 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2669 MVT WideTy = MVT::getVectorVT(VT: EltTy, NumElements: 2 * NarrowSize);
2670 SDLoc DL(V64Reg);
2671
2672 SDValue Undef =
2673 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: WideTy), 0);
2674 return DAG.getTargetInsertSubreg(SRIdx: AArch64::dsub, DL, VT: WideTy, Operand: Undef, Subreg: V64Reg);
2675 }
2676};
2677} // namespace
2678
2679/// NarrowVector - Given a value in the V128 register class, produce the
2680/// equivalent value in the V64 register class.
2681static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
2682 EVT VT = V128Reg.getValueType();
2683 unsigned WideSize = VT.getVectorNumElements();
2684 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2685 MVT NarrowTy = MVT::getVectorVT(VT: EltTy, NumElements: WideSize / 2);
2686
2687 return DAG.getTargetExtractSubreg(SRIdx: AArch64::dsub, DL: SDLoc(V128Reg), VT: NarrowTy,
2688 Operand: V128Reg);
2689}
2690
2691void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2692 unsigned Opc) {
2693 SDLoc dl(N);
2694 EVT VT = N->getValueType(ResNo: 0);
2695 bool Narrow = VT.getSizeInBits() == 64;
2696
2697 // Form a REG_SEQUENCE to force register allocation.
2698 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2699
2700 if (Narrow)
2701 transform(Range&: Regs, d_first: Regs.begin(),
2702 F: WidenVector(*CurDAG));
2703
2704 SDValue RegSeq = createQTuple(Regs);
2705
2706 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2707
2708 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2709
2710 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2711 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2712 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2713 SDValue SuperReg = SDValue(Ld, 0);
2714
2715 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2716 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2717 AArch64::qsub2, AArch64::qsub3 };
2718 for (unsigned i = 0; i < NumVecs; ++i) {
2719 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT, Operand: SuperReg);
2720 if (Narrow)
2721 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2722 ReplaceUses(F: SDValue(N, i), T: NV);
2723 }
2724
2725 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 1));
2726 CurDAG->RemoveDeadNode(N);
2727}
2728
2729void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2730 unsigned Opc) {
2731 SDLoc dl(N);
2732 EVT VT = N->getValueType(ResNo: 0);
2733 bool Narrow = VT.getSizeInBits() == 64;
2734
2735 // Form a REG_SEQUENCE to force register allocation.
2736 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2737
2738 if (Narrow)
2739 transform(Range&: Regs, d_first: Regs.begin(),
2740 F: WidenVector(*CurDAG));
2741
2742 SDValue RegSeq = createQTuple(Regs);
2743
2744 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2745 RegSeq->getValueType(ResNo: 0), MVT::Other};
2746
2747 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2748
2749 SDValue Ops[] = {RegSeq,
2750 CurDAG->getTargetConstant(Val: LaneNo, DL: dl,
2751 VT: MVT::i64), // Lane Number
2752 N->getOperand(Num: NumVecs + 2), // Base register
2753 N->getOperand(Num: NumVecs + 3), // Incremental
2754 N->getOperand(Num: 0)};
2755 SDNode *Ld = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2756
2757 // Update uses of the write back register
2758 ReplaceUses(F: SDValue(N, NumVecs), T: SDValue(Ld, 0));
2759
2760 // Update uses of the vector list
2761 SDValue SuperReg = SDValue(Ld, 1);
2762 if (NumVecs == 1) {
2763 ReplaceUses(F: SDValue(N, 0),
2764 T: Narrow ? NarrowVector(V128Reg: SuperReg, DAG&: *CurDAG) : SuperReg);
2765 } else {
2766 EVT WideVT = RegSeq.getOperand(i: 1)->getValueType(ResNo: 0);
2767 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2768 AArch64::qsub2, AArch64::qsub3 };
2769 for (unsigned i = 0; i < NumVecs; ++i) {
2770 SDValue NV = CurDAG->getTargetExtractSubreg(SRIdx: QSubs[i], DL: dl, VT: WideVT,
2771 Operand: SuperReg);
2772 if (Narrow)
2773 NV = NarrowVector(V128Reg: NV, DAG&: *CurDAG);
2774 ReplaceUses(F: SDValue(N, i), T: NV);
2775 }
2776 }
2777
2778 // Update the Chain
2779 ReplaceUses(F: SDValue(N, NumVecs + 1), T: SDValue(Ld, 2));
2780 CurDAG->RemoveDeadNode(N);
2781}
2782
2783void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
2784 unsigned Opc) {
2785 SDLoc dl(N);
2786 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2787 bool Narrow = VT.getSizeInBits() == 64;
2788
2789 // Form a REG_SEQUENCE to force register allocation.
2790 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 2, M: NumVecs));
2791
2792 if (Narrow)
2793 transform(Range&: Regs, d_first: Regs.begin(),
2794 F: WidenVector(*CurDAG));
2795
2796 SDValue RegSeq = createQTuple(Regs);
2797
2798 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 2);
2799
2800 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2801 N->getOperand(Num: NumVecs + 3), N->getOperand(Num: 0)};
2802 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::Other, Ops);
2803
2804 // Transfer memoperands.
2805 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2806 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2807
2808 ReplaceNode(F: N, T: St);
2809}
2810
2811void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
2812 unsigned Opc) {
2813 SDLoc dl(N);
2814 EVT VT = N->getOperand(Num: 2)->getValueType(ResNo: 0);
2815 bool Narrow = VT.getSizeInBits() == 64;
2816
2817 // Form a REG_SEQUENCE to force register allocation.
2818 SmallVector<SDValue, 4> Regs(N->ops().slice(N: 1, M: NumVecs));
2819
2820 if (Narrow)
2821 transform(Range&: Regs, d_first: Regs.begin(),
2822 F: WidenVector(*CurDAG));
2823
2824 SDValue RegSeq = createQTuple(Regs);
2825
2826 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2827 MVT::Other};
2828
2829 unsigned LaneNo = N->getConstantOperandVal(Num: NumVecs + 1);
2830
2831 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(Val: LaneNo, DL: dl, VT: MVT::i64),
2832 N->getOperand(Num: NumVecs + 2), // Base Register
2833 N->getOperand(Num: NumVecs + 3), // Incremental
2834 N->getOperand(Num: 0)};
2835 SDNode *St = CurDAG->getMachineNode(Opcode: Opc, dl, ResultTys: ResTys, Ops);
2836
2837 // Transfer memoperands.
2838 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(Val: N)->getMemOperand();
2839 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
2840
2841 ReplaceNode(F: N, T: St);
2842}
2843
2844static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
2845 unsigned &Opc, SDValue &Opd0,
2846 unsigned &LSB, unsigned &MSB,
2847 unsigned NumberOfIgnoredLowBits,
2848 bool BiggerPattern) {
2849 assert(N->getOpcode() == ISD::AND &&
2850 "N must be a AND operation to call this function");
2851
2852 EVT VT = N->getValueType(ResNo: 0);
2853
2854 // Here we can test the type of VT and return false when the type does not
2855 // match, but since it is done prior to that call in the current context
2856 // we turned that into an assert to avoid redundant code.
2857 assert((VT == MVT::i32 || VT == MVT::i64) &&
2858 "Type checking must have been done before calling this function");
2859
2860 // FIXME: simplify-demanded-bits in DAGCombine will probably have
2861 // changed the AND node to a 32-bit mask operation. We'll have to
2862 // undo that as part of the transform here if we want to catch all
2863 // the opportunities.
2864 // Currently the NumberOfIgnoredLowBits argument helps to recover
2865 // from these situations when matching bigger pattern (bitfield insert).
2866
2867 // For unsigned extracts, check for a shift right and mask
2868 uint64_t AndImm = 0;
2869 if (!isOpcWithIntImmediate(N, Opc: ISD::AND, Imm&: AndImm))
2870 return false;
2871
2872 const SDNode *Op0 = N->getOperand(Num: 0).getNode();
2873
2874 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
2875 // simplified. Try to undo that
2876 AndImm |= maskTrailingOnes<uint64_t>(N: NumberOfIgnoredLowBits);
2877
2878 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2879 if (AndImm & (AndImm + 1))
2880 return false;
2881
2882 bool ClampMSB = false;
2883 uint64_t SrlImm = 0;
2884 // Handle the SRL + ANY_EXTEND case.
2885 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
2886 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
2887 // Extend the incoming operand of the SRL to 64-bit.
2888 Opd0 = Widen(CurDAG, N: Op0->getOperand(Num: 0).getOperand(i: 0));
2889 // Make sure to clamp the MSB so that we preserve the semantics of the
2890 // original operations.
2891 ClampMSB = true;
2892 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
2893 isOpcWithIntImmediate(N: Op0->getOperand(Num: 0).getNode(), Opc: ISD::SRL,
2894 Imm&: SrlImm)) {
2895 // If the shift result was truncated, we can still combine them.
2896 Opd0 = Op0->getOperand(Num: 0).getOperand(i: 0);
2897
2898 // Use the type of SRL node.
2899 VT = Opd0->getValueType(ResNo: 0);
2900 } else if (isOpcWithIntImmediate(N: Op0, Opc: ISD::SRL, Imm&: SrlImm)) {
2901 Opd0 = Op0->getOperand(Num: 0);
2902 ClampMSB = (VT == MVT::i32);
2903 } else if (BiggerPattern) {
2904 // Let's pretend a 0 shift right has been performed.
2905 // The resulting code will be at least as good as the original one
2906 // plus it may expose more opportunities for bitfield insert pattern.
2907 // FIXME: Currently we limit this to the bigger pattern, because
2908 // some optimizations expect AND and not UBFM.
2909 Opd0 = N->getOperand(Num: 0);
2910 } else
2911 return false;
2912
2913 // Bail out on large immediates. This happens when no proper
2914 // combining/constant folding was performed.
2915 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
2916 LLVM_DEBUG(
2917 (dbgs() << N
2918 << ": Found large shift immediate, this should not happen\n"));
2919 return false;
2920 }
2921
2922 LSB = SrlImm;
2923 MSB = SrlImm +
2924 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(Value: AndImm)
2925 : llvm::countr_one<uint64_t>(Value: AndImm)) -
2926 1;
2927 if (ClampMSB)
2928 // Since we're moving the extend before the right shift operation, we need
2929 // to clamp the MSB to make sure we don't shift in undefined bits instead of
2930 // the zeros which would get shifted in with the original right shift
2931 // operation.
2932 MSB = MSB > 31 ? 31 : MSB;
2933
2934 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2935 return true;
2936}
2937
2938static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
2939 SDValue &Opd0, unsigned &Immr,
2940 unsigned &Imms) {
2941 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
2942
2943 EVT VT = N->getValueType(ResNo: 0);
2944 unsigned BitWidth = VT.getSizeInBits();
2945 assert((VT == MVT::i32 || VT == MVT::i64) &&
2946 "Type checking must have been done before calling this function");
2947
2948 SDValue Op = N->getOperand(Num: 0);
2949 if (Op->getOpcode() == ISD::TRUNCATE) {
2950 Op = Op->getOperand(Num: 0);
2951 VT = Op->getValueType(ResNo: 0);
2952 BitWidth = VT.getSizeInBits();
2953 }
2954
2955 uint64_t ShiftImm;
2956 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRL, Imm&: ShiftImm) &&
2957 !isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
2958 return false;
2959
2960 unsigned Width = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT().getSizeInBits();
2961 if (ShiftImm + Width > BitWidth)
2962 return false;
2963
2964 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
2965 Opd0 = Op.getOperand(i: 0);
2966 Immr = ShiftImm;
2967 Imms = ShiftImm + Width - 1;
2968 return true;
2969}
2970
2971static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
2972 SDValue &Opd0, unsigned &LSB,
2973 unsigned &MSB) {
2974 // We are looking for the following pattern which basically extracts several
2975 // continuous bits from the source value and places it from the LSB of the
2976 // destination value, all other bits of the destination value or set to zero:
2977 //
2978 // Value2 = AND Value, MaskImm
2979 // SRL Value2, ShiftImm
2980 //
2981 // with MaskImm >> ShiftImm to search for the bit width.
2982 //
2983 // This gets selected into a single UBFM:
2984 //
2985 // UBFM Value, ShiftImm, Log2_64(MaskImm)
2986 //
2987
2988 if (N->getOpcode() != ISD::SRL)
2989 return false;
2990
2991 uint64_t AndMask = 0;
2992 if (!isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::AND, Imm&: AndMask))
2993 return false;
2994
2995 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
2996
2997 uint64_t SrlImm = 0;
2998 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
2999 return false;
3000
3001 // Check whether we really have several bits extract here.
3002 if (!isMask_64(Value: AndMask >> SrlImm))
3003 return false;
3004
3005 Opc = N->getValueType(ResNo: 0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3006 LSB = SrlImm;
3007 MSB = llvm::Log2_64(Value: AndMask);
3008 return true;
3009}
3010
3011static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
3012 unsigned &Immr, unsigned &Imms,
3013 bool BiggerPattern) {
3014 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
3015 "N must be a SHR/SRA operation to call this function");
3016
3017 EVT VT = N->getValueType(ResNo: 0);
3018
3019 // Here we can test the type of VT and return false when the type does not
3020 // match, but since it is done prior to that call in the current context
3021 // we turned that into an assert to avoid redundant code.
3022 assert((VT == MVT::i32 || VT == MVT::i64) &&
3023 "Type checking must have been done before calling this function");
3024
3025 // Check for AND + SRL doing several bits extract.
3026 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, LSB&: Immr, MSB&: Imms))
3027 return true;
3028
3029 // We're looking for a shift of a shift.
3030 uint64_t ShlImm = 0;
3031 uint64_t TruncBits = 0;
3032 if (isOpcWithIntImmediate(N: N->getOperand(Num: 0).getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3033 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3034 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
3035 N->getOperand(Num: 0).getNode()->getOpcode() == ISD::TRUNCATE) {
3036 // We are looking for a shift of truncate. Truncate from i64 to i32 could
3037 // be considered as setting high 32 bits as zero. Our strategy here is to
3038 // always generate 64bit UBFM. This consistency will help the CSE pass
3039 // later find more redundancy.
3040 Opd0 = N->getOperand(Num: 0).getOperand(i: 0);
3041 TruncBits = Opd0->getValueType(ResNo: 0).getSizeInBits() - VT.getSizeInBits();
3042 VT = Opd0.getValueType();
3043 assert(VT == MVT::i64 && "the promoted type should be i64");
3044 } else if (BiggerPattern) {
3045 // Let's pretend a 0 shift left has been performed.
3046 // FIXME: Currently we limit this to the bigger pattern case,
3047 // because some optimizations expect AND and not UBFM
3048 Opd0 = N->getOperand(Num: 0);
3049 } else
3050 return false;
3051
3052 // Missing combines/constant folding may have left us with strange
3053 // constants.
3054 if (ShlImm >= VT.getSizeInBits()) {
3055 LLVM_DEBUG(
3056 (dbgs() << N
3057 << ": Found large shift immediate, this should not happen\n"));
3058 return false;
3059 }
3060
3061 uint64_t SrlImm = 0;
3062 if (!isIntImmediate(N: N->getOperand(Num: 1), Imm&: SrlImm))
3063 return false;
3064
3065 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
3066 "bad amount in shift node!");
3067 int immr = SrlImm - ShlImm;
3068 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
3069 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
3070 // SRA requires a signed extraction
3071 if (VT == MVT::i32)
3072 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
3073 else
3074 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
3075 return true;
3076}
3077
3078bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
3079 assert(N->getOpcode() == ISD::SIGN_EXTEND);
3080
3081 EVT VT = N->getValueType(ResNo: 0);
3082 EVT NarrowVT = N->getOperand(Num: 0)->getValueType(ResNo: 0);
3083 if (VT != MVT::i64 || NarrowVT != MVT::i32)
3084 return false;
3085
3086 uint64_t ShiftImm;
3087 SDValue Op = N->getOperand(Num: 0);
3088 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SRA, Imm&: ShiftImm))
3089 return false;
3090
3091 SDLoc dl(N);
3092 // Extend the incoming operand of the shift to 64-bits.
3093 SDValue Opd0 = Widen(CurDAG, N: Op.getOperand(i: 0));
3094 unsigned Immr = ShiftImm;
3095 unsigned Imms = NarrowVT.getSizeInBits() - 1;
3096 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3097 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3098 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::SBFMXri, VT, Ops);
3099 return true;
3100}
3101
3102static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3103 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3104 unsigned NumberOfIgnoredLowBits = 0,
3105 bool BiggerPattern = false) {
3106 if (N->getValueType(ResNo: 0) != MVT::i32 && N->getValueType(ResNo: 0) != MVT::i64)
3107 return false;
3108
3109 switch (N->getOpcode()) {
3110 default:
3111 if (!N->isMachineOpcode())
3112 return false;
3113 break;
3114 case ISD::AND:
3115 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, LSB&: Immr, MSB&: Imms,
3116 NumberOfIgnoredLowBits, BiggerPattern);
3117 case ISD::SRL:
3118 case ISD::SRA:
3119 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3120
3121 case ISD::SIGN_EXTEND_INREG:
3122 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3123 }
3124
3125 unsigned NOpc = N->getMachineOpcode();
3126 switch (NOpc) {
3127 default:
3128 return false;
3129 case AArch64::SBFMWri:
3130 case AArch64::UBFMWri:
3131 case AArch64::SBFMXri:
3132 case AArch64::UBFMXri:
3133 Opc = NOpc;
3134 Opd0 = N->getOperand(Num: 0);
3135 Immr = N->getConstantOperandVal(Num: 1);
3136 Imms = N->getConstantOperandVal(Num: 2);
3137 return true;
3138 }
3139 // Unreachable
3140 return false;
3141}
3142
3143bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3144 unsigned Opc, Immr, Imms;
3145 SDValue Opd0;
3146 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3147 return false;
3148
3149 EVT VT = N->getValueType(ResNo: 0);
3150 SDLoc dl(N);
3151
3152 // If the bit extract operation is 64bit but the original type is 32bit, we
3153 // need to add one EXTRACT_SUBREG.
3154 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3155 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT: MVT::i64),
3156 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT: MVT::i64)};
3157
3158 SDNode *BFM = CurDAG->getMachineNode(Opcode: Opc, dl, VT: MVT::i64, Ops: Ops64);
3159 SDValue Inner = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sub_32, DL: dl,
3160 VT: MVT::i32, Operand: SDValue(BFM, 0));
3161 ReplaceNode(F: N, T: Inner.getNode());
3162 return true;
3163 }
3164
3165 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Val: Immr, DL: dl, VT),
3166 CurDAG->getTargetConstant(Val: Imms, DL: dl, VT)};
3167 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3168 return true;
3169}
3170
3171/// Does DstMask form a complementary pair with the mask provided by
3172/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3173/// this asks whether DstMask zeroes precisely those bits that will be set by
3174/// the other half.
3175static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3176 unsigned NumberOfIgnoredHighBits, EVT VT) {
3177 assert((VT == MVT::i32 || VT == MVT::i64) &&
3178 "i32 or i64 mask type expected!");
3179 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3180
3181 // Enable implicitTrunc as we're intentionally ignoring high bits.
3182 APInt SignificantDstMask =
3183 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3184 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(width: BitWidth);
3185
3186 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3187 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3188}
3189
3190// Look for bits that will be useful for later uses.
3191// A bit is consider useless as soon as it is dropped and never used
3192// before it as been dropped.
3193// E.g., looking for useful bit of x
3194// 1. y = x & 0x7
3195// 2. z = y >> 2
3196// After #1, x useful bits are 0x7, then the useful bits of x, live through
3197// y.
3198// After #2, the useful bits of x are 0x4.
3199// However, if x is used on an unpredictable instruction, then all its bits
3200// are useful.
3201// E.g.
3202// 1. y = x & 0x7
3203// 2. z = y >> 2
3204// 3. str x, [@x]
3205static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3206
3207static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
3208 unsigned Depth) {
3209 uint64_t Imm =
3210 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3211 Imm = AArch64_AM::decodeLogicalImmediate(val: Imm, regSize: UsefulBits.getBitWidth());
3212 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3213 getUsefulBits(Op, UsefulBits, Depth: Depth + 1);
3214}
3215
3216static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
3217 uint64_t Imm, uint64_t MSB,
3218 unsigned Depth) {
3219 // inherit the bitwidth value
3220 APInt OpUsefulBits(UsefulBits);
3221 OpUsefulBits = 1;
3222
3223 if (MSB >= Imm) {
3224 OpUsefulBits <<= MSB - Imm + 1;
3225 --OpUsefulBits;
3226 // The interesting part will be in the lower part of the result
3227 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3228 // The interesting part was starting at Imm in the argument
3229 OpUsefulBits <<= Imm;
3230 } else {
3231 OpUsefulBits <<= MSB + 1;
3232 --OpUsefulBits;
3233 // The interesting part will be shifted in the result
3234 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3235 getUsefulBits(Op, UsefulBits&: OpUsefulBits, Depth: Depth + 1);
3236 // The interesting part was at zero in the argument
3237 OpUsefulBits.lshrInPlace(ShiftAmt: OpUsefulBits.getBitWidth() - Imm);
3238 }
3239
3240 UsefulBits &= OpUsefulBits;
3241}
3242
3243static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3244 unsigned Depth) {
3245 uint64_t Imm =
3246 cast<const ConstantSDNode>(Val: Op.getOperand(i: 1).getNode())->getZExtValue();
3247 uint64_t MSB =
3248 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3249
3250 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3251}
3252
3253static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
3254 unsigned Depth) {
3255 uint64_t ShiftTypeAndValue =
3256 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3257 APInt Mask(UsefulBits);
3258 Mask.clearAllBits();
3259 Mask.flipAllBits();
3260
3261 if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSL) {
3262 // Shift Left
3263 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3264 Mask <<= ShiftAmt;
3265 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3266 Mask.lshrInPlace(ShiftAmt);
3267 } else if (AArch64_AM::getShiftType(Imm: ShiftTypeAndValue) == AArch64_AM::LSR) {
3268 // Shift Right
3269 // We do not handle AArch64_AM::ASR, because the sign will change the
3270 // number of useful bits
3271 uint64_t ShiftAmt = AArch64_AM::getShiftValue(Imm: ShiftTypeAndValue);
3272 Mask.lshrInPlace(ShiftAmt);
3273 getUsefulBits(Op, UsefulBits&: Mask, Depth: Depth + 1);
3274 Mask <<= ShiftAmt;
3275 } else
3276 return;
3277
3278 UsefulBits &= Mask;
3279}
3280
3281static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3282 unsigned Depth) {
3283 uint64_t Imm =
3284 cast<const ConstantSDNode>(Val: Op.getOperand(i: 2).getNode())->getZExtValue();
3285 uint64_t MSB =
3286 cast<const ConstantSDNode>(Val: Op.getOperand(i: 3).getNode())->getZExtValue();
3287
3288 APInt OpUsefulBits(UsefulBits);
3289 OpUsefulBits = 1;
3290
3291 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3292 ResultUsefulBits.flipAllBits();
3293 APInt Mask(UsefulBits.getBitWidth(), 0);
3294
3295 getUsefulBits(Op, UsefulBits&: ResultUsefulBits, Depth: Depth + 1);
3296
3297 if (MSB >= Imm) {
3298 // The instruction is a BFXIL.
3299 uint64_t Width = MSB - Imm + 1;
3300 uint64_t LSB = Imm;
3301
3302 OpUsefulBits <<= Width;
3303 --OpUsefulBits;
3304
3305 if (Op.getOperand(i: 1) == Orig) {
3306 // Copy the low bits from the result to bits starting from LSB.
3307 Mask = ResultUsefulBits & OpUsefulBits;
3308 Mask <<= LSB;
3309 }
3310
3311 if (Op.getOperand(i: 0) == Orig)
3312 // Bits starting from LSB in the input contribute to the result.
3313 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3314 } else {
3315 // The instruction is a BFI.
3316 uint64_t Width = MSB + 1;
3317 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3318
3319 OpUsefulBits <<= Width;
3320 --OpUsefulBits;
3321 OpUsefulBits <<= LSB;
3322
3323 if (Op.getOperand(i: 1) == Orig) {
3324 // Copy the bits from the result to the zero bits.
3325 Mask = ResultUsefulBits & OpUsefulBits;
3326 Mask.lshrInPlace(ShiftAmt: LSB);
3327 }
3328
3329 if (Op.getOperand(i: 0) == Orig)
3330 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3331 }
3332
3333 UsefulBits &= Mask;
3334}
3335
3336static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3337 SDValue Orig, unsigned Depth) {
3338
3339 // Users of this node should have already been instruction selected
3340 // FIXME: Can we turn that into an assert?
3341 if (!UserNode->isMachineOpcode())
3342 return;
3343
3344 switch (UserNode->getMachineOpcode()) {
3345 default:
3346 return;
3347 case AArch64::ANDSWri:
3348 case AArch64::ANDSXri:
3349 case AArch64::ANDWri:
3350 case AArch64::ANDXri:
3351 // We increment Depth only when we call the getUsefulBits
3352 return getUsefulBitsFromAndWithImmediate(Op: SDValue(UserNode, 0), UsefulBits,
3353 Depth);
3354 case AArch64::UBFMWri:
3355 case AArch64::UBFMXri:
3356 return getUsefulBitsFromUBFM(Op: SDValue(UserNode, 0), UsefulBits, Depth);
3357
3358 case AArch64::ORRWrs:
3359 case AArch64::ORRXrs:
3360 if (UserNode->getOperand(Num: 0) != Orig && UserNode->getOperand(Num: 1) == Orig)
3361 getUsefulBitsFromOrWithShiftedReg(Op: SDValue(UserNode, 0), UsefulBits,
3362 Depth);
3363 return;
3364 case AArch64::BFMWri:
3365 case AArch64::BFMXri:
3366 return getUsefulBitsFromBFM(Op: SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3367
3368 case AArch64::STRBBui:
3369 case AArch64::STURBBi:
3370 if (UserNode->getOperand(Num: 0) != Orig)
3371 return;
3372 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3373 return;
3374
3375 case AArch64::STRHHui:
3376 case AArch64::STURHHi:
3377 if (UserNode->getOperand(Num: 0) != Orig)
3378 return;
3379 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3380 return;
3381 }
3382}
3383
3384static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3385 if (Depth >= SelectionDAG::MaxRecursionDepth)
3386 return;
3387 // Initialize UsefulBits
3388 if (!Depth) {
3389 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3390 // At the beginning, assume every produced bits is useful
3391 UsefulBits = APInt(Bitwidth, 0);
3392 UsefulBits.flipAllBits();
3393 }
3394 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3395
3396 for (SDNode *Node : Op.getNode()->users()) {
3397 // A use cannot produce useful bits
3398 APInt UsefulBitsForUse = APInt(UsefulBits);
3399 getUsefulBitsForUse(UserNode: Node, UsefulBits&: UsefulBitsForUse, Orig: Op, Depth);
3400 UsersUsefulBits |= UsefulBitsForUse;
3401 }
3402 // UsefulBits contains the produced bits that are meaningful for the
3403 // current definition, thus a user cannot make a bit meaningful at
3404 // this point
3405 UsefulBits &= UsersUsefulBits;
3406}
3407
3408/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3409/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3410/// 0, return Op unchanged.
3411static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3412 if (ShlAmount == 0)
3413 return Op;
3414
3415 EVT VT = Op.getValueType();
3416 SDLoc dl(Op);
3417 unsigned BitWidth = VT.getSizeInBits();
3418 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3419
3420 SDNode *ShiftNode;
3421 if (ShlAmount > 0) {
3422 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3423 ShiftNode = CurDAG->getMachineNode(
3424 Opcode: UBFMOpc, dl, VT, Op1: Op,
3425 Op2: CurDAG->getTargetConstant(Val: BitWidth - ShlAmount, DL: dl, VT),
3426 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1 - ShlAmount, DL: dl, VT));
3427 } else {
3428 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3429 assert(ShlAmount < 0 && "expected right shift");
3430 int ShrAmount = -ShlAmount;
3431 ShiftNode = CurDAG->getMachineNode(
3432 Opcode: UBFMOpc, dl, VT, Op1: Op, Op2: CurDAG->getTargetConstant(Val: ShrAmount, DL: dl, VT),
3433 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL: dl, VT));
3434 }
3435
3436 return SDValue(ShiftNode, 0);
3437}
3438
3439// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3440static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3441 bool BiggerPattern,
3442 const uint64_t NonZeroBits,
3443 SDValue &Src, int &DstLSB,
3444 int &Width);
3445
3446// For bit-field-positioning pattern "shl VAL, N)".
3447static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3448 bool BiggerPattern,
3449 const uint64_t NonZeroBits,
3450 SDValue &Src, int &DstLSB,
3451 int &Width);
3452
3453/// Does this tree qualify as an attempt to move a bitfield into position,
3454/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3455static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
3456 bool BiggerPattern, SDValue &Src,
3457 int &DstLSB, int &Width) {
3458 EVT VT = Op.getValueType();
3459 unsigned BitWidth = VT.getSizeInBits();
3460 (void)BitWidth;
3461 assert(BitWidth == 32 || BitWidth == 64);
3462
3463 KnownBits Known = CurDAG->computeKnownBits(Op);
3464
3465 // Non-zero in the sense that they're not provably zero, which is the key
3466 // point if we want to use this value
3467 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3468 if (!isShiftedMask_64(Value: NonZeroBits))
3469 return false;
3470
3471 switch (Op.getOpcode()) {
3472 default:
3473 break;
3474 case ISD::AND:
3475 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3476 NonZeroBits, Src, DstLSB, Width);
3477 case ISD::SHL:
3478 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3479 NonZeroBits, Src, DstLSB, Width);
3480 }
3481
3482 return false;
3483}
3484
3485static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3486 bool BiggerPattern,
3487 const uint64_t NonZeroBits,
3488 SDValue &Src, int &DstLSB,
3489 int &Width) {
3490 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3491
3492 EVT VT = Op.getValueType();
3493 assert((VT == MVT::i32 || VT == MVT::i64) &&
3494 "Caller guarantees VT is one of i32 or i64");
3495 (void)VT;
3496
3497 uint64_t AndImm;
3498 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::AND, Imm&: AndImm))
3499 return false;
3500
3501 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3502 // 1) (AndImm & (1 << POS) == 0)
3503 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3504 //
3505 // 1) and 2) don't agree so something must be wrong (e.g., in
3506 // 'SelectionDAG::computeKnownBits')
3507 assert((~AndImm & NonZeroBits) == 0 &&
3508 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3509
3510 SDValue AndOp0 = Op.getOperand(i: 0);
3511
3512 uint64_t ShlImm;
3513 SDValue ShlOp0;
3514 if (isOpcWithIntImmediate(N: AndOp0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3515 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3516 ShlOp0 = AndOp0.getOperand(i: 0);
3517 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3518 isOpcWithIntImmediate(N: AndOp0.getOperand(i: 0).getNode(), Opc: ISD::SHL,
3519 Imm&: ShlImm)) {
3520 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3521
3522 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3523 SDValue ShlVal = AndOp0.getOperand(i: 0);
3524
3525 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3526 // expect VT to be MVT::i32.
3527 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3528
3529 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3530 ShlOp0 = Widen(CurDAG, N: ShlVal.getOperand(i: 0));
3531 } else
3532 return false;
3533
3534 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3535 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3536 // AndOp0+AND.
3537 if (!BiggerPattern && !AndOp0.hasOneUse())
3538 return false;
3539
3540 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3541 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3542
3543 // Bail out on large Width. This happens when no proper combining / constant
3544 // folding was performed.
3545 if (Width >= (int)VT.getSizeInBits()) {
3546 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3547 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3548 // "val".
3549 // If VT is i32, what Width >= 32 means:
3550 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3551 // demands at least 'Width' bits (after dag-combiner). This together with
3552 // `any_extend` Op (undefined higher bits) indicates missed combination
3553 // when lowering the 'and' IR instruction to an machine IR instruction.
3554 LLVM_DEBUG(
3555 dbgs()
3556 << "Found large Width in bit-field-positioning -- this indicates no "
3557 "proper combining / constant folding was performed\n");
3558 return false;
3559 }
3560
3561 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3562 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3563 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3564 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3565 // which case it is not profitable to insert an extra shift.
3566 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3567 return false;
3568
3569 Src = getLeftShift(CurDAG, Op: ShlOp0, ShlAmount: ShlImm - DstLSB);
3570 return true;
3571}
3572
3573// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3574// UBFIZ.
3575static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op,
3576 SDValue &Src, int &DstLSB,
3577 int &Width) {
3578 // Caller should have verified that N is a left shift with constant shift
3579 // amount; asserts that.
3580 assert(Op.getOpcode() == ISD::SHL &&
3581 "Op.getNode() should be a SHL node to call this function");
3582 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3583 "Op.getNode() should shift ShlImm to call this function");
3584
3585 uint64_t AndImm = 0;
3586 SDValue Op0 = Op.getOperand(i: 0);
3587 if (!isOpcWithIntImmediate(N: Op0.getNode(), Opc: ISD::AND, Imm&: AndImm))
3588 return false;
3589
3590 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3591 if (isMask_64(Value: ShiftedAndImm)) {
3592 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3593 // should end with Mask, and could be prefixed with random bits if those
3594 // bits are shifted out.
3595 //
3596 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3597 // the AND result corresponding to those bits are shifted out, so it's fine
3598 // to not extract them.
3599 Width = llvm::countr_one(Value: ShiftedAndImm);
3600 DstLSB = ShlImm;
3601 Src = Op0.getOperand(i: 0);
3602 return true;
3603 }
3604 return false;
3605}
3606
3607static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3608 bool BiggerPattern,
3609 const uint64_t NonZeroBits,
3610 SDValue &Src, int &DstLSB,
3611 int &Width) {
3612 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3613
3614 EVT VT = Op.getValueType();
3615 assert((VT == MVT::i32 || VT == MVT::i64) &&
3616 "Caller guarantees that type is i32 or i64");
3617 (void)VT;
3618
3619 uint64_t ShlImm;
3620 if (!isOpcWithIntImmediate(N: Op.getNode(), Opc: ISD::SHL, Imm&: ShlImm))
3621 return false;
3622
3623 if (!BiggerPattern && !Op.hasOneUse())
3624 return false;
3625
3626 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3627 return true;
3628
3629 DstLSB = llvm::countr_zero(Val: NonZeroBits);
3630 Width = llvm::countr_one(Value: NonZeroBits >> DstLSB);
3631
3632 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3633 return false;
3634
3635 Src = getLeftShift(CurDAG, Op: Op.getOperand(i: 0), ShlAmount: ShlImm - DstLSB);
3636 return true;
3637}
3638
3639static bool isShiftedMask(uint64_t Mask, EVT VT) {
3640 assert(VT == MVT::i32 || VT == MVT::i64);
3641 if (VT == MVT::i32)
3642 return isShiftedMask_32(Value: Mask);
3643 return isShiftedMask_64(Value: Mask);
3644}
3645
3646// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3647// inserted only sets known zero bits.
3648static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
3649 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3650
3651 EVT VT = N->getValueType(ResNo: 0);
3652 if (VT != MVT::i32 && VT != MVT::i64)
3653 return false;
3654
3655 unsigned BitWidth = VT.getSizeInBits();
3656
3657 uint64_t OrImm;
3658 if (!isOpcWithIntImmediate(N, Opc: ISD::OR, Imm&: OrImm))
3659 return false;
3660
3661 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3662 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3663 // performance neutral.
3664 if (AArch64_AM::isLogicalImmediate(imm: OrImm, regSize: BitWidth))
3665 return false;
3666
3667 uint64_t MaskImm;
3668 SDValue And = N->getOperand(Num: 0);
3669 // Must be a single use AND with an immediate operand.
3670 if (!And.hasOneUse() ||
3671 !isOpcWithIntImmediate(N: And.getNode(), Opc: ISD::AND, Imm&: MaskImm))
3672 return false;
3673
3674 // Compute the Known Zero for the AND as this allows us to catch more general
3675 // cases than just looking for AND with imm.
3676 KnownBits Known = CurDAG->computeKnownBits(Op: And);
3677
3678 // Non-zero in the sense that they're not provably zero, which is the key
3679 // point if we want to use this value.
3680 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3681
3682 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3683 if (!isShiftedMask(Mask: Known.Zero.getZExtValue(), VT))
3684 return false;
3685
3686 // The bits being inserted must only set those bits that are known to be zero.
3687 if ((OrImm & NotKnownZero) != 0) {
3688 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3689 // currently handle this case.
3690 return false;
3691 }
3692
3693 // BFI/BFXIL dst, src, #lsb, #width.
3694 int LSB = llvm::countr_one(Value: NotKnownZero);
3695 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3696
3697 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3698 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3699 unsigned ImmS = Width - 1;
3700
3701 // If we're creating a BFI instruction avoid cases where we need more
3702 // instructions to materialize the BFI constant as compared to the original
3703 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3704 // should be no worse in this case.
3705 bool IsBFI = LSB != 0;
3706 uint64_t BFIImm = OrImm >> LSB;
3707 if (IsBFI && !AArch64_AM::isLogicalImmediate(imm: BFIImm, regSize: BitWidth)) {
3708 // We have a BFI instruction and we know the constant can't be materialized
3709 // with a ORR-immediate with the zero register.
3710 unsigned OrChunks = 0, BFIChunks = 0;
3711 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3712 if (((OrImm >> Shift) & 0xFFFF) != 0)
3713 ++OrChunks;
3714 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3715 ++BFIChunks;
3716 }
3717 if (BFIChunks > OrChunks)
3718 return false;
3719 }
3720
3721 // Materialize the constant to be inserted.
3722 SDLoc DL(N);
3723 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3724 SDNode *MOVI = CurDAG->getMachineNode(
3725 Opcode: MOVIOpc, dl: DL, VT, Op1: CurDAG->getTargetConstant(Val: BFIImm, DL, VT));
3726
3727 // Create the BFI/BFXIL instruction.
3728 SDValue Ops[] = {And.getOperand(i: 0), SDValue(MOVI, 0),
3729 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
3730 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
3731 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3732 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
3733 return true;
3734}
3735
3736static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG,
3737 SDValue &ShiftedOperand,
3738 uint64_t &EncodedShiftImm) {
3739 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3740 if (!Dst.hasOneUse())
3741 return false;
3742
3743 EVT VT = Dst.getValueType();
3744 assert((VT == MVT::i32 || VT == MVT::i64) &&
3745 "Caller should guarantee that VT is one of i32 or i64");
3746 const unsigned SizeInBits = VT.getSizeInBits();
3747
3748 SDLoc DL(Dst.getNode());
3749 uint64_t AndImm, ShlImm;
3750 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::AND, Imm&: AndImm) &&
3751 isShiftedMask_64(Value: AndImm)) {
3752 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3753 SDValue DstOp0 = Dst.getOperand(i: 0);
3754 if (!DstOp0.hasOneUse())
3755 return false;
3756
3757 // An example to illustrate the transformation
3758 // From:
3759 // lsr x8, x1, #1
3760 // and x8, x8, #0x3f80
3761 // bfxil x8, x1, #0, #7
3762 // To:
3763 // and x8, x23, #0x7f
3764 // ubfx x9, x23, #8, #7
3765 // orr x23, x8, x9, lsl #7
3766 //
3767 // The number of instructions remains the same, but ORR is faster than BFXIL
3768 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3769 // the dependency chain is improved after the transformation.
3770 uint64_t SrlImm;
3771 if (isOpcWithIntImmediate(N: DstOp0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3772 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(Val: AndImm);
3773 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3774 unsigned MaskWidth =
3775 llvm::countr_one(Value: AndImm >> NumTrailingZeroInShiftedMask);
3776 unsigned UBFMOpc =
3777 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3778 SDNode *UBFMNode = CurDAG->getMachineNode(
3779 Opcode: UBFMOpc, dl: DL, VT, Op1: DstOp0.getOperand(i: 0),
3780 Op2: CurDAG->getTargetConstant(Val: SrlImm + NumTrailingZeroInShiftedMask, DL,
3781 VT),
3782 Op3: CurDAG->getTargetConstant(
3783 Val: SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3784 ShiftedOperand = SDValue(UBFMNode, 0);
3785 EncodedShiftImm = AArch64_AM::getShifterImm(
3786 ST: AArch64_AM::LSL, Imm: NumTrailingZeroInShiftedMask);
3787 return true;
3788 }
3789 }
3790 return false;
3791 }
3792
3793 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3794 ShiftedOperand = Dst.getOperand(i: 0);
3795 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm);
3796 return true;
3797 }
3798
3799 uint64_t SrlImm;
3800 if (isOpcWithIntImmediate(N: Dst.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3801 ShiftedOperand = Dst.getOperand(i: 0);
3802 EncodedShiftImm = AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm);
3803 return true;
3804 }
3805 return false;
3806}
3807
3808// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3809// the operands and select it to AArch64::ORR with shifted registers if
3810// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3811static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3812 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3813 const bool BiggerPattern) {
3814 EVT VT = N->getValueType(ResNo: 0);
3815 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3816 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3817 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3818 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3819 assert((VT == MVT::i32 || VT == MVT::i64) &&
3820 "Expect result type to be i32 or i64 since N is combinable to BFM");
3821 SDLoc DL(N);
3822
3823 // Bail out if BFM simplifies away one node in BFM Dst.
3824 if (OrOpd1 != Dst)
3825 return false;
3826
3827 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3828 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3829 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3830 if (BiggerPattern) {
3831 uint64_t SrcAndImm;
3832 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::AND, Imm&: SrcAndImm) &&
3833 isMask_64(Value: SrcAndImm) && OrOpd0.getOperand(i: 0) == Src) {
3834 // OrOpd0 = AND Src, #Mask
3835 // So BFM simplifies away one AND node from Src and doesn't simplify away
3836 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3837 // one node (from Rd), ORR is better since it has higher throughput and
3838 // smaller latency than BFM on many AArch64 processors (and for the rest
3839 // ORR is at least as good as BFM).
3840 SDValue ShiftedOperand;
3841 uint64_t EncodedShiftImm;
3842 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3843 EncodedShiftImm)) {
3844 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3845 CurDAG->getTargetConstant(Val: EncodedShiftImm, DL, VT)};
3846 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3847 return true;
3848 }
3849 }
3850 return false;
3851 }
3852
3853 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3854
3855 uint64_t ShlImm;
3856 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SHL, Imm&: ShlImm)) {
3857 if (OrOpd0.getOperand(i: 0) == Src && OrOpd0.hasOneUse()) {
3858 SDValue Ops[] = {
3859 Dst, Src,
3860 CurDAG->getTargetConstant(
3861 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3862 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3863 return true;
3864 }
3865
3866 // Select the following pattern to left-shifted operand rather than BFI.
3867 // %val1 = op ..
3868 // %val2 = shl %val1, #imm
3869 // %res = or %val1, %val2
3870 //
3871 // If N is selected to be BFI, we know that
3872 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3873 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3874 //
3875 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3876 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3877 SDValue Ops[] = {
3878 OrOpd1, OrOpd1,
3879 CurDAG->getTargetConstant(
3880 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: ShlImm), DL, VT)};
3881 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3882 return true;
3883 }
3884 }
3885
3886 uint64_t SrlImm;
3887 if (isOpcWithIntImmediate(N: OrOpd0.getNode(), Opc: ISD::SRL, Imm&: SrlImm)) {
3888 // Select the following pattern to right-shifted operand rather than BFXIL.
3889 // %val1 = op ..
3890 // %val2 = lshr %val1, #imm
3891 // %res = or %val1, %val2
3892 //
3893 // If N is selected to be BFXIL, we know that
3894 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3895 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
3896 //
3897 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
3898 if (OrOpd0.getOperand(i: 0) == OrOpd1) {
3899 SDValue Ops[] = {
3900 OrOpd1, OrOpd1,
3901 CurDAG->getTargetConstant(
3902 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSR, Imm: SrlImm), DL, VT)};
3903 CurDAG->SelectNodeTo(N, MachineOpc: OrrOpc, VT, Ops);
3904 return true;
3905 }
3906 }
3907
3908 return false;
3909}
3910
3911static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
3912 SelectionDAG *CurDAG) {
3913 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3914
3915 EVT VT = N->getValueType(ResNo: 0);
3916 if (VT != MVT::i32 && VT != MVT::i64)
3917 return false;
3918
3919 unsigned BitWidth = VT.getSizeInBits();
3920
3921 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
3922 // have the expected shape. Try to undo that.
3923
3924 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
3925 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
3926
3927 // Given a OR operation, check if we have the following pattern
3928 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
3929 // isBitfieldExtractOp)
3930 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
3931 // countTrailingZeros(mask2) == imm2 - imm + 1
3932 // f = d | c
3933 // if yes, replace the OR instruction with:
3934 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
3935
3936 // OR is commutative, check all combinations of operand order and values of
3937 // BiggerPattern, i.e.
3938 // Opd0, Opd1, BiggerPattern=false
3939 // Opd1, Opd0, BiggerPattern=false
3940 // Opd0, Opd1, BiggerPattern=true
3941 // Opd1, Opd0, BiggerPattern=true
3942 // Several of these combinations may match, so check with BiggerPattern=false
3943 // first since that will produce better results by matching more instructions
3944 // and/or inserting fewer extra instructions.
3945 for (int I = 0; I < 4; ++I) {
3946
3947 SDValue Dst, Src;
3948 unsigned ImmR, ImmS;
3949 bool BiggerPattern = I / 2;
3950 SDValue OrOpd0Val = N->getOperand(Num: I % 2);
3951 SDNode *OrOpd0 = OrOpd0Val.getNode();
3952 SDValue OrOpd1Val = N->getOperand(Num: (I + 1) % 2);
3953 SDNode *OrOpd1 = OrOpd1Val.getNode();
3954
3955 unsigned BFXOpc;
3956 int DstLSB, Width;
3957 if (isBitfieldExtractOp(CurDAG, N: OrOpd0, Opc&: BFXOpc, Opd0&: Src, Immr&: ImmR, Imms&: ImmS,
3958 NumberOfIgnoredLowBits, BiggerPattern)) {
3959 // Check that the returned opcode is compatible with the pattern,
3960 // i.e., same type and zero extended (U and not S)
3961 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
3962 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
3963 continue;
3964
3965 // Compute the width of the bitfield insertion
3966 DstLSB = 0;
3967 Width = ImmS - ImmR + 1;
3968 // FIXME: This constraint is to catch bitfield insertion we may
3969 // want to widen the pattern if we want to grab general bitfield
3970 // move case
3971 if (Width <= 0)
3972 continue;
3973
3974 // If the mask on the insertee is correct, we have a BFXIL operation. We
3975 // can share the ImmR and ImmS values from the already-computed UBFM.
3976 } else if (isBitfieldPositioningOp(CurDAG, Op: OrOpd0Val,
3977 BiggerPattern,
3978 Src, DstLSB, Width)) {
3979 ImmR = (BitWidth - DstLSB) % BitWidth;
3980 ImmS = Width - 1;
3981 } else
3982 continue;
3983
3984 // Check the second part of the pattern
3985 EVT VT = OrOpd1Val.getValueType();
3986 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
3987
3988 // Compute the Known Zero for the candidate of the first operand.
3989 // This allows to catch more general case than just looking for
3990 // AND with imm. Indeed, simplify-demanded-bits may have removed
3991 // the AND instruction because it proves it was useless.
3992 KnownBits Known = CurDAG->computeKnownBits(Op: OrOpd1Val);
3993
3994 // Check if there is enough room for the second operand to appear
3995 // in the first one
3996 APInt BitsToBeInserted =
3997 APInt::getBitsSet(numBits: Known.getBitWidth(), loBit: DstLSB, hiBit: DstLSB + Width);
3998
3999 if ((BitsToBeInserted & ~Known.Zero) != 0)
4000 continue;
4001
4002 // Set the first operand
4003 uint64_t Imm;
4004 if (isOpcWithIntImmediate(N: OrOpd1, Opc: ISD::AND, Imm) &&
4005 isBitfieldDstMask(DstMask: Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
4006 // In that case, we can eliminate the AND
4007 Dst = OrOpd1->getOperand(Num: 0);
4008 else
4009 // Maybe the AND has been removed by simplify-demanded-bits
4010 // or is useful because it discards more bits
4011 Dst = OrOpd1Val;
4012
4013 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
4014 // with shifted operand is more efficient.
4015 if (tryOrrWithShift(N, OrOpd0: OrOpd0Val, OrOpd1: OrOpd1Val, Src, Dst, CurDAG,
4016 BiggerPattern))
4017 return true;
4018
4019 // both parts match
4020 SDLoc DL(N);
4021 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4022 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4023 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4024 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4025 return true;
4026 }
4027
4028 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
4029 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
4030 // mask (e.g., 0x000ffff0).
4031 uint64_t Mask0Imm, Mask1Imm;
4032 SDValue And0 = N->getOperand(Num: 0);
4033 SDValue And1 = N->getOperand(Num: 1);
4034 if (And0.hasOneUse() && And1.hasOneUse() &&
4035 isOpcWithIntImmediate(N: And0.getNode(), Opc: ISD::AND, Imm&: Mask0Imm) &&
4036 isOpcWithIntImmediate(N: And1.getNode(), Opc: ISD::AND, Imm&: Mask1Imm) &&
4037 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
4038 (isShiftedMask(Mask: Mask0Imm, VT) || isShiftedMask(Mask: Mask1Imm, VT))) {
4039
4040 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
4041 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
4042 // bits to be inserted.
4043 if (isShiftedMask(Mask: Mask0Imm, VT)) {
4044 std::swap(a&: And0, b&: And1);
4045 std::swap(a&: Mask0Imm, b&: Mask1Imm);
4046 }
4047
4048 SDValue Src = And1->getOperand(Num: 0);
4049 SDValue Dst = And0->getOperand(Num: 0);
4050 unsigned LSB = llvm::countr_zero(Val: Mask1Imm);
4051 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
4052
4053 // The BFXIL inserts the low-order bits from a source register, so right
4054 // shift the needed bits into place.
4055 SDLoc DL(N);
4056 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4057 uint64_t LsrImm = LSB;
4058 if (Src->hasOneUse() &&
4059 isOpcWithIntImmediate(N: Src.getNode(), Opc: ISD::SRL, Imm&: LsrImm) &&
4060 (LsrImm + LSB) < BitWidth) {
4061 Src = Src->getOperand(Num: 0);
4062 LsrImm += LSB;
4063 }
4064
4065 SDNode *LSR = CurDAG->getMachineNode(
4066 Opcode: ShiftOpc, dl: DL, VT, Op1: Src, Op2: CurDAG->getTargetConstant(Val: LsrImm, DL, VT),
4067 Op3: CurDAG->getTargetConstant(Val: BitWidth - 1, DL, VT));
4068
4069 // BFXIL is an alias of BFM, so translate to BFM operands.
4070 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4071 unsigned ImmS = Width - 1;
4072
4073 // Create the BFXIL instruction.
4074 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4075 CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4076 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4077 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4078 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4079 return true;
4080 }
4081
4082 return false;
4083}
4084
4085bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4086 if (N->getOpcode() != ISD::OR)
4087 return false;
4088
4089 APInt NUsefulBits;
4090 getUsefulBits(Op: SDValue(N, 0), UsefulBits&: NUsefulBits);
4091
4092 // If all bits are not useful, just return UNDEF.
4093 if (!NUsefulBits) {
4094 CurDAG->SelectNodeTo(N, MachineOpc: TargetOpcode::IMPLICIT_DEF, VT: N->getValueType(ResNo: 0));
4095 return true;
4096 }
4097
4098 if (tryBitfieldInsertOpFromOr(N, UsefulBits: NUsefulBits, CurDAG))
4099 return true;
4100
4101 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4102}
4103
4104/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4105/// equivalent of a left shift by a constant amount followed by an and masking
4106/// out a contiguous set of bits.
4107bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4108 if (N->getOpcode() != ISD::AND)
4109 return false;
4110
4111 EVT VT = N->getValueType(ResNo: 0);
4112 if (VT != MVT::i32 && VT != MVT::i64)
4113 return false;
4114
4115 SDValue Op0;
4116 int DstLSB, Width;
4117 if (!isBitfieldPositioningOp(CurDAG, Op: SDValue(N, 0), /*BiggerPattern=*/false,
4118 Src&: Op0, DstLSB, Width))
4119 return false;
4120
4121 // ImmR is the rotate right amount.
4122 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4123 // ImmS is the most significant bit of the source to be moved.
4124 unsigned ImmS = Width - 1;
4125
4126 SDLoc DL(N);
4127 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(Val: ImmR, DL, VT),
4128 CurDAG->getTargetConstant(Val: ImmS, DL, VT)};
4129 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4130 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4131 return true;
4132}
4133
4134/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
4135/// variable shift/rotate instructions.
4136bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
4137 EVT VT = N->getValueType(ResNo: 0);
4138
4139 unsigned Opc;
4140 switch (N->getOpcode()) {
4141 case ISD::ROTR:
4142 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
4143 break;
4144 case ISD::SHL:
4145 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
4146 break;
4147 case ISD::SRL:
4148 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
4149 break;
4150 case ISD::SRA:
4151 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
4152 break;
4153 default:
4154 return false;
4155 }
4156
4157 uint64_t Size;
4158 uint64_t Bits;
4159 if (VT == MVT::i32) {
4160 Bits = 5;
4161 Size = 32;
4162 } else if (VT == MVT::i64) {
4163 Bits = 6;
4164 Size = 64;
4165 } else
4166 return false;
4167
4168 SDValue ShiftAmt = N->getOperand(Num: 1);
4169 SDLoc DL(N);
4170 SDValue NewShiftAmt;
4171
4172 // Skip over an extend of the shift amount.
4173 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
4174 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
4175 ShiftAmt = ShiftAmt->getOperand(Num: 0);
4176
4177 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
4178 SDValue Add0 = ShiftAmt->getOperand(Num: 0);
4179 SDValue Add1 = ShiftAmt->getOperand(Num: 1);
4180 uint64_t Add0Imm;
4181 uint64_t Add1Imm;
4182 if (isIntImmediate(N: Add1, Imm&: Add1Imm) && (Add1Imm % Size == 0)) {
4183 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
4184 // to avoid the ADD/SUB.
4185 NewShiftAmt = Add0;
4186 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4187 isIntImmediate(N: Add0, Imm&: Add0Imm) && Add0Imm != 0 &&
4188 (Add0Imm % Size == 0)) {
4189 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
4190 // to generate a NEG instead of a SUB from a constant.
4191 unsigned NegOpc;
4192 unsigned ZeroReg;
4193 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4194 if (SubVT == MVT::i32) {
4195 NegOpc = AArch64::SUBWrr;
4196 ZeroReg = AArch64::WZR;
4197 } else {
4198 assert(SubVT == MVT::i64);
4199 NegOpc = AArch64::SUBXrr;
4200 ZeroReg = AArch64::XZR;
4201 }
4202 SDValue Zero =
4203 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4204 MachineSDNode *Neg =
4205 CurDAG->getMachineNode(Opcode: NegOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4206 NewShiftAmt = SDValue(Neg, 0);
4207 } else if (ShiftAmt->getOpcode() == ISD::SUB &&
4208 isIntImmediate(N: Add0, Imm&: Add0Imm) && (Add0Imm % Size == Size - 1)) {
4209 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4210 // to generate a NOT instead of a SUB from a constant.
4211 unsigned NotOpc;
4212 unsigned ZeroReg;
4213 EVT SubVT = ShiftAmt->getValueType(ResNo: 0);
4214 if (SubVT == MVT::i32) {
4215 NotOpc = AArch64::ORNWrr;
4216 ZeroReg = AArch64::WZR;
4217 } else {
4218 assert(SubVT == MVT::i64);
4219 NotOpc = AArch64::ORNXrr;
4220 ZeroReg = AArch64::XZR;
4221 }
4222 SDValue Zero =
4223 CurDAG->getCopyFromReg(Chain: CurDAG->getEntryNode(), dl: DL, Reg: ZeroReg, VT: SubVT);
4224 MachineSDNode *Not =
4225 CurDAG->getMachineNode(Opcode: NotOpc, dl: DL, VT: SubVT, Op1: Zero, Op2: Add1);
4226 NewShiftAmt = SDValue(Not, 0);
4227 } else
4228 return false;
4229 } else {
4230 // If the shift amount is masked with an AND, check that the mask covers the
4231 // bits that are implicitly ANDed off by the above opcodes and if so, skip
4232 // the AND.
4233 uint64_t MaskImm;
4234 if (!isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: ISD::AND, Imm&: MaskImm) &&
4235 !isOpcWithIntImmediate(N: ShiftAmt.getNode(), Opc: AArch64ISD::ANDS, Imm&: MaskImm))
4236 return false;
4237
4238 if ((unsigned)llvm::countr_one(Value: MaskImm) < Bits)
4239 return false;
4240
4241 NewShiftAmt = ShiftAmt->getOperand(Num: 0);
4242 }
4243
4244 // Narrow/widen the shift amount to match the size of the shift operation.
4245 if (VT == MVT::i32)
4246 NewShiftAmt = narrowIfNeeded(CurDAG, N: NewShiftAmt);
4247 else if (VT == MVT::i64 && NewShiftAmt->getValueType(ResNo: 0) == MVT::i32) {
4248 SDValue SubReg = CurDAG->getTargetConstant(Val: AArch64::sub_32, DL, VT: MVT::i32);
4249 MachineSDNode *Ext = CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT,
4250 Op1: NewShiftAmt, Op2: SubReg);
4251 NewShiftAmt = SDValue(Ext, 0);
4252 }
4253
4254 SDValue Ops[] = {N->getOperand(Num: 0), NewShiftAmt};
4255 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
4256 return true;
4257}
4258
4259static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N,
4260 SDValue &FixedPos,
4261 unsigned RegWidth,
4262 bool isReciprocal) {
4263 APFloat FVal(0.0);
4264 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: N))
4265 FVal = CN->getValueAPF();
4266 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(Val&: N)) {
4267 // Some otherwise illegal constants are allowed in this case.
4268 if (LN->getOperand(Num: 1).getOpcode() != AArch64ISD::ADDlow ||
4269 !isa<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1)))
4270 return false;
4271
4272 ConstantPoolSDNode *CN =
4273 dyn_cast<ConstantPoolSDNode>(Val: LN->getOperand(Num: 1)->getOperand(Num: 1));
4274 FVal = cast<ConstantFP>(Val: CN->getConstVal())->getValueAPF();
4275 } else
4276 return false;
4277
4278 if (unsigned FBits =
4279 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4280 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4281 return true;
4282 }
4283
4284 return false;
4285}
4286
4287static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG,
4288 SDValue N,
4289 SDValue &FixedPos,
4290 unsigned RegWidth,
4291 bool isReciprocal) {
4292 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4293 N.getValueType().getScalarSizeInBits() ==
4294 N.getOperand(i: 0).getValueType().getScalarSizeInBits())
4295 N = N.getOperand(i: 0);
4296
4297 auto ImmToFloat = [RegWidth](APInt Imm) {
4298 switch (RegWidth) {
4299 case 16:
4300 return APFloat(APFloat::IEEEhalf(), Imm);
4301 case 32:
4302 return APFloat(APFloat::IEEEsingle(), Imm);
4303 case 64:
4304 return APFloat(APFloat::IEEEdouble(), Imm);
4305 default:
4306 llvm_unreachable("Unexpected RegWidth!");
4307 };
4308 };
4309
4310 APFloat FVal(0.0);
4311 switch (N->getOpcode()) {
4312 case AArch64ISD::MOVIshift:
4313 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(i: 0)
4314 << N.getConstantOperandVal(i: 1)));
4315 break;
4316 case AArch64ISD::FMOV:
4317 FVal = ImmToFloat(DecodeFMOVImm(Imm: N.getConstantOperandVal(i: 0), RegWidth));
4318 break;
4319 case AArch64ISD::DUP:
4320 if (isa<ConstantSDNode>(Val: N.getOperand(i: 0)))
4321 FVal = ImmToFloat(N.getConstantOperandAPInt(i: 0).trunc(width: RegWidth));
4322 else
4323 return false;
4324 break;
4325 default:
4326 return false;
4327 }
4328
4329 if (unsigned FBits =
4330 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4331 FixedPos = CurDAG->getTargetConstant(Val: FBits, DL: SDLoc(N), VT: MVT::i32);
4332 return true;
4333 }
4334
4335 return false;
4336}
4337
4338bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4339 unsigned RegWidth) {
4340 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4341 /*isReciprocal*/ false);
4342}
4343
4344bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4345 unsigned RegWidth) {
4346 return checkCVTFixedPointOperandWithFBitsForVectors(
4347 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4348}
4349
4350bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4351 SDValue &FixedPos,
4352 unsigned RegWidth) {
4353 return checkCVTFixedPointOperandWithFBitsForVectors(
4354 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4355}
4356
4357bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4358 SDValue &FixedPos,
4359 unsigned RegWidth) {
4360 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4361 /*isReciprocal*/ true);
4362}
4363
4364// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4365// of the string and obtains the integer values from them and combines these
4366// into a single value to be used in the MRS/MSR instruction.
4367static int getIntOperandFromRegisterString(StringRef RegString) {
4368 SmallVector<StringRef, 5> Fields;
4369 RegString.split(A&: Fields, Separator: ':');
4370
4371 if (Fields.size() == 1)
4372 return -1;
4373
4374 assert(Fields.size() == 5
4375 && "Invalid number of fields in read register string");
4376
4377 SmallVector<int, 5> Ops;
4378 bool AllIntFields = true;
4379
4380 for (StringRef Field : Fields) {
4381 unsigned IntField;
4382 AllIntFields &= !Field.getAsInteger(Radix: 10, Result&: IntField);
4383 Ops.push_back(Elt: IntField);
4384 }
4385
4386 assert(AllIntFields &&
4387 "Unexpected non-integer value in special register string.");
4388 (void)AllIntFields;
4389
4390 // Need to combine the integer fields of the string into a single value
4391 // based on the bit encoding of MRS/MSR instruction.
4392 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4393 (Ops[4]);
4394}
4395
4396// Lower the read_register intrinsic to an MRS instruction node if the special
4397// register string argument is either of the form detailed in the ALCE (the
4398// form described in getIntOperandsFromRegisterString) or is a named register
4399// known by the MRS SysReg mapper.
4400bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4401 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4402 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4403 SDLoc DL(N);
4404
4405 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4406
4407 unsigned Opcode64Bit = AArch64::MRS;
4408 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4409 if (Imm == -1) {
4410 // No match, Use the sysreg mapper to map the remaining possible strings to
4411 // the value for the register to be used for the instruction operand.
4412 const auto *TheReg =
4413 AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4414 if (TheReg && TheReg->Readable &&
4415 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4416 Imm = TheReg->Encoding;
4417 else
4418 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4419
4420 if (Imm == -1) {
4421 // Still no match, see if this is "pc" or give up.
4422 if (!ReadIs128Bit && RegString->getString() == "pc") {
4423 Opcode64Bit = AArch64::ADR;
4424 Imm = 0;
4425 } else {
4426 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4427 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4428 // carries the source register as an immediate so the read does not
4429 // reference an undefined physical register (which the machine verifier
4430 // rejects); the AsmPrinter materializes the real mov/fmov.
4431 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4432 RegName: RegString->getString());
4433 unsigned PseudoOp = 0;
4434 if (AArch64::GPR64RegClass.contains(Reg: PReg))
4435 PseudoOp = AArch64::READ_REGISTER_GPR64;
4436 else if (AArch64::FPR64RegClass.contains(Reg: PReg))
4437 PseudoOp = AArch64::READ_REGISTER_FPR64;
4438 if (!ReadIs128Bit && PseudoOp && N->getValueType(ResNo: 0) == MVT::i64) {
4439 CurDAG->SelectNodeTo(N, MachineOpc: PseudoOp, VT1: MVT::i64, VT2: MVT::Other,
4440 Ops: {CurDAG->getTargetConstant(Val: PReg, DL, VT: MVT::i32),
4441 N->getOperand(Num: 0)});
4442 return true;
4443 }
4444 return false;
4445 }
4446 }
4447 }
4448
4449 SDValue InChain = N->getOperand(Num: 0);
4450 SDValue SysRegImm = CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32);
4451 if (!ReadIs128Bit) {
4452 CurDAG->SelectNodeTo(N, MachineOpc: Opcode64Bit, VT1: MVT::i64, VT2: MVT::Other /* Chain */,
4453 Ops: {SysRegImm, InChain});
4454 } else {
4455 SDNode *MRRS = CurDAG->getMachineNode(
4456 Opcode: AArch64::MRRS, dl: DL,
4457 ResultTys: {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4458 Ops: {SysRegImm, InChain});
4459
4460 // Sysregs are not endian. The even register always contains the low half
4461 // of the register.
4462 SDValue Lo = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::sube64, DL, VT: MVT::i64,
4463 Operand: SDValue(MRRS, 0));
4464 SDValue Hi = CurDAG->getTargetExtractSubreg(SRIdx: AArch64::subo64, DL, VT: MVT::i64,
4465 Operand: SDValue(MRRS, 0));
4466 SDValue OutChain = SDValue(MRRS, 1);
4467
4468 ReplaceUses(F: SDValue(N, 0), T: Lo);
4469 ReplaceUses(F: SDValue(N, 1), T: Hi);
4470 ReplaceUses(F: SDValue(N, 2), T: OutChain);
4471 };
4472 return true;
4473}
4474
4475// Lower the write_register intrinsic to an MSR instruction node if the special
4476// register string argument is either of the form detailed in the ALCE (the
4477// form described in getIntOperandsFromRegisterString) or is a named register
4478// known by the MSR SysReg mapper.
4479bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4480 const auto *MD = cast<MDNodeSDNode>(Val: N->getOperand(Num: 1));
4481 const auto *RegString = cast<MDString>(Val: MD->getMD()->getOperand(I: 0));
4482 SDLoc DL(N);
4483
4484 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4485
4486 if (!WriteIs128Bit) {
4487 // Check if the register was one of those allowed as the pstatefield value
4488 // in the MSR (immediate) instruction. To accept the values allowed in the
4489 // pstatefield for the MSR (immediate) instruction, we also require that an
4490 // immediate value has been provided as an argument, we know that this is
4491 // the case as it has been ensured by semantic checking.
4492 auto trySelectPState = [&](auto PMapper, unsigned State) {
4493 if (PMapper) {
4494 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4495 "Expected a constant integer expression.");
4496 unsigned Reg = PMapper->Encoding;
4497 uint64_t Immed = N->getConstantOperandVal(Num: 2);
4498 CurDAG->SelectNodeTo(
4499 N, MachineOpc: State, VT: MVT::Other, Op1: CurDAG->getTargetConstant(Val: Reg, DL, VT: MVT::i32),
4500 Op2: CurDAG->getTargetConstant(Val: Immed, DL, VT: MVT::i16), Op3: N->getOperand(Num: 0));
4501 return true;
4502 }
4503 return false;
4504 };
4505
4506 if (trySelectPState(
4507 AArch64PState::lookupPStateImm0_15ByName(Name: RegString->getString()),
4508 AArch64::MSRpstateImm4))
4509 return true;
4510 if (trySelectPState(
4511 AArch64PState::lookupPStateImm0_1ByName(Name: RegString->getString()),
4512 AArch64::MSRpstateImm1))
4513 return true;
4514 }
4515
4516 int Imm = getIntOperandFromRegisterString(RegString: RegString->getString());
4517 if (Imm == -1) {
4518 // Use the sysreg mapper to attempt to map the remaining possible strings
4519 // to the value for the register to be used for the MSR (register)
4520 // instruction operand.
4521 auto TheReg = AArch64SysReg::lookupSysRegByName(Name: RegString->getString());
4522 if (TheReg && TheReg->Writeable &&
4523 TheReg->haveFeatures(ActiveFeatures: Subtarget->getFeatureBits()))
4524 Imm = TheReg->Encoding;
4525 else
4526 Imm = AArch64SysReg::parseGenericRegister(Name: RegString->getString());
4527
4528 if (Imm == -1) {
4529 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4530 // the physical register and keep it live with a FAKE_USE so the write is
4531 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4532 // so the generic write path cannot handle these.)
4533 Register PReg = Subtarget->getTargetLowering()->matchRegisterName(
4534 RegName: RegString->getString());
4535 bool IsGPR = AArch64::GPR64RegClass.contains(Reg: PReg);
4536 bool IsFPR = AArch64::FPR64RegClass.contains(Reg: PReg);
4537 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4538 N->getOperand(Num: 2).getValueType() == MVT::i64) {
4539 SDValue Copy =
4540 CurDAG->getCopyToReg(Chain: N->getOperand(Num: 0), dl: DL, Reg: PReg, N: N->getOperand(Num: 2));
4541 SDValue RegOp = CurDAG->getRegister(Reg: PReg, VT: MVT::i64);
4542 SDNode *FakeUse = CurDAG->getMachineNode(Opcode: TargetOpcode::FAKE_USE, dl: DL,
4543 VT: MVT::Other, Ops: {RegOp, Copy});
4544 ReplaceUses(F: SDValue(N, 0), T: SDValue(FakeUse, 0));
4545 CurDAG->RemoveDeadNode(N);
4546 return true;
4547 }
4548 return false;
4549 }
4550 }
4551
4552 SDValue InChain = N->getOperand(Num: 0);
4553 if (!WriteIs128Bit) {
4554 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSR, VT: MVT::Other,
4555 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4556 Op2: N->getOperand(Num: 2), Op3: InChain);
4557 } else {
4558 // No endian swap. The lower half always goes into the even subreg, and the
4559 // higher half always into the odd supreg.
4560 SDNode *Pair = CurDAG->getMachineNode(
4561 Opcode: TargetOpcode::REG_SEQUENCE, dl: DL, VT: MVT::Untyped /* XSeqPair */,
4562 Ops: {CurDAG->getTargetConstant(Val: AArch64::XSeqPairsClassRegClass.getID(), DL,
4563 VT: MVT::i32),
4564 N->getOperand(Num: 2),
4565 CurDAG->getTargetConstant(Val: AArch64::sube64, DL, VT: MVT::i32),
4566 N->getOperand(Num: 3),
4567 CurDAG->getTargetConstant(Val: AArch64::subo64, DL, VT: MVT::i32)});
4568
4569 CurDAG->SelectNodeTo(N, MachineOpc: AArch64::MSRR, VT: MVT::Other,
4570 Op1: CurDAG->getTargetConstant(Val: Imm, DL, VT: MVT::i32),
4571 Op2: SDValue(Pair, 0), Op3: InChain);
4572 }
4573
4574 return true;
4575}
4576
4577/// We've got special pseudo-instructions for these
4578bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4579 unsigned Opcode;
4580 EVT MemTy = cast<MemSDNode>(Val: N)->getMemoryVT();
4581
4582 // Leave IR for LSE if subtarget supports it.
4583 if (Subtarget->hasLSE()) return false;
4584
4585 if (MemTy == MVT::i8)
4586 Opcode = AArch64::CMP_SWAP_8;
4587 else if (MemTy == MVT::i16)
4588 Opcode = AArch64::CMP_SWAP_16;
4589 else if (MemTy == MVT::i32)
4590 Opcode = AArch64::CMP_SWAP_32;
4591 else if (MemTy == MVT::i64)
4592 Opcode = AArch64::CMP_SWAP_64;
4593 else
4594 llvm_unreachable("Unknown AtomicCmpSwap type");
4595
4596 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4597 SDValue Ops[] = {N->getOperand(Num: 1), N->getOperand(Num: 2), N->getOperand(Num: 3),
4598 N->getOperand(Num: 0)};
4599 SDNode *CmpSwap = CurDAG->getMachineNode(
4600 Opcode, dl: SDLoc(N),
4601 VTs: CurDAG->getVTList(VT1: RegTy, VT2: MVT::i32, VT3: MVT::Other), Ops);
4602
4603 MachineMemOperand *MemOp = cast<MemSDNode>(Val: N)->getMemOperand();
4604 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: CmpSwap), NewMemRefs: {MemOp});
4605
4606 ReplaceUses(F: SDValue(N, 0), T: SDValue(CmpSwap, 0));
4607 ReplaceUses(F: SDValue(N, 1), T: SDValue(CmpSwap, 2));
4608 CurDAG->RemoveDeadNode(N);
4609
4610 return true;
4611}
4612
4613bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4614 SDValue &Shift, bool Negate) {
4615 if (!isa<ConstantSDNode>(Val: N))
4616 return false;
4617
4618 APInt Val =
4619 cast<ConstantSDNode>(Val&: N)->getAPIntValue().trunc(width: VT.getFixedSizeInBits());
4620
4621 return SelectSVEAddSubImm(DL: SDLoc(N), Value: Val, VT, Imm, Shift, Negate);
4622}
4623
4624bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4625 SDValue &Imm, SDValue &Shift,
4626 bool Negate) {
4627 if (Negate)
4628 Val = -Val;
4629
4630 switch (VT.SimpleTy) {
4631 case MVT::i8:
4632 // All immediates are supported.
4633 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4634 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4635 return true;
4636 case MVT::i16:
4637 case MVT::i32:
4638 case MVT::i64:
4639 // Support 8bit unsigned immediates.
4640 if ((Val & ~0xff) == 0) {
4641 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4642 Imm = CurDAG->getTargetConstant(Val: Val.getZExtValue(), DL, VT: MVT::i32);
4643 return true;
4644 }
4645 // Support 16bit unsigned immediates that are a multiple of 256.
4646 if ((Val & ~0xff00) == 0) {
4647 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4648 Imm = CurDAG->getTargetConstant(Val: Val.lshr(shiftAmt: 8).getZExtValue(), DL, VT: MVT::i32);
4649 return true;
4650 }
4651 break;
4652 default:
4653 break;
4654 }
4655
4656 return false;
4657}
4658
4659bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4660 SDValue &Imm, SDValue &Shift,
4661 bool Negate) {
4662 if (!isa<ConstantSDNode>(Val: N))
4663 return false;
4664
4665 SDLoc DL(N);
4666 int64_t Val = cast<ConstantSDNode>(Val&: N)
4667 ->getAPIntValue()
4668 .trunc(width: VT.getFixedSizeInBits())
4669 .getSExtValue();
4670
4671 if (Negate)
4672 Val = -Val;
4673
4674 // Signed saturating instructions treat their immediate operand as unsigned,
4675 // whereas the related intrinsics define their operands to be signed. This
4676 // means we can only use the immediate form when the operand is non-negative.
4677 if (Val < 0)
4678 return false;
4679
4680 switch (VT.SimpleTy) {
4681 case MVT::i8:
4682 // All positive immediates are supported.
4683 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4684 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4685 return true;
4686 case MVT::i16:
4687 case MVT::i32:
4688 case MVT::i64:
4689 // Support 8bit positive immediates.
4690 if (Val <= 255) {
4691 Shift = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32);
4692 Imm = CurDAG->getTargetConstant(Val, DL, VT: MVT::i32);
4693 return true;
4694 }
4695 // Support 16bit positive immediates that are a multiple of 256.
4696 if (Val <= 65280 && Val % 256 == 0) {
4697 Shift = CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32);
4698 Imm = CurDAG->getTargetConstant(Val: Val >> 8, DL, VT: MVT::i32);
4699 return true;
4700 }
4701 break;
4702 default:
4703 break;
4704 }
4705
4706 return false;
4707}
4708
4709bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4710 SDValue &Shift) {
4711 if (!isa<ConstantSDNode>(Val: N))
4712 return false;
4713
4714 SDLoc DL(N);
4715 int64_t Val = cast<ConstantSDNode>(Val&: N)
4716 ->getAPIntValue()
4717 .trunc(width: VT.getFixedSizeInBits())
4718 .getSExtValue();
4719 int32_t ImmVal, ShiftVal;
4720 if (!AArch64_AM::isSVECpyDupImm(SizeInBits: VT.getScalarSizeInBits(), Val, Imm&: ImmVal,
4721 Shift&: ShiftVal))
4722 return false;
4723
4724 Shift = CurDAG->getTargetConstant(Val: ShiftVal, DL, VT: MVT::i32);
4725 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4726 return true;
4727}
4728
4729bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4730 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N))
4731 return SelectSVESignedArithImm(DL: SDLoc(N), Value: CNode->getAPIntValue(), Imm);
4732 return false;
4733}
4734
4735bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4736 SDValue &Imm) {
4737 int64_t ImmVal = Val.getSExtValue();
4738 if (ImmVal >= -128 && ImmVal < 128) {
4739 Imm = CurDAG->getSignedTargetConstant(Val: ImmVal, DL, VT: MVT::i32);
4740 return true;
4741 }
4742 return false;
4743}
4744
4745bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4746 if (auto CNode = dyn_cast<ConstantSDNode>(Val&: N)) {
4747 uint64_t ImmVal = CNode->getZExtValue();
4748
4749 switch (VT.SimpleTy) {
4750 case MVT::i8:
4751 ImmVal &= 0xFF;
4752 break;
4753 case MVT::i16:
4754 ImmVal &= 0xFFFF;
4755 break;
4756 case MVT::i32:
4757 ImmVal &= 0xFFFFFFFF;
4758 break;
4759 case MVT::i64:
4760 break;
4761 default:
4762 llvm_unreachable("Unexpected type");
4763 }
4764
4765 if (ImmVal < 256) {
4766 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4767 return true;
4768 }
4769 }
4770 return false;
4771}
4772
4773bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4774 bool Invert) {
4775 uint64_t ImmVal;
4776 if (auto CI = dyn_cast<ConstantSDNode>(Val&: N))
4777 ImmVal = CI->getZExtValue();
4778 else if (auto CFP = dyn_cast<ConstantFPSDNode>(Val&: N))
4779 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4780 else
4781 return false;
4782
4783 if (Invert)
4784 ImmVal = ~ImmVal;
4785
4786 uint64_t encoding;
4787 if (!AArch64_AM::isSVELogicalImm(SizeInBits: VT.getScalarSizeInBits(), ImmVal, Encoding&: encoding))
4788 return false;
4789
4790 Imm = CurDAG->getTargetConstant(Val: encoding, DL: SDLoc(N), VT: MVT::i64);
4791 return true;
4792}
4793
4794// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4795// Rather than attempt to normalise everything we can sometimes saturate the
4796// shift amount during selection. This function also allows for consistent
4797// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4798// required by the instructions.
4799bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4800 uint64_t High, bool AllowSaturation,
4801 SDValue &Imm) {
4802 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
4803 uint64_t ImmVal = CN->getZExtValue();
4804
4805 // Reject shift amounts that are too small.
4806 if (ImmVal < Low)
4807 return false;
4808
4809 // Reject or saturate shift amounts that are too big.
4810 if (ImmVal > High) {
4811 if (!AllowSaturation)
4812 return false;
4813 ImmVal = High;
4814 }
4815
4816 Imm = CurDAG->getTargetConstant(Val: ImmVal, DL: SDLoc(N), VT: MVT::i32);
4817 return true;
4818 }
4819
4820 return false;
4821}
4822
4823bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4824 // tagp(FrameIndex, IRGstack, tag_offset):
4825 // since the offset between FrameIndex and IRGstack is a compile-time
4826 // constant, this can be lowered to a single ADDG instruction.
4827 if (!(isa<FrameIndexSDNode>(Val: N->getOperand(Num: 1)))) {
4828 return false;
4829 }
4830
4831 SDValue IRG_SP = N->getOperand(Num: 2);
4832 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4833 IRG_SP->getConstantOperandVal(Num: 1) != Intrinsic::aarch64_irg_sp) {
4834 return false;
4835 }
4836
4837 const TargetLowering *TLI = getTargetLowering();
4838 SDLoc DL(N);
4839 int FI = cast<FrameIndexSDNode>(Val: N->getOperand(Num: 1))->getIndex();
4840 SDValue FiOp = CurDAG->getTargetFrameIndex(
4841 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
4842 int TagOffset = N->getConstantOperandVal(Num: 3);
4843
4844 SDNode *Out = CurDAG->getMachineNode(
4845 Opcode: AArch64::TAGPstack, dl: DL, VT: MVT::i64,
4846 Ops: {FiOp, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64), N->getOperand(Num: 2),
4847 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4848 ReplaceNode(F: N, T: Out);
4849 return true;
4850}
4851
4852void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4853 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4854 "llvm.aarch64.tagp third argument must be an immediate");
4855 if (trySelectStackSlotTagP(N))
4856 return;
4857 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4858 // compile-time constant, not just for stack allocations.
4859
4860 // General case for unrelated pointers in Op1 and Op2.
4861 SDLoc DL(N);
4862 int TagOffset = N->getConstantOperandVal(Num: 3);
4863 SDNode *N1 = CurDAG->getMachineNode(Opcode: AArch64::SUBP, dl: DL, VT: MVT::i64,
4864 Ops: {N->getOperand(Num: 1), N->getOperand(Num: 2)});
4865 SDNode *N2 = CurDAG->getMachineNode(Opcode: AArch64::ADDXrr, dl: DL, VT: MVT::i64,
4866 Ops: {SDValue(N1, 0), N->getOperand(Num: 2)});
4867 SDNode *N3 = CurDAG->getMachineNode(
4868 Opcode: AArch64::ADDG, dl: DL, VT: MVT::i64,
4869 Ops: {SDValue(N2, 0), CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64),
4870 CurDAG->getTargetConstant(Val: TagOffset, DL, VT: MVT::i64)});
4871 ReplaceNode(F: N, T: N3);
4872}
4873
4874bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4875 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4876
4877 // Bail when not a "cast" like insert_subvector.
4878 if (N->getConstantOperandVal(Num: 2) != 0)
4879 return false;
4880 if (!N->getOperand(Num: 0).isUndef())
4881 return false;
4882
4883 // Bail when normal isel should do the job.
4884 EVT VT = N->getValueType(ResNo: 0);
4885 EVT InVT = N->getOperand(Num: 1).getValueType();
4886 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4887 return false;
4888 if (InVT.getSizeInBits() <= 128)
4889 return false;
4890
4891 // NOTE: We can only get here when doing fixed length SVE code generation.
4892 // We do manual selection because the types involved are not linked to real
4893 // registers (despite being legal) and must be coerced into SVE registers.
4894
4895 assert(VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4896 "Expected to insert into a packed scalable vector!");
4897
4898 SDLoc DL(N);
4899 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4900 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4901 Op1: N->getOperand(Num: 1), Op2: RC));
4902 return true;
4903}
4904
4905bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4906 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4907
4908 // Bail when not a "cast" like extract_subvector.
4909 if (N->getConstantOperandVal(Num: 1) != 0)
4910 return false;
4911
4912 // Bail when normal isel can do the job.
4913 EVT VT = N->getValueType(ResNo: 0);
4914 EVT InVT = N->getOperand(Num: 0).getValueType();
4915 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4916 return false;
4917 if (VT.getSizeInBits() <= 128)
4918 return false;
4919
4920 // NOTE: We can only get here when doing fixed length SVE code generation.
4921 // We do manual selection because the types involved are not linked to real
4922 // registers (despite being legal) and must be coerced into SVE registers.
4923
4924 assert(InVT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock &&
4925 "Expected to extract from a packed scalable vector!");
4926
4927 SDLoc DL(N);
4928 auto RC = CurDAG->getTargetConstant(Val: AArch64::ZPRRegClassID, DL, VT: MVT::i64);
4929 ReplaceNode(F: N, T: CurDAG->getMachineNode(Opcode: TargetOpcode::COPY_TO_REGCLASS, dl: DL, VT,
4930 Op1: N->getOperand(Num: 0), Op2: RC));
4931 return true;
4932}
4933
4934bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4935 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4936
4937 SDValue N0 = N->getOperand(Num: 0);
4938 SDValue N1 = N->getOperand(Num: 1);
4939
4940 EVT VT = N->getValueType(ResNo: 0);
4941 SDLoc DL(N);
4942
4943 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4944 // Rotate by a constant is a funnel shift in IR which is expanded to
4945 // an OR with shifted operands.
4946 // We do the following transform:
4947 // OR N0, N1 -> xar (x, y, imm)
4948 // Where:
4949 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4950 // N0 = SHL_PRED true, V, splat(bits-imm)
4951 // V = (xor x, y)
4952 if (VT.isScalableVector() &&
4953 (Subtarget->hasSVE2() ||
4954 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4955 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4956 N1.getOpcode() != AArch64ISD::SRL_PRED)
4957 std::swap(a&: N0, b&: N1);
4958 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4959 N1.getOpcode() != AArch64ISD::SRL_PRED)
4960 return false;
4961
4962 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4963 if (!TLI->isAllActivePredicate(DAG: *CurDAG, N: N0.getOperand(i: 0)) ||
4964 !TLI->isAllActivePredicate(DAG: *CurDAG, N: N1.getOperand(i: 0)))
4965 return false;
4966
4967 if (N0.getOperand(i: 1) != N1.getOperand(i: 1))
4968 return false;
4969
4970 SDValue R1, R2;
4971 bool IsXOROperand = true;
4972 if (N0.getOperand(i: 1).getOpcode() != ISD::XOR) {
4973 IsXOROperand = false;
4974 } else {
4975 R1 = N0.getOperand(i: 1).getOperand(i: 0);
4976 R2 = N1.getOperand(i: 1).getOperand(i: 1);
4977 }
4978
4979 APInt ShlAmt, ShrAmt;
4980 if (!ISD::isConstantSplatVector(N: N0.getOperand(i: 2).getNode(), SplatValue&: ShlAmt) ||
4981 !ISD::isConstantSplatVector(N: N1.getOperand(i: 2).getNode(), SplatValue&: ShrAmt))
4982 return false;
4983
4984 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4985 return false;
4986
4987 if (!IsXOROperand) {
4988 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
4989 SDNode *MOV = CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT, Op1: Zero);
4990 SDValue MOVIV = SDValue(MOV, 0);
4991
4992 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
4993 SDNode *SubRegToReg =
4994 CurDAG->getMachineNode(Opcode: AArch64::SUBREG_TO_REG, dl: DL, VT, Op1: MOVIV, Op2: ZSub);
4995
4996 R1 = N1->getOperand(Num: 1);
4997 R2 = SDValue(SubRegToReg, 0);
4998 }
4999
5000 SDValue Imm =
5001 CurDAG->getTargetConstant(Val: ShrAmt.getZExtValue(), DL, VT: MVT::i32);
5002
5003 SDValue Ops[] = {R1, R2, Imm};
5004 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5005 VT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5006 AArch64::XAR_ZZZI_D})) {
5007 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Ops);
5008 return true;
5009 }
5010 return false;
5011 }
5012
5013 // We have Neon SHA3 XAR operation for v2i64 but for types
5014 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
5015 // is available.
5016 EVT SVT;
5017 switch (VT.getSimpleVT().SimpleTy) {
5018 case MVT::v4i32:
5019 case MVT::v2i32:
5020 SVT = MVT::nxv4i32;
5021 break;
5022 case MVT::v8i16:
5023 case MVT::v4i16:
5024 SVT = MVT::nxv8i16;
5025 break;
5026 case MVT::v16i8:
5027 case MVT::v8i8:
5028 SVT = MVT::nxv16i8;
5029 break;
5030 case MVT::v2i64:
5031 case MVT::v1i64:
5032 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5033 break;
5034 default:
5035 return false;
5036 }
5037
5038 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
5039 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
5040 return false;
5041
5042 if (N0->getOpcode() != AArch64ISD::VSHL ||
5043 N1->getOpcode() != AArch64ISD::VLSHR)
5044 return false;
5045
5046 if (N0->getOperand(Num: 0) != N1->getOperand(Num: 0))
5047 return false;
5048
5049 SDValue R1, R2;
5050 bool IsXOROperand = true;
5051 if (N1->getOperand(Num: 0)->getOpcode() != ISD::XOR) {
5052 IsXOROperand = false;
5053 } else {
5054 SDValue XOR = N0.getOperand(i: 0);
5055 R1 = XOR.getOperand(i: 0);
5056 R2 = XOR.getOperand(i: 1);
5057 }
5058
5059 unsigned HsAmt = N0.getConstantOperandVal(i: 1);
5060 unsigned ShAmt = N1.getConstantOperandVal(i: 1);
5061
5062 SDValue Imm = CurDAG->getTargetConstant(
5063 Val: ShAmt, DL, VT: N0.getOperand(i: 1).getValueType(), isOpaque: false);
5064
5065 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5066 if (ShAmt + HsAmt != VTSizeInBits)
5067 return false;
5068
5069 if (!IsXOROperand) {
5070 SDValue Zero = CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i64);
5071 SDNode *MOV =
5072 CurDAG->getMachineNode(Opcode: AArch64::MOVIv2d_ns, dl: DL, VT: MVT::v2i64, Op1: Zero);
5073 SDValue MOVIV = SDValue(MOV, 0);
5074
5075 R1 = N1->getOperand(Num: 0);
5076 R2 = MOVIV;
5077 }
5078
5079 if (SVT != VT) {
5080 SDValue Undef =
5081 SDValue(CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: SVT), 0);
5082
5083 if (SVT.isScalableVector() && VT.is64BitVector()) {
5084 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5085
5086 SDValue UndefQ = SDValue(
5087 CurDAG->getMachineNode(Opcode: TargetOpcode::IMPLICIT_DEF, dl: DL, VT: QVT), 0);
5088 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5089
5090 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5091 Op1: UndefQ, Op2: R1, Op3: DSub),
5092 0);
5093 if (R2.getValueType() == VT)
5094 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: QVT,
5095 Op1: UndefQ, Op2: R2, Op3: DSub),
5096 0);
5097 }
5098
5099 SDValue SubReg = CurDAG->getTargetConstant(
5100 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, VT: MVT::i32);
5101
5102 R1 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT, Op1: Undef,
5103 Op2: R1, Op3: SubReg),
5104 0);
5105
5106 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5107 R2 = SDValue(CurDAG->getMachineNode(Opcode: AArch64::INSERT_SUBREG, dl: DL, VT: SVT,
5108 Op1: Undef, Op2: R2, Op3: SubReg),
5109 0);
5110 }
5111
5112 SDValue Ops[] = {R1, R2, Imm};
5113 SDNode *XAR = nullptr;
5114
5115 if (SVT.isScalableVector()) {
5116 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::Int>(
5117 VT: SVT, Opcodes: {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5118 AArch64::XAR_ZZZI_D}))
5119 XAR = CurDAG->getMachineNode(Opcode: Opc, dl: DL, VT: SVT, Ops);
5120 } else {
5121 XAR = CurDAG->getMachineNode(Opcode: AArch64::XAR, dl: DL, VT: SVT, Ops);
5122 }
5123
5124 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5125
5126 if (SVT != VT) {
5127 if (VT.is64BitVector() && SVT.isScalableVector()) {
5128 EVT QVT = VT.getDoubleNumVectorElementsVT(Context&: *CurDAG->getContext());
5129
5130 SDValue ZSub = CurDAG->getTargetConstant(Val: AArch64::zsub, DL, VT: MVT::i32);
5131 SDNode *Q = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT: QVT,
5132 Op1: SDValue(XAR, 0), Op2: ZSub);
5133
5134 SDValue DSub = CurDAG->getTargetConstant(Val: AArch64::dsub, DL, VT: MVT::i32);
5135 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5136 Op1: SDValue(Q, 0), Op2: DSub);
5137 } else {
5138 SDValue SubReg = CurDAG->getTargetConstant(
5139 Val: (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5140 VT: MVT::i32);
5141 XAR = CurDAG->getMachineNode(Opcode: AArch64::EXTRACT_SUBREG, dl: DL, VT,
5142 Op1: SDValue(XAR, 0), Op2: SubReg);
5143 }
5144 }
5145 ReplaceNode(F: N, T: XAR);
5146 return true;
5147}
5148
5149/// Returns a copy from WZR or XZR. This can be used during instruction
5150/// selection (it does not require any further selection/legalization).
5151static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT) {
5152 assert(VT == MVT::i32 || VT == MVT::i64);
5153 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL,
5154 Reg: VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5155}
5156
5157void AArch64DAGToDAGISel::Select(SDNode *Node) {
5158 // If we have a custom node, we already have selected!
5159 if (Node->isMachineOpcode()) {
5160 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5161 Node->setNodeId(-1);
5162 return;
5163 }
5164
5165 // Few custom selection stuff.
5166 EVT VT = Node->getValueType(ResNo: 0);
5167
5168 switch (Node->getOpcode()) {
5169 default:
5170 break;
5171
5172 case ISD::ATOMIC_CMP_SWAP:
5173 if (SelectCMP_SWAP(N: Node))
5174 return;
5175 break;
5176
5177 case ISD::READ_REGISTER:
5178 case AArch64ISD::MRRS:
5179 if (tryReadRegister(N: Node))
5180 return;
5181 break;
5182
5183 case ISD::WRITE_REGISTER:
5184 case AArch64ISD::MSRR:
5185 if (tryWriteRegister(N: Node))
5186 return;
5187 break;
5188
5189 case ISD::LOAD: {
5190 // Try to select as an indexed load. Fall through to normal processing
5191 // if we can't.
5192 if (tryIndexedLoad(N: Node))
5193 return;
5194 break;
5195 }
5196
5197 case ISD::SRL:
5198 case ISD::AND:
5199 case ISD::SRA:
5200 case ISD::SIGN_EXTEND_INREG:
5201 if (tryBitfieldExtractOp(N: Node))
5202 return;
5203 if (tryBitfieldInsertInZeroOp(N: Node))
5204 return;
5205 [[fallthrough]];
5206 case ISD::ROTR:
5207 case ISD::SHL:
5208 if (tryShiftAmountMod(N: Node))
5209 return;
5210 break;
5211
5212 case ISD::SIGN_EXTEND:
5213 if (tryBitfieldExtractOpFromSExt(N: Node))
5214 return;
5215 break;
5216
5217 case ISD::OR:
5218 if (tryBitfieldInsertOp(N: Node))
5219 return;
5220 if (trySelectXAR(N: Node))
5221 return;
5222 break;
5223
5224 case ISD::EXTRACT_SUBVECTOR: {
5225 if (trySelectCastScalableToFixedLengthVector(N: Node))
5226 return;
5227 break;
5228 }
5229
5230 case ISD::INSERT_SUBVECTOR: {
5231 if (trySelectCastFixedLengthToScalableVector(N: Node))
5232 return;
5233 break;
5234 }
5235
5236 case AArch64ISD::CSEL:
5237 if (tryFoldCselToFMaxMin(N: Node))
5238 return;
5239 break;
5240
5241 case ISD::Constant: {
5242 // Materialize zero constants as copies from WZR/XZR. This allows
5243 // the coalescer to propagate these into other instructions.
5244 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Val: Node);
5245 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5246 ReplaceNode(F: Node, T: getZeroRegister(DAG&: *CurDAG, DL: SDLoc(Node), VT).getNode());
5247 return;
5248 }
5249 break;
5250 }
5251
5252 case ISD::FrameIndex: {
5253 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5254 int FI = cast<FrameIndexSDNode>(Val: Node)->getIndex();
5255 unsigned Shifter = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
5256 const TargetLowering *TLI = getTargetLowering();
5257 SDValue TFI = CurDAG->getTargetFrameIndex(
5258 FI, VT: TLI->getPointerTy(DL: CurDAG->getDataLayout()));
5259 SDLoc DL(Node);
5260 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32),
5261 CurDAG->getTargetConstant(Val: Shifter, DL, VT: MVT::i32) };
5262 CurDAG->SelectNodeTo(N: Node, MachineOpc: AArch64::ADDXri, VT: MVT::i64, Ops);
5263 return;
5264 }
5265 case ISD::INTRINSIC_W_CHAIN: {
5266 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
5267 switch (IntNo) {
5268 default:
5269 break;
5270 case Intrinsic::aarch64_gcsss: {
5271 SDLoc DL(Node);
5272 SDValue Chain = Node->getOperand(Num: 0);
5273 SDValue Val = Node->getOperand(Num: 2);
5274 SDValue Zero = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::XZR, VT: MVT::i64);
5275 SDNode *SS1 =
5276 CurDAG->getMachineNode(Opcode: AArch64::GCSSS1, dl: DL, VT: MVT::Other, Op1: Val, Op2: Chain);
5277 SDNode *SS2 = CurDAG->getMachineNode(Opcode: AArch64::GCSSS2, dl: DL, VT1: MVT::i64,
5278 VT2: MVT::Other, Op1: Zero, Op2: SDValue(SS1, 0));
5279 ReplaceNode(F: Node, T: SS2);
5280 return;
5281 }
5282 case Intrinsic::aarch64_ldaxp:
5283 case Intrinsic::aarch64_ldxp: {
5284 unsigned Op =
5285 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5286 SDValue MemAddr = Node->getOperand(Num: 2);
5287 SDLoc DL(Node);
5288 SDValue Chain = Node->getOperand(Num: 0);
5289
5290 SDNode *Ld = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i64, VT2: MVT::i64,
5291 VT3: MVT::Other, Op1: MemAddr, Op2: Chain);
5292
5293 // Transfer memoperands.
5294 MachineMemOperand *MemOp =
5295 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5296 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: Ld), NewMemRefs: {MemOp});
5297 ReplaceNode(F: Node, T: Ld);
5298 return;
5299 }
5300 case Intrinsic::aarch64_stlxp:
5301 case Intrinsic::aarch64_stxp: {
5302 unsigned Op =
5303 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5304 SDLoc DL(Node);
5305 SDValue Chain = Node->getOperand(Num: 0);
5306 SDValue ValLo = Node->getOperand(Num: 2);
5307 SDValue ValHi = Node->getOperand(Num: 3);
5308 SDValue MemAddr = Node->getOperand(Num: 4);
5309
5310 // Place arguments in the right order.
5311 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5312
5313 SDNode *St = CurDAG->getMachineNode(Opcode: Op, dl: DL, VT1: MVT::i32, VT2: MVT::Other, Ops);
5314 // Transfer memoperands.
5315 MachineMemOperand *MemOp =
5316 cast<MemIntrinsicSDNode>(Val: Node)->getMemOperand();
5317 CurDAG->setNodeMemRefs(N: cast<MachineSDNode>(Val: St), NewMemRefs: {MemOp});
5318
5319 ReplaceNode(F: Node, T: St);
5320 return;
5321 }
5322 case Intrinsic::aarch64_neon_ld1x2:
5323 if (VT == MVT::v8i8) {
5324 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b, SubRegIdx: AArch64::dsub0);
5325 return;
5326 } else if (VT == MVT::v16i8) {
5327 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b, SubRegIdx: AArch64::qsub0);
5328 return;
5329 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5330 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h, SubRegIdx: AArch64::dsub0);
5331 return;
5332 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5333 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h, SubRegIdx: AArch64::qsub0);
5334 return;
5335 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5336 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s, SubRegIdx: AArch64::dsub0);
5337 return;
5338 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5339 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s, SubRegIdx: AArch64::qsub0);
5340 return;
5341 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5342 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5343 return;
5344 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5345 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d, SubRegIdx: AArch64::qsub0);
5346 return;
5347 }
5348 break;
5349 case Intrinsic::aarch64_neon_ld1x3:
5350 if (VT == MVT::v8i8) {
5351 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b, SubRegIdx: AArch64::dsub0);
5352 return;
5353 } else if (VT == MVT::v16i8) {
5354 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b, SubRegIdx: AArch64::qsub0);
5355 return;
5356 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5357 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h, SubRegIdx: AArch64::dsub0);
5358 return;
5359 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5360 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h, SubRegIdx: AArch64::qsub0);
5361 return;
5362 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5363 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s, SubRegIdx: AArch64::dsub0);
5364 return;
5365 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5366 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s, SubRegIdx: AArch64::qsub0);
5367 return;
5368 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5369 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5370 return;
5371 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5372 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d, SubRegIdx: AArch64::qsub0);
5373 return;
5374 }
5375 break;
5376 case Intrinsic::aarch64_neon_ld1x4:
5377 if (VT == MVT::v8i8) {
5378 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b, SubRegIdx: AArch64::dsub0);
5379 return;
5380 } else if (VT == MVT::v16i8) {
5381 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b, SubRegIdx: AArch64::qsub0);
5382 return;
5383 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5384 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h, SubRegIdx: AArch64::dsub0);
5385 return;
5386 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5387 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h, SubRegIdx: AArch64::qsub0);
5388 return;
5389 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5390 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s, SubRegIdx: AArch64::dsub0);
5391 return;
5392 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5393 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s, SubRegIdx: AArch64::qsub0);
5394 return;
5395 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5396 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5397 return;
5398 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5399 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d, SubRegIdx: AArch64::qsub0);
5400 return;
5401 }
5402 break;
5403 case Intrinsic::aarch64_neon_ld2:
5404 if (VT == MVT::v8i8) {
5405 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b, SubRegIdx: AArch64::dsub0);
5406 return;
5407 } else if (VT == MVT::v16i8) {
5408 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b, SubRegIdx: AArch64::qsub0);
5409 return;
5410 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5411 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h, SubRegIdx: AArch64::dsub0);
5412 return;
5413 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5414 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h, SubRegIdx: AArch64::qsub0);
5415 return;
5416 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5417 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s, SubRegIdx: AArch64::dsub0);
5418 return;
5419 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5420 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s, SubRegIdx: AArch64::qsub0);
5421 return;
5422 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5423 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d, SubRegIdx: AArch64::dsub0);
5424 return;
5425 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5426 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d, SubRegIdx: AArch64::qsub0);
5427 return;
5428 }
5429 break;
5430 case Intrinsic::aarch64_neon_ld3:
5431 if (VT == MVT::v8i8) {
5432 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b, SubRegIdx: AArch64::dsub0);
5433 return;
5434 } else if (VT == MVT::v16i8) {
5435 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b, SubRegIdx: AArch64::qsub0);
5436 return;
5437 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5438 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h, SubRegIdx: AArch64::dsub0);
5439 return;
5440 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5441 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h, SubRegIdx: AArch64::qsub0);
5442 return;
5443 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5444 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s, SubRegIdx: AArch64::dsub0);
5445 return;
5446 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5447 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s, SubRegIdx: AArch64::qsub0);
5448 return;
5449 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5450 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d, SubRegIdx: AArch64::dsub0);
5451 return;
5452 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5453 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d, SubRegIdx: AArch64::qsub0);
5454 return;
5455 }
5456 break;
5457 case Intrinsic::aarch64_neon_ld4:
5458 if (VT == MVT::v8i8) {
5459 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b, SubRegIdx: AArch64::dsub0);
5460 return;
5461 } else if (VT == MVT::v16i8) {
5462 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b, SubRegIdx: AArch64::qsub0);
5463 return;
5464 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5465 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h, SubRegIdx: AArch64::dsub0);
5466 return;
5467 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5468 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h, SubRegIdx: AArch64::qsub0);
5469 return;
5470 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5471 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s, SubRegIdx: AArch64::dsub0);
5472 return;
5473 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5474 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s, SubRegIdx: AArch64::qsub0);
5475 return;
5476 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5477 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d, SubRegIdx: AArch64::dsub0);
5478 return;
5479 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5480 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d, SubRegIdx: AArch64::qsub0);
5481 return;
5482 }
5483 break;
5484 case Intrinsic::aarch64_neon_ld2r:
5485 if (VT == MVT::v8i8) {
5486 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b, SubRegIdx: AArch64::dsub0);
5487 return;
5488 } else if (VT == MVT::v16i8) {
5489 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b, SubRegIdx: AArch64::qsub0);
5490 return;
5491 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5492 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h, SubRegIdx: AArch64::dsub0);
5493 return;
5494 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5495 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h, SubRegIdx: AArch64::qsub0);
5496 return;
5497 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5498 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s, SubRegIdx: AArch64::dsub0);
5499 return;
5500 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5501 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s, SubRegIdx: AArch64::qsub0);
5502 return;
5503 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5504 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d, SubRegIdx: AArch64::dsub0);
5505 return;
5506 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5507 SelectLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d, SubRegIdx: AArch64::qsub0);
5508 return;
5509 }
5510 break;
5511 case Intrinsic::aarch64_neon_ld3r:
5512 if (VT == MVT::v8i8) {
5513 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b, SubRegIdx: AArch64::dsub0);
5514 return;
5515 } else if (VT == MVT::v16i8) {
5516 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b, SubRegIdx: AArch64::qsub0);
5517 return;
5518 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5519 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h, SubRegIdx: AArch64::dsub0);
5520 return;
5521 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5522 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h, SubRegIdx: AArch64::qsub0);
5523 return;
5524 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5525 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s, SubRegIdx: AArch64::dsub0);
5526 return;
5527 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5528 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s, SubRegIdx: AArch64::qsub0);
5529 return;
5530 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5531 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d, SubRegIdx: AArch64::dsub0);
5532 return;
5533 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5534 SelectLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d, SubRegIdx: AArch64::qsub0);
5535 return;
5536 }
5537 break;
5538 case Intrinsic::aarch64_neon_ld4r:
5539 if (VT == MVT::v8i8) {
5540 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b, SubRegIdx: AArch64::dsub0);
5541 return;
5542 } else if (VT == MVT::v16i8) {
5543 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b, SubRegIdx: AArch64::qsub0);
5544 return;
5545 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5546 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h, SubRegIdx: AArch64::dsub0);
5547 return;
5548 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5549 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h, SubRegIdx: AArch64::qsub0);
5550 return;
5551 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5552 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s, SubRegIdx: AArch64::dsub0);
5553 return;
5554 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5555 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s, SubRegIdx: AArch64::qsub0);
5556 return;
5557 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5558 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d, SubRegIdx: AArch64::dsub0);
5559 return;
5560 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5561 SelectLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d, SubRegIdx: AArch64::qsub0);
5562 return;
5563 }
5564 break;
5565 case Intrinsic::aarch64_neon_ld2lane:
5566 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5567 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8);
5568 return;
5569 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5570 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5571 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16);
5572 return;
5573 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5574 VT == MVT::v2f32) {
5575 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32);
5576 return;
5577 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5578 VT == MVT::v1f64) {
5579 SelectLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64);
5580 return;
5581 }
5582 break;
5583 case Intrinsic::aarch64_neon_ld3lane:
5584 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5585 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8);
5586 return;
5587 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5588 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5589 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16);
5590 return;
5591 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5592 VT == MVT::v2f32) {
5593 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32);
5594 return;
5595 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5596 VT == MVT::v1f64) {
5597 SelectLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64);
5598 return;
5599 }
5600 break;
5601 case Intrinsic::aarch64_neon_ld4lane:
5602 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5603 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8);
5604 return;
5605 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5606 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5607 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16);
5608 return;
5609 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5610 VT == MVT::v2f32) {
5611 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32);
5612 return;
5613 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5614 VT == MVT::v1f64) {
5615 SelectLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64);
5616 return;
5617 }
5618 break;
5619 case Intrinsic::aarch64_ld64b:
5620 SelectLoad(N: Node, NumVecs: 8, Opc: AArch64::LD64B, SubRegIdx: AArch64::x8sub_0);
5621 return;
5622 case Intrinsic::aarch64_sve_ld2q_sret: {
5623 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 4, Opc_ri: AArch64::LD2Q_IMM, Opc_rr: AArch64::LD2Q, IsIntr: true);
5624 return;
5625 }
5626 case Intrinsic::aarch64_sve_ld3q_sret: {
5627 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 4, Opc_ri: AArch64::LD3Q_IMM, Opc_rr: AArch64::LD3Q, IsIntr: true);
5628 return;
5629 }
5630 case Intrinsic::aarch64_sve_ld4q_sret: {
5631 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 4, Opc_ri: AArch64::LD4Q_IMM, Opc_rr: AArch64::LD4Q, IsIntr: true);
5632 return;
5633 }
5634 case Intrinsic::aarch64_sve_ld2_sret: {
5635 if (VT == MVT::nxv16i8) {
5636 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD2B_IMM, Opc_rr: AArch64::LD2B,
5637 IsIntr: true);
5638 return;
5639 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5640 VT == MVT::nxv8bf16) {
5641 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD2H_IMM, Opc_rr: AArch64::LD2H,
5642 IsIntr: true);
5643 return;
5644 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5645 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD2W_IMM, Opc_rr: AArch64::LD2W,
5646 IsIntr: true);
5647 return;
5648 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5649 SelectPredicatedLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD2D_IMM, Opc_rr: AArch64::LD2D,
5650 IsIntr: true);
5651 return;
5652 }
5653 break;
5654 }
5655 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5656 if (VT == MVT::nxv16i8) {
5657 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5658 SelectContiguousMultiVectorLoad(
5659 N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_2Z_PSEUDO);
5660 else if (Subtarget->hasSVE2p1())
5661 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LD1B_2Z_IMM,
5662 Opc_rr: AArch64::LD1B_2Z);
5663 else
5664 break;
5665 return;
5666 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5667 VT == MVT::nxv8bf16) {
5668 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5669 SelectContiguousMultiVectorLoad(
5670 N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_2Z_PSEUDO);
5671 else if (Subtarget->hasSVE2p1())
5672 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LD1H_2Z_IMM,
5673 Opc_rr: AArch64::LD1H_2Z);
5674 else
5675 break;
5676 return;
5677 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5678 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5679 SelectContiguousMultiVectorLoad(
5680 N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_2Z_PSEUDO);
5681 else if (Subtarget->hasSVE2p1())
5682 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LD1W_2Z_IMM,
5683 Opc_rr: AArch64::LD1W_2Z);
5684 else
5685 break;
5686 return;
5687 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5688 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5689 SelectContiguousMultiVectorLoad(
5690 N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_2Z_PSEUDO);
5691 else if (Subtarget->hasSVE2p1())
5692 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LD1D_2Z_IMM,
5693 Opc_rr: AArch64::LD1D_2Z);
5694 else
5695 break;
5696 return;
5697 }
5698 break;
5699 }
5700 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5701 if (VT == MVT::nxv16i8) {
5702 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5703 SelectContiguousMultiVectorLoad(
5704 N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1B_4Z_PSEUDO);
5705 else if (Subtarget->hasSVE2p1())
5706 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD1B_4Z_IMM,
5707 Opc_rr: AArch64::LD1B_4Z);
5708 else
5709 break;
5710 return;
5711 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5712 VT == MVT::nxv8bf16) {
5713 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5714 SelectContiguousMultiVectorLoad(
5715 N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1H_4Z_PSEUDO);
5716 else if (Subtarget->hasSVE2p1())
5717 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD1H_4Z_IMM,
5718 Opc_rr: AArch64::LD1H_4Z);
5719 else
5720 break;
5721 return;
5722 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5723 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5724 SelectContiguousMultiVectorLoad(
5725 N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1W_4Z_PSEUDO);
5726 else if (Subtarget->hasSVE2p1())
5727 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD1W_4Z_IMM,
5728 Opc_rr: AArch64::LD1W_4Z);
5729 else
5730 break;
5731 return;
5732 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5733 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5734 SelectContiguousMultiVectorLoad(
5735 N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM_PSEUDO, Opc_rr: AArch64::LD1D_4Z_PSEUDO);
5736 else if (Subtarget->hasSVE2p1())
5737 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD1D_4Z_IMM,
5738 Opc_rr: AArch64::LD1D_4Z);
5739 else
5740 break;
5741 return;
5742 }
5743 break;
5744 }
5745 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5746 if (VT == MVT::nxv16i8) {
5747 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5748 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0,
5749 Opc_ri: AArch64::LDNT1B_2Z_IMM_PSEUDO,
5750 Opc_rr: AArch64::LDNT1B_2Z_PSEUDO);
5751 else if (Subtarget->hasSVE2p1())
5752 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 0, Opc_ri: AArch64::LDNT1B_2Z_IMM,
5753 Opc_rr: AArch64::LDNT1B_2Z);
5754 else
5755 break;
5756 return;
5757 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5758 VT == MVT::nxv8bf16) {
5759 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5760 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1,
5761 Opc_ri: AArch64::LDNT1H_2Z_IMM_PSEUDO,
5762 Opc_rr: AArch64::LDNT1H_2Z_PSEUDO);
5763 else if (Subtarget->hasSVE2p1())
5764 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 1, Opc_ri: AArch64::LDNT1H_2Z_IMM,
5765 Opc_rr: AArch64::LDNT1H_2Z);
5766 else
5767 break;
5768 return;
5769 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5770 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5771 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2,
5772 Opc_ri: AArch64::LDNT1W_2Z_IMM_PSEUDO,
5773 Opc_rr: AArch64::LDNT1W_2Z_PSEUDO);
5774 else if (Subtarget->hasSVE2p1())
5775 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 2, Opc_ri: AArch64::LDNT1W_2Z_IMM,
5776 Opc_rr: AArch64::LDNT1W_2Z);
5777 else
5778 break;
5779 return;
5780 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5781 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5782 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3,
5783 Opc_ri: AArch64::LDNT1D_2Z_IMM_PSEUDO,
5784 Opc_rr: AArch64::LDNT1D_2Z_PSEUDO);
5785 else if (Subtarget->hasSVE2p1())
5786 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 2, Scale: 3, Opc_ri: AArch64::LDNT1D_2Z_IMM,
5787 Opc_rr: AArch64::LDNT1D_2Z);
5788 else
5789 break;
5790 return;
5791 }
5792 break;
5793 }
5794 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5795 if (VT == MVT::nxv16i8) {
5796 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5797 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0,
5798 Opc_ri: AArch64::LDNT1B_4Z_IMM_PSEUDO,
5799 Opc_rr: AArch64::LDNT1B_4Z_PSEUDO);
5800 else if (Subtarget->hasSVE2p1())
5801 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LDNT1B_4Z_IMM,
5802 Opc_rr: AArch64::LDNT1B_4Z);
5803 else
5804 break;
5805 return;
5806 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5807 VT == MVT::nxv8bf16) {
5808 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5809 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1,
5810 Opc_ri: AArch64::LDNT1H_4Z_IMM_PSEUDO,
5811 Opc_rr: AArch64::LDNT1H_4Z_PSEUDO);
5812 else if (Subtarget->hasSVE2p1())
5813 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LDNT1H_4Z_IMM,
5814 Opc_rr: AArch64::LDNT1H_4Z);
5815 else
5816 break;
5817 return;
5818 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5819 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5820 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2,
5821 Opc_ri: AArch64::LDNT1W_4Z_IMM_PSEUDO,
5822 Opc_rr: AArch64::LDNT1W_4Z_PSEUDO);
5823 else if (Subtarget->hasSVE2p1())
5824 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LDNT1W_4Z_IMM,
5825 Opc_rr: AArch64::LDNT1W_4Z);
5826 else
5827 break;
5828 return;
5829 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5830 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5831 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3,
5832 Opc_ri: AArch64::LDNT1D_4Z_IMM_PSEUDO,
5833 Opc_rr: AArch64::LDNT1D_4Z_PSEUDO);
5834 else if (Subtarget->hasSVE2p1())
5835 SelectContiguousMultiVectorLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LDNT1D_4Z_IMM,
5836 Opc_rr: AArch64::LDNT1D_4Z);
5837 else
5838 break;
5839 return;
5840 }
5841 break;
5842 }
5843 case Intrinsic::aarch64_sve_ld3_sret: {
5844 if (VT == MVT::nxv16i8) {
5845 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 0, Opc_ri: AArch64::LD3B_IMM, Opc_rr: AArch64::LD3B,
5846 IsIntr: true);
5847 return;
5848 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5849 VT == MVT::nxv8bf16) {
5850 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 1, Opc_ri: AArch64::LD3H_IMM, Opc_rr: AArch64::LD3H,
5851 IsIntr: true);
5852 return;
5853 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5854 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 2, Opc_ri: AArch64::LD3W_IMM, Opc_rr: AArch64::LD3W,
5855 IsIntr: true);
5856 return;
5857 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5858 SelectPredicatedLoad(N: Node, NumVecs: 3, Scale: 3, Opc_ri: AArch64::LD3D_IMM, Opc_rr: AArch64::LD3D,
5859 IsIntr: true);
5860 return;
5861 }
5862 break;
5863 }
5864 case Intrinsic::aarch64_sve_ld4_sret: {
5865 if (VT == MVT::nxv16i8) {
5866 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 0, Opc_ri: AArch64::LD4B_IMM, Opc_rr: AArch64::LD4B,
5867 IsIntr: true);
5868 return;
5869 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5870 VT == MVT::nxv8bf16) {
5871 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 1, Opc_ri: AArch64::LD4H_IMM, Opc_rr: AArch64::LD4H,
5872 IsIntr: true);
5873 return;
5874 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5875 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 2, Opc_ri: AArch64::LD4W_IMM, Opc_rr: AArch64::LD4W,
5876 IsIntr: true);
5877 return;
5878 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5879 SelectPredicatedLoad(N: Node, NumVecs: 4, Scale: 3, Opc_ri: AArch64::LD4D_IMM, Opc_rr: AArch64::LD4D,
5880 IsIntr: true);
5881 return;
5882 }
5883 break;
5884 }
5885 case Intrinsic::aarch64_sme_read_hor_vg2: {
5886 if (VT == MVT::nxv16i8) {
5887 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5888 Op: AArch64::MOVA_2ZMXI_H_B);
5889 return;
5890 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5891 VT == MVT::nxv8bf16) {
5892 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5893 Op: AArch64::MOVA_2ZMXI_H_H);
5894 return;
5895 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5896 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5897 Op: AArch64::MOVA_2ZMXI_H_S);
5898 return;
5899 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5900 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5901 Op: AArch64::MOVA_2ZMXI_H_D);
5902 return;
5903 }
5904 break;
5905 }
5906 case Intrinsic::aarch64_sme_read_ver_vg2: {
5907 if (VT == MVT::nxv16i8) {
5908 SelectMultiVectorMove<14, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAB0,
5909 Op: AArch64::MOVA_2ZMXI_V_B);
5910 return;
5911 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5912 VT == MVT::nxv8bf16) {
5913 SelectMultiVectorMove<6, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAH0,
5914 Op: AArch64::MOVA_2ZMXI_V_H);
5915 return;
5916 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5917 SelectMultiVectorMove<2, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAS0,
5918 Op: AArch64::MOVA_2ZMXI_V_S);
5919 return;
5920 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5921 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 2, BaseReg: AArch64::ZAD0,
5922 Op: AArch64::MOVA_2ZMXI_V_D);
5923 return;
5924 }
5925 break;
5926 }
5927 case Intrinsic::aarch64_sme_read_hor_vg4: {
5928 if (VT == MVT::nxv16i8) {
5929 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5930 Op: AArch64::MOVA_4ZMXI_H_B);
5931 return;
5932 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5933 VT == MVT::nxv8bf16) {
5934 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5935 Op: AArch64::MOVA_4ZMXI_H_H);
5936 return;
5937 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5938 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5939 Op: AArch64::MOVA_4ZMXI_H_S);
5940 return;
5941 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5942 SelectMultiVectorMove<0, 2>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5943 Op: AArch64::MOVA_4ZMXI_H_D);
5944 return;
5945 }
5946 break;
5947 }
5948 case Intrinsic::aarch64_sme_read_ver_vg4: {
5949 if (VT == MVT::nxv16i8) {
5950 SelectMultiVectorMove<12, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAB0,
5951 Op: AArch64::MOVA_4ZMXI_V_B);
5952 return;
5953 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5954 VT == MVT::nxv8bf16) {
5955 SelectMultiVectorMove<4, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAH0,
5956 Op: AArch64::MOVA_4ZMXI_V_H);
5957 return;
5958 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5959 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAS0,
5960 Op: AArch64::MOVA_4ZMXI_V_S);
5961 return;
5962 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5963 SelectMultiVectorMove<0, 4>(N: Node, NumVecs: 4, BaseReg: AArch64::ZAD0,
5964 Op: AArch64::MOVA_4ZMXI_V_D);
5965 return;
5966 }
5967 break;
5968 }
5969 case Intrinsic::aarch64_sme_read_vg1x2: {
5970 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 2, BaseReg: AArch64::ZA,
5971 Op: AArch64::MOVA_VG2_2ZMXI);
5972 return;
5973 }
5974 case Intrinsic::aarch64_sme_read_vg1x4: {
5975 SelectMultiVectorMove<7, 1>(N: Node, NumVecs: 4, BaseReg: AArch64::ZA,
5976 Op: AArch64::MOVA_VG4_4ZMXI);
5977 return;
5978 }
5979 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5980 if (VT == MVT::nxv16i8) {
5981 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_B_PSEUDO, MaxIdx: 14, Scale: 2);
5982 return;
5983 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5984 VT == MVT::nxv8bf16) {
5985 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_H_PSEUDO, MaxIdx: 6, Scale: 2);
5986 return;
5987 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5988 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_S_PSEUDO, MaxIdx: 2, Scale: 2);
5989 return;
5990 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5991 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 2);
5992 return;
5993 }
5994 break;
5995 }
5996 case Intrinsic::aarch64_sme_readz_vert_x2: {
5997 if (VT == MVT::nxv16i8) {
5998 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_B_PSEUDO, MaxIdx: 14, Scale: 2);
5999 return;
6000 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6001 VT == MVT::nxv8bf16) {
6002 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_H_PSEUDO, MaxIdx: 6, Scale: 2);
6003 return;
6004 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6005 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_S_PSEUDO, MaxIdx: 2, Scale: 2);
6006 return;
6007 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6008 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_2ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 2);
6009 return;
6010 }
6011 break;
6012 }
6013 case Intrinsic::aarch64_sme_readz_horiz_x4: {
6014 if (VT == MVT::nxv16i8) {
6015 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_B_PSEUDO, MaxIdx: 12, Scale: 4);
6016 return;
6017 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6018 VT == MVT::nxv8bf16) {
6019 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_H_PSEUDO, MaxIdx: 4, Scale: 4);
6020 return;
6021 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6022 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_S_PSEUDO, MaxIdx: 0, Scale: 4);
6023 return;
6024 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6025 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_H_D_PSEUDO, MaxIdx: 0, Scale: 4);
6026 return;
6027 }
6028 break;
6029 }
6030 case Intrinsic::aarch64_sme_readz_vert_x4: {
6031 if (VT == MVT::nxv16i8) {
6032 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_B_PSEUDO, MaxIdx: 12, Scale: 4);
6033 return;
6034 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6035 VT == MVT::nxv8bf16) {
6036 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_H_PSEUDO, MaxIdx: 4, Scale: 4);
6037 return;
6038 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6039 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_S_PSEUDO, MaxIdx: 0, Scale: 4);
6040 return;
6041 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6042 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_4ZMI_V_D_PSEUDO, MaxIdx: 0, Scale: 4);
6043 return;
6044 }
6045 break;
6046 }
6047 case Intrinsic::aarch64_sme_readz_x2: {
6048 SelectMultiVectorMoveZ(N: Node, NumVecs: 2, Op: AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6049 BaseReg: AArch64::ZA);
6050 return;
6051 }
6052 case Intrinsic::aarch64_sme_readz_x4: {
6053 SelectMultiVectorMoveZ(N: Node, NumVecs: 4, Op: AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, MaxIdx: 7, Scale: 1,
6054 BaseReg: AArch64::ZA);
6055 return;
6056 }
6057 case Intrinsic::swift_async_context_addr: {
6058 SDLoc DL(Node);
6059 SDValue Chain = Node->getOperand(Num: 0);
6060 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, dl: DL, Reg: AArch64::FP, VT: MVT::i64);
6061 SDValue Res = SDValue(
6062 CurDAG->getMachineNode(Opcode: AArch64::SUBXri, dl: DL, VT: MVT::i64, Op1: CopyFP,
6063 Op2: CurDAG->getTargetConstant(Val: 8, DL, VT: MVT::i32),
6064 Op3: CurDAG->getTargetConstant(Val: 0, DL, VT: MVT::i32)),
6065 0);
6066 ReplaceUses(F: SDValue(Node, 0), T: Res);
6067 ReplaceUses(F: SDValue(Node, 1), T: CopyFP.getValue(R: 1));
6068 CurDAG->RemoveDeadNode(N: Node);
6069
6070 auto &MF = CurDAG->getMachineFunction();
6071 MF.getFrameInfo().setFrameAddressIsTaken(true);
6072 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6073 return;
6074 }
6075 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6076 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6077 VT: Node->getValueType(ResNo: 0),
6078 Opcodes: {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6079 AArch64::LUTI2_4ZTZI_S}))
6080 // Second Immediate must be <= 3:
6081 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 3);
6082 return;
6083 }
6084 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6085 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6086 VT: Node->getValueType(ResNo: 0),
6087 Opcodes: {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6088 // Second Immediate must be <= 1:
6089 SelectMultiVectorLutiLane(Node, NumOutVecs: 4, Opc, MaxImm: 1);
6090 return;
6091 }
6092 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6093 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6094 VT: Node->getValueType(ResNo: 0),
6095 Opcodes: {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6096 AArch64::LUTI2_2ZTZI_S}))
6097 // Second Immediate must be <= 7:
6098 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 7);
6099 return;
6100 }
6101 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6102 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6103 VT: Node->getValueType(ResNo: 0),
6104 Opcodes: {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6105 AArch64::LUTI4_2ZTZI_S}))
6106 // Second Immediate must be <= 3:
6107 SelectMultiVectorLutiLane(Node, NumOutVecs: 2, Opc, MaxImm: 3);
6108 return;
6109 }
6110 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6111 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI4_4ZZT2Z, NumInVecs: 2);
6112 return;
6113 }
6114 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6115 SelectMultiVectorLuti(Node, NumOutVecs: 4, Opc: AArch64::LUTI6_4ZT3Z, NumInVecs: 3);
6116 return;
6117 }
6118 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6119 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6120 VT: Node->getValueType(ResNo: 0),
6121 Opcodes: {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6122 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6123 return;
6124 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6125 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6126 VT: Node->getValueType(ResNo: 0),
6127 Opcodes: {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6128 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6129 return;
6130 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6131 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6132 VT: Node->getValueType(ResNo: 0),
6133 Opcodes: {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6134 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6135 return;
6136 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6137 if (auto Opc = SelectOpcodeFromVT<SelectTypeKind::FP>(
6138 VT: Node->getValueType(ResNo: 0),
6139 Opcodes: {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6140 SelectCVTIntrinsicFP8(N: Node, NumVecs: 2, Opcode: Opc);
6141 return;
6142 case Intrinsic::ptrauth_resign_load_relative:
6143 SelectPtrauthResign(N: Node);
6144 return;
6145 }
6146 } break;
6147 case ISD::INTRINSIC_WO_CHAIN: {
6148 unsigned IntNo = Node->getConstantOperandVal(Num: 0);
6149 switch (IntNo) {
6150 default:
6151 break;
6152 case Intrinsic::aarch64_tagp:
6153 SelectTagP(N: Node);
6154 return;
6155
6156 case Intrinsic::ptrauth_auth:
6157 SelectPtrauthAuth(N: Node);
6158 return;
6159
6160 case Intrinsic::ptrauth_resign:
6161 SelectPtrauthResign(N: Node);
6162 return;
6163
6164 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6165 SelectPtrauthResignWithPC(N: Node);
6166 return;
6167
6168 case Intrinsic::aarch64_neon_tbl2:
6169 SelectTable(N: Node, NumVecs: 2,
6170 Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6171 isExt: false);
6172 return;
6173 case Intrinsic::aarch64_neon_tbl3:
6174 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6175 : AArch64::TBLv16i8Three,
6176 isExt: false);
6177 return;
6178 case Intrinsic::aarch64_neon_tbl4:
6179 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6180 : AArch64::TBLv16i8Four,
6181 isExt: false);
6182 return;
6183 case Intrinsic::aarch64_neon_tbx2:
6184 SelectTable(N: Node, NumVecs: 2,
6185 Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6186 isExt: true);
6187 return;
6188 case Intrinsic::aarch64_neon_tbx3:
6189 SelectTable(N: Node, NumVecs: 3, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6190 : AArch64::TBXv16i8Three,
6191 isExt: true);
6192 return;
6193 case Intrinsic::aarch64_neon_tbx4:
6194 SelectTable(N: Node, NumVecs: 4, Opc: VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6195 : AArch64::TBXv16i8Four,
6196 isExt: true);
6197 return;
6198 case Intrinsic::aarch64_sve_srshl_single_x2:
6199 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6200 VT: Node->getValueType(ResNo: 0),
6201 Opcodes: {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6202 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6203 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6204 return;
6205 case Intrinsic::aarch64_sve_srshl_single_x4:
6206 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6207 VT: Node->getValueType(ResNo: 0),
6208 Opcodes: {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6209 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6210 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6211 return;
6212 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6213 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 2);
6214 return;
6215 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6216 SelectMultiVectorLuti6LaneX4(Node, NumIndexVecs: 3);
6217 return;
6218 case Intrinsic::aarch64_sve_urshl_single_x2:
6219 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6220 VT: Node->getValueType(ResNo: 0),
6221 Opcodes: {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6222 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6223 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6224 return;
6225 case Intrinsic::aarch64_sve_urshl_single_x4:
6226 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6227 VT: Node->getValueType(ResNo: 0),
6228 Opcodes: {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6229 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6230 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6231 return;
6232 case Intrinsic::aarch64_sve_srshl_x2:
6233 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6234 VT: Node->getValueType(ResNo: 0),
6235 Opcodes: {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6236 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6237 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6238 return;
6239 case Intrinsic::aarch64_sve_srshl_x4:
6240 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6241 VT: Node->getValueType(ResNo: 0),
6242 Opcodes: {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6243 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6244 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6245 return;
6246 case Intrinsic::aarch64_sve_urshl_x2:
6247 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6248 VT: Node->getValueType(ResNo: 0),
6249 Opcodes: {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6250 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6251 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6252 return;
6253 case Intrinsic::aarch64_sve_urshl_x4:
6254 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6255 VT: Node->getValueType(ResNo: 0),
6256 Opcodes: {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6257 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6258 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6259 return;
6260 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6261 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6262 VT: Node->getValueType(ResNo: 0),
6263 Opcodes: {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6264 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6265 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6266 return;
6267 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6268 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6269 VT: Node->getValueType(ResNo: 0),
6270 Opcodes: {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6271 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6272 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6273 return;
6274 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6275 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6276 VT: Node->getValueType(ResNo: 0),
6277 Opcodes: {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6278 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6279 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6280 return;
6281 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6282 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6283 VT: Node->getValueType(ResNo: 0),
6284 Opcodes: {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6285 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6286 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6287 return;
6288 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6289 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6290 VT: Node->getValueType(ResNo: 0),
6291 Opcodes: {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6292 AArch64::FSCALE_2ZZ_D}))
6293 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6294 return;
6295 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6296 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6297 VT: Node->getValueType(ResNo: 0),
6298 Opcodes: {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6299 AArch64::FSCALE_4ZZ_D}))
6300 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6301 return;
6302 case Intrinsic::aarch64_sme_fp8_scale_x2:
6303 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6304 VT: Node->getValueType(ResNo: 0),
6305 Opcodes: {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6306 AArch64::FSCALE_2Z2Z_D}))
6307 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6308 return;
6309 case Intrinsic::aarch64_sme_fp8_scale_x4:
6310 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6311 VT: Node->getValueType(ResNo: 0),
6312 Opcodes: {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6313 AArch64::FSCALE_4Z4Z_D}))
6314 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6315 return;
6316 case Intrinsic::aarch64_sve_whilege_x2:
6317 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6318 VT: Node->getValueType(ResNo: 0),
6319 Opcodes: {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6320 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6321 SelectWhilePair(N: Node, Opc: Op);
6322 return;
6323 case Intrinsic::aarch64_sve_whilegt_x2:
6324 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6325 VT: Node->getValueType(ResNo: 0),
6326 Opcodes: {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6327 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6328 SelectWhilePair(N: Node, Opc: Op);
6329 return;
6330 case Intrinsic::aarch64_sve_whilehi_x2:
6331 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6332 VT: Node->getValueType(ResNo: 0),
6333 Opcodes: {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6334 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6335 SelectWhilePair(N: Node, Opc: Op);
6336 return;
6337 case Intrinsic::aarch64_sve_whilehs_x2:
6338 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6339 VT: Node->getValueType(ResNo: 0),
6340 Opcodes: {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6341 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6342 SelectWhilePair(N: Node, Opc: Op);
6343 return;
6344 case Intrinsic::aarch64_sve_whilele_x2:
6345 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6346 VT: Node->getValueType(ResNo: 0),
6347 Opcodes: {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6348 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6349 SelectWhilePair(N: Node, Opc: Op);
6350 return;
6351 case Intrinsic::aarch64_sve_whilelo_x2:
6352 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6353 VT: Node->getValueType(ResNo: 0),
6354 Opcodes: {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6355 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6356 SelectWhilePair(N: Node, Opc: Op);
6357 return;
6358 case Intrinsic::aarch64_sve_whilels_x2:
6359 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6360 VT: Node->getValueType(ResNo: 0),
6361 Opcodes: {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6362 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6363 SelectWhilePair(N: Node, Opc: Op);
6364 return;
6365 case Intrinsic::aarch64_sve_whilelt_x2:
6366 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int1>(
6367 VT: Node->getValueType(ResNo: 0),
6368 Opcodes: {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6369 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6370 SelectWhilePair(N: Node, Opc: Op);
6371 return;
6372 case Intrinsic::aarch64_sve_smax_single_x2:
6373 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6374 VT: Node->getValueType(ResNo: 0),
6375 Opcodes: {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6376 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6377 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6378 return;
6379 case Intrinsic::aarch64_sve_umax_single_x2:
6380 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6381 VT: Node->getValueType(ResNo: 0),
6382 Opcodes: {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6383 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6384 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6385 return;
6386 case Intrinsic::aarch64_sve_fmax_single_x2:
6387 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6388 VT: Node->getValueType(ResNo: 0),
6389 Opcodes: {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6390 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6391 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6392 return;
6393 case Intrinsic::aarch64_sve_smax_single_x4:
6394 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6395 VT: Node->getValueType(ResNo: 0),
6396 Opcodes: {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6397 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6398 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6399 return;
6400 case Intrinsic::aarch64_sve_umax_single_x4:
6401 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6402 VT: Node->getValueType(ResNo: 0),
6403 Opcodes: {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6404 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6405 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6406 return;
6407 case Intrinsic::aarch64_sve_fmax_single_x4:
6408 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6409 VT: Node->getValueType(ResNo: 0),
6410 Opcodes: {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6411 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6412 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6413 return;
6414 case Intrinsic::aarch64_sve_smin_single_x2:
6415 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6416 VT: Node->getValueType(ResNo: 0),
6417 Opcodes: {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6418 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6419 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6420 return;
6421 case Intrinsic::aarch64_sve_umin_single_x2:
6422 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6423 VT: Node->getValueType(ResNo: 0),
6424 Opcodes: {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6425 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6426 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6427 return;
6428 case Intrinsic::aarch64_sve_fmin_single_x2:
6429 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6430 VT: Node->getValueType(ResNo: 0),
6431 Opcodes: {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6432 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6433 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6434 return;
6435 case Intrinsic::aarch64_sve_smin_single_x4:
6436 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6437 VT: Node->getValueType(ResNo: 0),
6438 Opcodes: {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6439 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6440 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6441 return;
6442 case Intrinsic::aarch64_sve_umin_single_x4:
6443 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6444 VT: Node->getValueType(ResNo: 0),
6445 Opcodes: {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6446 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6447 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6448 return;
6449 case Intrinsic::aarch64_sve_fmin_single_x4:
6450 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6451 VT: Node->getValueType(ResNo: 0),
6452 Opcodes: {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6453 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6454 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6455 return;
6456 case Intrinsic::aarch64_sve_smax_x2:
6457 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6458 VT: Node->getValueType(ResNo: 0),
6459 Opcodes: {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6460 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6461 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6462 return;
6463 case Intrinsic::aarch64_sve_umax_x2:
6464 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6465 VT: Node->getValueType(ResNo: 0),
6466 Opcodes: {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6467 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6468 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6469 return;
6470 case Intrinsic::aarch64_sve_fmax_x2:
6471 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6472 VT: Node->getValueType(ResNo: 0),
6473 Opcodes: {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6474 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6475 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6476 return;
6477 case Intrinsic::aarch64_sve_smax_x4:
6478 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6479 VT: Node->getValueType(ResNo: 0),
6480 Opcodes: {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6481 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6482 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6483 return;
6484 case Intrinsic::aarch64_sve_umax_x4:
6485 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6486 VT: Node->getValueType(ResNo: 0),
6487 Opcodes: {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6488 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6489 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6490 return;
6491 case Intrinsic::aarch64_sve_fmax_x4:
6492 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6493 VT: Node->getValueType(ResNo: 0),
6494 Opcodes: {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6495 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6496 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6497 return;
6498 case Intrinsic::aarch64_sme_famax_x2:
6499 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6500 VT: Node->getValueType(ResNo: 0),
6501 Opcodes: {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6502 AArch64::FAMAX_2Z2Z_D}))
6503 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6504 return;
6505 case Intrinsic::aarch64_sme_famax_x4:
6506 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6507 VT: Node->getValueType(ResNo: 0),
6508 Opcodes: {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6509 AArch64::FAMAX_4Z4Z_D}))
6510 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6511 return;
6512 case Intrinsic::aarch64_sme_famin_x2:
6513 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6514 VT: Node->getValueType(ResNo: 0),
6515 Opcodes: {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6516 AArch64::FAMIN_2Z2Z_D}))
6517 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6518 return;
6519 case Intrinsic::aarch64_sme_famin_x4:
6520 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6521 VT: Node->getValueType(ResNo: 0),
6522 Opcodes: {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6523 AArch64::FAMIN_4Z4Z_D}))
6524 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6525 return;
6526 case Intrinsic::aarch64_sve_smin_x2:
6527 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6528 VT: Node->getValueType(ResNo: 0),
6529 Opcodes: {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6530 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6531 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6532 return;
6533 case Intrinsic::aarch64_sve_umin_x2:
6534 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6535 VT: Node->getValueType(ResNo: 0),
6536 Opcodes: {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6537 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6538 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6539 return;
6540 case Intrinsic::aarch64_sve_fmin_x2:
6541 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6542 VT: Node->getValueType(ResNo: 0),
6543 Opcodes: {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6544 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6545 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6546 return;
6547 case Intrinsic::aarch64_sve_smin_x4:
6548 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6549 VT: Node->getValueType(ResNo: 0),
6550 Opcodes: {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6551 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6552 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6553 return;
6554 case Intrinsic::aarch64_sve_umin_x4:
6555 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6556 VT: Node->getValueType(ResNo: 0),
6557 Opcodes: {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6558 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6559 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6560 return;
6561 case Intrinsic::aarch64_sve_fmin_x4:
6562 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6563 VT: Node->getValueType(ResNo: 0),
6564 Opcodes: {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6565 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6566 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6567 return;
6568 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6569 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6570 VT: Node->getValueType(ResNo: 0),
6571 Opcodes: {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6572 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6573 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6574 return;
6575 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6576 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6577 VT: Node->getValueType(ResNo: 0),
6578 Opcodes: {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6579 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6580 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6581 return;
6582 case Intrinsic::aarch64_sve_fminnm_single_x2:
6583 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6584 VT: Node->getValueType(ResNo: 0),
6585 Opcodes: {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6586 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6587 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6588 return;
6589 case Intrinsic::aarch64_sve_fminnm_single_x4:
6590 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6591 VT: Node->getValueType(ResNo: 0),
6592 Opcodes: {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6593 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6594 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6595 return;
6596 case Intrinsic::aarch64_sve_fscale_single_x4:
6597 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::BFSCALE_4ZZ);
6598 return;
6599 case Intrinsic::aarch64_sve_fscale_single_x2:
6600 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::BFSCALE_2ZZ);
6601 return;
6602 case Intrinsic::aarch64_sve_fmul_single_x4:
6603 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6604 VT: Node->getValueType(ResNo: 0),
6605 Opcodes: {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6606 AArch64::FMUL_4ZZ_D}))
6607 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6608 return;
6609 case Intrinsic::aarch64_sve_fmul_single_x2:
6610 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6611 VT: Node->getValueType(ResNo: 0),
6612 Opcodes: {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6613 AArch64::FMUL_2ZZ_D}))
6614 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6615 return;
6616 case Intrinsic::aarch64_sve_fmaxnm_x2:
6617 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6618 VT: Node->getValueType(ResNo: 0),
6619 Opcodes: {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6620 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6621 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6622 return;
6623 case Intrinsic::aarch64_sve_fmaxnm_x4:
6624 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6625 VT: Node->getValueType(ResNo: 0),
6626 Opcodes: {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6627 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6628 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6629 return;
6630 case Intrinsic::aarch64_sve_fminnm_x2:
6631 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6632 VT: Node->getValueType(ResNo: 0),
6633 Opcodes: {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6634 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6635 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6636 return;
6637 case Intrinsic::aarch64_sve_fminnm_x4:
6638 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6639 VT: Node->getValueType(ResNo: 0),
6640 Opcodes: {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6641 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6642 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6643 return;
6644 case Intrinsic::aarch64_sve_aese_lane_x2:
6645 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESE_2ZZI_B);
6646 return;
6647 case Intrinsic::aarch64_sve_aesd_lane_x2:
6648 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESD_2ZZI_B);
6649 return;
6650 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6651 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESEMC_2ZZI_B);
6652 return;
6653 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6654 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::AESDIMC_2ZZI_B);
6655 return;
6656 case Intrinsic::aarch64_sve_aese_lane_x4:
6657 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESE_4ZZI_B);
6658 return;
6659 case Intrinsic::aarch64_sve_aesd_lane_x4:
6660 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESD_4ZZI_B);
6661 return;
6662 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6663 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESEMC_4ZZI_B);
6664 return;
6665 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6666 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: AArch64::AESDIMC_4ZZI_B);
6667 return;
6668 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6669 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: AArch64::PMLAL_2ZZZ_Q);
6670 return;
6671 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6672 SDLoc DL(Node);
6673 SmallVector<SDValue, 4> Regs(Node->ops().slice(N: 1, M: 2));
6674 SDNode *Res =
6675 CurDAG->getMachineNode(Opcode: AArch64::PMULL_2ZZZ_Q, dl: DL, VT: MVT::Untyped, Ops: Regs);
6676 SDValue SuperReg = SDValue(Res, 0);
6677 for (unsigned I = 0; I < 2; I++)
6678 ReplaceUses(F: SDValue(Node, I),
6679 T: CurDAG->getTargetExtractSubreg(SRIdx: AArch64::zsub0 + I, DL, VT,
6680 Operand: SuperReg));
6681 CurDAG->RemoveDeadNode(N: Node);
6682 return;
6683 }
6684 case Intrinsic::aarch64_sve_fscale_x4:
6685 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: AArch64::BFSCALE_4Z4Z);
6686 return;
6687 case Intrinsic::aarch64_sve_fscale_x2:
6688 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: AArch64::BFSCALE_2Z2Z);
6689 return;
6690 case Intrinsic::aarch64_sve_fmul_x4:
6691 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6692 VT: Node->getValueType(ResNo: 0),
6693 Opcodes: {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6694 AArch64::FMUL_4Z4Z_D}))
6695 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op);
6696 return;
6697 case Intrinsic::aarch64_sve_fmul_x2:
6698 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6699 VT: Node->getValueType(ResNo: 0),
6700 Opcodes: {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6701 AArch64::FMUL_2Z2Z_D}))
6702 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op);
6703 return;
6704 case Intrinsic::aarch64_sve_fcvtzs_x2:
6705 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZS_2Z2Z_StoS);
6706 return;
6707 case Intrinsic::aarch64_sve_scvtf_x2:
6708 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::SCVTF_2Z2Z_StoS);
6709 return;
6710 case Intrinsic::aarch64_sve_fcvtzu_x2:
6711 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::FCVTZU_2Z2Z_StoS);
6712 return;
6713 case Intrinsic::aarch64_sve_ucvtf_x2:
6714 SelectCVTIntrinsic(N: Node, NumVecs: 2, Opcode: AArch64::UCVTF_2Z2Z_StoS);
6715 return;
6716 case Intrinsic::aarch64_sve_fcvtzs_x4:
6717 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZS_4Z4Z_StoS);
6718 return;
6719 case Intrinsic::aarch64_sve_scvtf_x4:
6720 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::SCVTF_4Z4Z_StoS);
6721 return;
6722 case Intrinsic::aarch64_sve_fcvtzu_x4:
6723 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::FCVTZU_4Z4Z_StoS);
6724 return;
6725 case Intrinsic::aarch64_sve_ucvtf_x4:
6726 SelectCVTIntrinsic(N: Node, NumVecs: 4, Opcode: AArch64::UCVTF_4Z4Z_StoS);
6727 return;
6728 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6729 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVT_2ZZ_H_S);
6730 return;
6731 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6732 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, IsTupleInput: false, Opc: AArch64::FCVTL_2ZZ_H_S);
6733 return;
6734 case Intrinsic::aarch64_sve_sclamp_single_x2:
6735 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6736 VT: Node->getValueType(ResNo: 0),
6737 Opcodes: {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6738 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6739 SelectClamp(N: Node, NumVecs: 2, Op);
6740 return;
6741 case Intrinsic::aarch64_sve_uclamp_single_x2:
6742 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6743 VT: Node->getValueType(ResNo: 0),
6744 Opcodes: {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6745 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6746 SelectClamp(N: Node, NumVecs: 2, Op);
6747 return;
6748 case Intrinsic::aarch64_sve_fclamp_single_x2:
6749 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6750 VT: Node->getValueType(ResNo: 0),
6751 Opcodes: {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6752 AArch64::FCLAMP_VG2_2Z2Z_D}))
6753 SelectClamp(N: Node, NumVecs: 2, Op);
6754 return;
6755 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6756 SelectClamp(N: Node, NumVecs: 2, Op: AArch64::BFCLAMP_VG2_2ZZZ_H);
6757 return;
6758 case Intrinsic::aarch64_sve_sclamp_single_x4:
6759 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6760 VT: Node->getValueType(ResNo: 0),
6761 Opcodes: {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6762 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6763 SelectClamp(N: Node, NumVecs: 4, Op);
6764 return;
6765 case Intrinsic::aarch64_sve_uclamp_single_x4:
6766 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6767 VT: Node->getValueType(ResNo: 0),
6768 Opcodes: {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6769 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6770 SelectClamp(N: Node, NumVecs: 4, Op);
6771 return;
6772 case Intrinsic::aarch64_sve_fclamp_single_x4:
6773 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::FP>(
6774 VT: Node->getValueType(ResNo: 0),
6775 Opcodes: {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6776 AArch64::FCLAMP_VG4_4Z4Z_D}))
6777 SelectClamp(N: Node, NumVecs: 4, Op);
6778 return;
6779 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6780 SelectClamp(N: Node, NumVecs: 4, Op: AArch64::BFCLAMP_VG4_4ZZZ_H);
6781 return;
6782 case Intrinsic::aarch64_sve_add_single_x2:
6783 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6784 VT: Node->getValueType(ResNo: 0),
6785 Opcodes: {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6786 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6787 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: false, Opcode: Op);
6788 return;
6789 case Intrinsic::aarch64_sve_add_single_x4:
6790 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6791 VT: Node->getValueType(ResNo: 0),
6792 Opcodes: {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6793 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6794 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: false, Opcode: Op);
6795 return;
6796 case Intrinsic::aarch64_sve_zip_x2:
6797 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6798 VT: Node->getValueType(ResNo: 0),
6799 Opcodes: {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6800 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6801 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6802 return;
6803 case Intrinsic::aarch64_sve_zipq_x2:
6804 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6805 Opc: AArch64::ZIP_VG2_2ZZZ_Q);
6806 return;
6807 case Intrinsic::aarch64_sve_zip_x4:
6808 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6809 VT: Node->getValueType(ResNo: 0),
6810 Opcodes: {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6811 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6812 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6813 return;
6814 case Intrinsic::aarch64_sve_zipq_x4:
6815 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6816 Opc: AArch64::ZIP_VG4_4Z4Z_Q);
6817 return;
6818 case Intrinsic::aarch64_sve_uzp_x2:
6819 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6820 VT: Node->getValueType(ResNo: 0),
6821 Opcodes: {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6822 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6823 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6824 return;
6825 case Intrinsic::aarch64_sve_uzpq_x2:
6826 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false,
6827 Opc: AArch64::UZP_VG2_2ZZZ_Q);
6828 return;
6829 case Intrinsic::aarch64_sve_uzp_x4:
6830 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6831 VT: Node->getValueType(ResNo: 0),
6832 Opcodes: {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6833 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6834 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6835 return;
6836 case Intrinsic::aarch64_sve_uzpq_x4:
6837 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true,
6838 Opc: AArch64::UZP_VG4_4Z4Z_Q);
6839 return;
6840 case Intrinsic::aarch64_sve_sel_x2:
6841 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6842 VT: Node->getValueType(ResNo: 0),
6843 Opcodes: {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6844 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6845 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 2, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6846 return;
6847 case Intrinsic::aarch64_sve_sel_x4:
6848 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6849 VT: Node->getValueType(ResNo: 0),
6850 Opcodes: {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6851 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6852 SelectDestructiveMultiIntrinsic(N: Node, NumVecs: 4, IsZmMulti: true, Opcode: Op, /*HasPred=*/true);
6853 return;
6854 case Intrinsic::aarch64_sve_frinta_x2:
6855 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTA_2Z2Z_S);
6856 return;
6857 case Intrinsic::aarch64_sve_frinta_x4:
6858 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTA_4Z4Z_S);
6859 return;
6860 case Intrinsic::aarch64_sve_frintm_x2:
6861 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTM_2Z2Z_S);
6862 return;
6863 case Intrinsic::aarch64_sve_frintm_x4:
6864 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTM_4Z4Z_S);
6865 return;
6866 case Intrinsic::aarch64_sve_frintn_x2:
6867 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTN_2Z2Z_S);
6868 return;
6869 case Intrinsic::aarch64_sve_frintn_x4:
6870 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTN_4Z4Z_S);
6871 return;
6872 case Intrinsic::aarch64_sve_frintp_x2:
6873 SelectFrintFromVT(N: Node, NumVecs: 2, Opcode: AArch64::FRINTP_2Z2Z_S);
6874 return;
6875 case Intrinsic::aarch64_sve_frintp_x4:
6876 SelectFrintFromVT(N: Node, NumVecs: 4, Opcode: AArch64::FRINTP_4Z4Z_S);
6877 return;
6878 case Intrinsic::aarch64_sve_sunpk_x2:
6879 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6880 VT: Node->getValueType(ResNo: 0),
6881 Opcodes: {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6882 AArch64::SUNPK_VG2_2ZZ_D}))
6883 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6884 return;
6885 case Intrinsic::aarch64_sve_uunpk_x2:
6886 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6887 VT: Node->getValueType(ResNo: 0),
6888 Opcodes: {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6889 AArch64::UUNPK_VG2_2ZZ_D}))
6890 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 2, /*IsTupleInput=*/false, Opc: Op);
6891 return;
6892 case Intrinsic::aarch64_sve_sunpk_x4:
6893 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6894 VT: Node->getValueType(ResNo: 0),
6895 Opcodes: {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6896 AArch64::SUNPK_VG4_4Z2Z_D}))
6897 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6898 return;
6899 case Intrinsic::aarch64_sve_uunpk_x4:
6900 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::Int>(
6901 VT: Node->getValueType(ResNo: 0),
6902 Opcodes: {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6903 AArch64::UUNPK_VG4_4Z2Z_D}))
6904 SelectUnaryMultiIntrinsic(N: Node, NumOutVecs: 4, /*IsTupleInput=*/true, Opc: Op);
6905 return;
6906 case Intrinsic::aarch64_sve_pext_x2: {
6907 if (auto Op = SelectOpcodeFromVT<SelectTypeKind::AnyType>(
6908 VT: Node->getValueType(ResNo: 0),
6909 Opcodes: {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6910 AArch64::PEXT_2PCI_D}))
6911 SelectPExtPair(N: Node, Opc: Op);
6912 return;
6913 }
6914 }
6915 break;
6916 }
6917 case ISD::INTRINSIC_VOID: {
6918 unsigned IntNo = Node->getConstantOperandVal(Num: 1);
6919 if (Node->getNumOperands() >= 3)
6920 VT = Node->getOperand(Num: 2)->getValueType(ResNo: 0);
6921 switch (IntNo) {
6922 default:
6923 break;
6924 case Intrinsic::aarch64_neon_st1x2: {
6925 if (VT == MVT::v8i8) {
6926 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b);
6927 return;
6928 } else if (VT == MVT::v16i8) {
6929 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b);
6930 return;
6931 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6932 VT == MVT::v4bf16) {
6933 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h);
6934 return;
6935 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6936 VT == MVT::v8bf16) {
6937 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h);
6938 return;
6939 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6940 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s);
6941 return;
6942 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6943 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s);
6944 return;
6945 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6946 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d);
6947 return;
6948 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6949 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
6950 return;
6951 }
6952 break;
6953 }
6954 case Intrinsic::aarch64_neon_st1x3: {
6955 if (VT == MVT::v8i8) {
6956 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b);
6957 return;
6958 } else if (VT == MVT::v16i8) {
6959 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b);
6960 return;
6961 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6962 VT == MVT::v4bf16) {
6963 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h);
6964 return;
6965 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6966 VT == MVT::v8bf16) {
6967 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h);
6968 return;
6969 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6970 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s);
6971 return;
6972 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6973 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s);
6974 return;
6975 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6976 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d);
6977 return;
6978 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6979 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
6980 return;
6981 }
6982 break;
6983 }
6984 case Intrinsic::aarch64_neon_st1x4: {
6985 if (VT == MVT::v8i8) {
6986 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b);
6987 return;
6988 } else if (VT == MVT::v16i8) {
6989 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b);
6990 return;
6991 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6992 VT == MVT::v4bf16) {
6993 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h);
6994 return;
6995 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6996 VT == MVT::v8bf16) {
6997 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h);
6998 return;
6999 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7000 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s);
7001 return;
7002 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7003 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s);
7004 return;
7005 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7006 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d);
7007 return;
7008 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7009 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7010 return;
7011 }
7012 break;
7013 }
7014 case Intrinsic::aarch64_neon_st2: {
7015 if (VT == MVT::v8i8) {
7016 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b);
7017 return;
7018 } else if (VT == MVT::v16i8) {
7019 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b);
7020 return;
7021 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7022 VT == MVT::v4bf16) {
7023 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h);
7024 return;
7025 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7026 VT == MVT::v8bf16) {
7027 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h);
7028 return;
7029 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7030 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s);
7031 return;
7032 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7033 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s);
7034 return;
7035 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7036 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d);
7037 return;
7038 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7039 SelectStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d);
7040 return;
7041 }
7042 break;
7043 }
7044 case Intrinsic::aarch64_neon_st3: {
7045 if (VT == MVT::v8i8) {
7046 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b);
7047 return;
7048 } else if (VT == MVT::v16i8) {
7049 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b);
7050 return;
7051 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7052 VT == MVT::v4bf16) {
7053 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h);
7054 return;
7055 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7056 VT == MVT::v8bf16) {
7057 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h);
7058 return;
7059 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7060 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s);
7061 return;
7062 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7063 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s);
7064 return;
7065 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7066 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d);
7067 return;
7068 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7069 SelectStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d);
7070 return;
7071 }
7072 break;
7073 }
7074 case Intrinsic::aarch64_neon_st4: {
7075 if (VT == MVT::v8i8) {
7076 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b);
7077 return;
7078 } else if (VT == MVT::v16i8) {
7079 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b);
7080 return;
7081 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7082 VT == MVT::v4bf16) {
7083 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h);
7084 return;
7085 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7086 VT == MVT::v8bf16) {
7087 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h);
7088 return;
7089 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7090 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s);
7091 return;
7092 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7093 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s);
7094 return;
7095 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7096 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d);
7097 return;
7098 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7099 SelectStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d);
7100 return;
7101 }
7102 break;
7103 }
7104 case Intrinsic::aarch64_neon_st2lane: {
7105 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7106 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8);
7107 return;
7108 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7109 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7110 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16);
7111 return;
7112 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7113 VT == MVT::v2f32) {
7114 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32);
7115 return;
7116 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7117 VT == MVT::v1f64) {
7118 SelectStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64);
7119 return;
7120 }
7121 break;
7122 }
7123 case Intrinsic::aarch64_neon_st3lane: {
7124 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7125 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8);
7126 return;
7127 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7128 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7129 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16);
7130 return;
7131 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7132 VT == MVT::v2f32) {
7133 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32);
7134 return;
7135 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7136 VT == MVT::v1f64) {
7137 SelectStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64);
7138 return;
7139 }
7140 break;
7141 }
7142 case Intrinsic::aarch64_neon_st4lane: {
7143 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7144 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8);
7145 return;
7146 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7147 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7148 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16);
7149 return;
7150 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7151 VT == MVT::v2f32) {
7152 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32);
7153 return;
7154 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7155 VT == MVT::v1f64) {
7156 SelectStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64);
7157 return;
7158 }
7159 break;
7160 }
7161 case Intrinsic::aarch64_sve_st2q: {
7162 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 4, Opc_rr: AArch64::ST2Q, Opc_ri: AArch64::ST2Q_IMM);
7163 return;
7164 }
7165 case Intrinsic::aarch64_sve_st3q: {
7166 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 4, Opc_rr: AArch64::ST3Q, Opc_ri: AArch64::ST3Q_IMM);
7167 return;
7168 }
7169 case Intrinsic::aarch64_sve_st4q: {
7170 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 4, Opc_rr: AArch64::ST4Q, Opc_ri: AArch64::ST4Q_IMM);
7171 return;
7172 }
7173 case Intrinsic::aarch64_sve_st2: {
7174 if (VT == MVT::nxv16i8) {
7175 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 0, Opc_rr: AArch64::ST2B, Opc_ri: AArch64::ST2B_IMM);
7176 return;
7177 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7178 VT == MVT::nxv8bf16) {
7179 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 1, Opc_rr: AArch64::ST2H, Opc_ri: AArch64::ST2H_IMM);
7180 return;
7181 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7182 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 2, Opc_rr: AArch64::ST2W, Opc_ri: AArch64::ST2W_IMM);
7183 return;
7184 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7185 SelectPredicatedStore(N: Node, NumVecs: 2, Scale: 3, Opc_rr: AArch64::ST2D, Opc_ri: AArch64::ST2D_IMM);
7186 return;
7187 }
7188 break;
7189 }
7190 case Intrinsic::aarch64_sve_st3: {
7191 if (VT == MVT::nxv16i8) {
7192 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 0, Opc_rr: AArch64::ST3B, Opc_ri: AArch64::ST3B_IMM);
7193 return;
7194 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7195 VT == MVT::nxv8bf16) {
7196 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 1, Opc_rr: AArch64::ST3H, Opc_ri: AArch64::ST3H_IMM);
7197 return;
7198 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7199 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 2, Opc_rr: AArch64::ST3W, Opc_ri: AArch64::ST3W_IMM);
7200 return;
7201 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7202 SelectPredicatedStore(N: Node, NumVecs: 3, Scale: 3, Opc_rr: AArch64::ST3D, Opc_ri: AArch64::ST3D_IMM);
7203 return;
7204 }
7205 break;
7206 }
7207 case Intrinsic::aarch64_sve_st4: {
7208 if (VT == MVT::nxv16i8) {
7209 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 0, Opc_rr: AArch64::ST4B, Opc_ri: AArch64::ST4B_IMM);
7210 return;
7211 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7212 VT == MVT::nxv8bf16) {
7213 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 1, Opc_rr: AArch64::ST4H, Opc_ri: AArch64::ST4H_IMM);
7214 return;
7215 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7216 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 2, Opc_rr: AArch64::ST4W, Opc_ri: AArch64::ST4W_IMM);
7217 return;
7218 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7219 SelectPredicatedStore(N: Node, NumVecs: 4, Scale: 3, Opc_rr: AArch64::ST4D, Opc_ri: AArch64::ST4D_IMM);
7220 return;
7221 }
7222 break;
7223 }
7224 }
7225 break;
7226 }
7227 case AArch64ISD::LD2post: {
7228 if (VT == MVT::v8i8) {
7229 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8b_POST, SubRegIdx: AArch64::dsub0);
7230 return;
7231 } else if (VT == MVT::v16i8) {
7232 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov16b_POST, SubRegIdx: AArch64::qsub0);
7233 return;
7234 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7235 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4h_POST, SubRegIdx: AArch64::dsub0);
7236 return;
7237 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7238 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov8h_POST, SubRegIdx: AArch64::qsub0);
7239 return;
7240 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7241 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2s_POST, SubRegIdx: AArch64::dsub0);
7242 return;
7243 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7244 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov4s_POST, SubRegIdx: AArch64::qsub0);
7245 return;
7246 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7247 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7248 return;
7249 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7250 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Twov2d_POST, SubRegIdx: AArch64::qsub0);
7251 return;
7252 }
7253 break;
7254 }
7255 case AArch64ISD::LD3post: {
7256 if (VT == MVT::v8i8) {
7257 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8b_POST, SubRegIdx: AArch64::dsub0);
7258 return;
7259 } else if (VT == MVT::v16i8) {
7260 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev16b_POST, SubRegIdx: AArch64::qsub0);
7261 return;
7262 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7263 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4h_POST, SubRegIdx: AArch64::dsub0);
7264 return;
7265 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7266 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev8h_POST, SubRegIdx: AArch64::qsub0);
7267 return;
7268 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7269 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2s_POST, SubRegIdx: AArch64::dsub0);
7270 return;
7271 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7272 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev4s_POST, SubRegIdx: AArch64::qsub0);
7273 return;
7274 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7275 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7276 return;
7277 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7278 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Threev2d_POST, SubRegIdx: AArch64::qsub0);
7279 return;
7280 }
7281 break;
7282 }
7283 case AArch64ISD::LD4post: {
7284 if (VT == MVT::v8i8) {
7285 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7286 return;
7287 } else if (VT == MVT::v16i8) {
7288 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7289 return;
7290 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7291 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7292 return;
7293 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7294 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7295 return;
7296 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7297 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7298 return;
7299 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7300 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7301 return;
7302 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7303 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7304 return;
7305 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7306 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7307 return;
7308 }
7309 break;
7310 }
7311 case AArch64ISD::LD1x2post: {
7312 if (VT == MVT::v8i8) {
7313 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8b_POST, SubRegIdx: AArch64::dsub0);
7314 return;
7315 } else if (VT == MVT::v16i8) {
7316 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov16b_POST, SubRegIdx: AArch64::qsub0);
7317 return;
7318 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7319 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4h_POST, SubRegIdx: AArch64::dsub0);
7320 return;
7321 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7322 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov8h_POST, SubRegIdx: AArch64::qsub0);
7323 return;
7324 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7325 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2s_POST, SubRegIdx: AArch64::dsub0);
7326 return;
7327 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7328 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov4s_POST, SubRegIdx: AArch64::qsub0);
7329 return;
7330 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7331 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov1d_POST, SubRegIdx: AArch64::dsub0);
7332 return;
7333 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7334 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD1Twov2d_POST, SubRegIdx: AArch64::qsub0);
7335 return;
7336 }
7337 break;
7338 }
7339 case AArch64ISD::LD1x3post: {
7340 if (VT == MVT::v8i8) {
7341 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8b_POST, SubRegIdx: AArch64::dsub0);
7342 return;
7343 } else if (VT == MVT::v16i8) {
7344 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev16b_POST, SubRegIdx: AArch64::qsub0);
7345 return;
7346 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7347 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4h_POST, SubRegIdx: AArch64::dsub0);
7348 return;
7349 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7350 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev8h_POST, SubRegIdx: AArch64::qsub0);
7351 return;
7352 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7353 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2s_POST, SubRegIdx: AArch64::dsub0);
7354 return;
7355 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7356 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev4s_POST, SubRegIdx: AArch64::qsub0);
7357 return;
7358 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7359 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev1d_POST, SubRegIdx: AArch64::dsub0);
7360 return;
7361 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7362 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD1Threev2d_POST, SubRegIdx: AArch64::qsub0);
7363 return;
7364 }
7365 break;
7366 }
7367 case AArch64ISD::LD1x4post: {
7368 if (VT == MVT::v8i8) {
7369 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8b_POST, SubRegIdx: AArch64::dsub0);
7370 return;
7371 } else if (VT == MVT::v16i8) {
7372 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv16b_POST, SubRegIdx: AArch64::qsub0);
7373 return;
7374 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7375 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4h_POST, SubRegIdx: AArch64::dsub0);
7376 return;
7377 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7378 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv8h_POST, SubRegIdx: AArch64::qsub0);
7379 return;
7380 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7381 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2s_POST, SubRegIdx: AArch64::dsub0);
7382 return;
7383 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7384 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv4s_POST, SubRegIdx: AArch64::qsub0);
7385 return;
7386 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7387 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv1d_POST, SubRegIdx: AArch64::dsub0);
7388 return;
7389 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7390 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD1Fourv2d_POST, SubRegIdx: AArch64::qsub0);
7391 return;
7392 }
7393 break;
7394 }
7395 case AArch64ISD::LD1DUPpost: {
7396 if (VT == MVT::v8i8) {
7397 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8b_POST, SubRegIdx: AArch64::dsub0);
7398 return;
7399 } else if (VT == MVT::v16i8) {
7400 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv16b_POST, SubRegIdx: AArch64::qsub0);
7401 return;
7402 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7403 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4h_POST, SubRegIdx: AArch64::dsub0);
7404 return;
7405 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7406 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv8h_POST, SubRegIdx: AArch64::qsub0);
7407 return;
7408 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7409 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2s_POST, SubRegIdx: AArch64::dsub0);
7410 return;
7411 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7412 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv4s_POST, SubRegIdx: AArch64::qsub0);
7413 return;
7414 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7415 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv1d_POST, SubRegIdx: AArch64::dsub0);
7416 return;
7417 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7418 SelectPostLoad(N: Node, NumVecs: 1, Opc: AArch64::LD1Rv2d_POST, SubRegIdx: AArch64::qsub0);
7419 return;
7420 }
7421 break;
7422 }
7423 case AArch64ISD::LD2DUPpost: {
7424 if (VT == MVT::v8i8) {
7425 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8b_POST, SubRegIdx: AArch64::dsub0);
7426 return;
7427 } else if (VT == MVT::v16i8) {
7428 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv16b_POST, SubRegIdx: AArch64::qsub0);
7429 return;
7430 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7431 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4h_POST, SubRegIdx: AArch64::dsub0);
7432 return;
7433 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7434 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv8h_POST, SubRegIdx: AArch64::qsub0);
7435 return;
7436 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7437 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2s_POST, SubRegIdx: AArch64::dsub0);
7438 return;
7439 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7440 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv4s_POST, SubRegIdx: AArch64::qsub0);
7441 return;
7442 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7443 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv1d_POST, SubRegIdx: AArch64::dsub0);
7444 return;
7445 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7446 SelectPostLoad(N: Node, NumVecs: 2, Opc: AArch64::LD2Rv2d_POST, SubRegIdx: AArch64::qsub0);
7447 return;
7448 }
7449 break;
7450 }
7451 case AArch64ISD::LD3DUPpost: {
7452 if (VT == MVT::v8i8) {
7453 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8b_POST, SubRegIdx: AArch64::dsub0);
7454 return;
7455 } else if (VT == MVT::v16i8) {
7456 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv16b_POST, SubRegIdx: AArch64::qsub0);
7457 return;
7458 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7459 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4h_POST, SubRegIdx: AArch64::dsub0);
7460 return;
7461 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7462 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv8h_POST, SubRegIdx: AArch64::qsub0);
7463 return;
7464 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7465 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2s_POST, SubRegIdx: AArch64::dsub0);
7466 return;
7467 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7468 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv4s_POST, SubRegIdx: AArch64::qsub0);
7469 return;
7470 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7471 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv1d_POST, SubRegIdx: AArch64::dsub0);
7472 return;
7473 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7474 SelectPostLoad(N: Node, NumVecs: 3, Opc: AArch64::LD3Rv2d_POST, SubRegIdx: AArch64::qsub0);
7475 return;
7476 }
7477 break;
7478 }
7479 case AArch64ISD::LD4DUPpost: {
7480 if (VT == MVT::v8i8) {
7481 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8b_POST, SubRegIdx: AArch64::dsub0);
7482 return;
7483 } else if (VT == MVT::v16i8) {
7484 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv16b_POST, SubRegIdx: AArch64::qsub0);
7485 return;
7486 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7487 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4h_POST, SubRegIdx: AArch64::dsub0);
7488 return;
7489 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7490 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv8h_POST, SubRegIdx: AArch64::qsub0);
7491 return;
7492 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7493 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2s_POST, SubRegIdx: AArch64::dsub0);
7494 return;
7495 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7496 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv4s_POST, SubRegIdx: AArch64::qsub0);
7497 return;
7498 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7499 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv1d_POST, SubRegIdx: AArch64::dsub0);
7500 return;
7501 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7502 SelectPostLoad(N: Node, NumVecs: 4, Opc: AArch64::LD4Rv2d_POST, SubRegIdx: AArch64::qsub0);
7503 return;
7504 }
7505 break;
7506 }
7507 case AArch64ISD::LD1LANEpost: {
7508 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7509 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i8_POST);
7510 return;
7511 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7512 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7513 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i16_POST);
7514 return;
7515 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7516 VT == MVT::v2f32) {
7517 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i32_POST);
7518 return;
7519 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7520 VT == MVT::v1f64) {
7521 SelectPostLoadLane(N: Node, NumVecs: 1, Opc: AArch64::LD1i64_POST);
7522 return;
7523 }
7524 break;
7525 }
7526 case AArch64ISD::LD2LANEpost: {
7527 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7528 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i8_POST);
7529 return;
7530 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7531 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7532 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i16_POST);
7533 return;
7534 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7535 VT == MVT::v2f32) {
7536 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i32_POST);
7537 return;
7538 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7539 VT == MVT::v1f64) {
7540 SelectPostLoadLane(N: Node, NumVecs: 2, Opc: AArch64::LD2i64_POST);
7541 return;
7542 }
7543 break;
7544 }
7545 case AArch64ISD::LD3LANEpost: {
7546 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7547 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i8_POST);
7548 return;
7549 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7550 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7551 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i16_POST);
7552 return;
7553 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7554 VT == MVT::v2f32) {
7555 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i32_POST);
7556 return;
7557 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7558 VT == MVT::v1f64) {
7559 SelectPostLoadLane(N: Node, NumVecs: 3, Opc: AArch64::LD3i64_POST);
7560 return;
7561 }
7562 break;
7563 }
7564 case AArch64ISD::LD4LANEpost: {
7565 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7566 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i8_POST);
7567 return;
7568 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7569 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7570 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i16_POST);
7571 return;
7572 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7573 VT == MVT::v2f32) {
7574 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i32_POST);
7575 return;
7576 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7577 VT == MVT::v1f64) {
7578 SelectPostLoadLane(N: Node, NumVecs: 4, Opc: AArch64::LD4i64_POST);
7579 return;
7580 }
7581 break;
7582 }
7583 case AArch64ISD::ST2post: {
7584 VT = Node->getOperand(Num: 1).getValueType();
7585 if (VT == MVT::v8i8) {
7586 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8b_POST);
7587 return;
7588 } else if (VT == MVT::v16i8) {
7589 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov16b_POST);
7590 return;
7591 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7592 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4h_POST);
7593 return;
7594 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7595 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov8h_POST);
7596 return;
7597 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7598 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2s_POST);
7599 return;
7600 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7601 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov4s_POST);
7602 return;
7603 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7604 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST2Twov2d_POST);
7605 return;
7606 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7607 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7608 return;
7609 }
7610 break;
7611 }
7612 case AArch64ISD::ST3post: {
7613 VT = Node->getOperand(Num: 1).getValueType();
7614 if (VT == MVT::v8i8) {
7615 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8b_POST);
7616 return;
7617 } else if (VT == MVT::v16i8) {
7618 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev16b_POST);
7619 return;
7620 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7621 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4h_POST);
7622 return;
7623 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7624 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev8h_POST);
7625 return;
7626 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7627 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2s_POST);
7628 return;
7629 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7630 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev4s_POST);
7631 return;
7632 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7633 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST3Threev2d_POST);
7634 return;
7635 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7636 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7637 return;
7638 }
7639 break;
7640 }
7641 case AArch64ISD::ST4post: {
7642 VT = Node->getOperand(Num: 1).getValueType();
7643 if (VT == MVT::v8i8) {
7644 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8b_POST);
7645 return;
7646 } else if (VT == MVT::v16i8) {
7647 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv16b_POST);
7648 return;
7649 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7650 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4h_POST);
7651 return;
7652 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7653 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv8h_POST);
7654 return;
7655 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7656 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2s_POST);
7657 return;
7658 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7659 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv4s_POST);
7660 return;
7661 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7662 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST4Fourv2d_POST);
7663 return;
7664 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7665 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7666 return;
7667 }
7668 break;
7669 }
7670 case AArch64ISD::ST1x2post: {
7671 VT = Node->getOperand(Num: 1).getValueType();
7672 if (VT == MVT::v8i8) {
7673 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8b_POST);
7674 return;
7675 } else if (VT == MVT::v16i8) {
7676 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov16b_POST);
7677 return;
7678 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7679 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4h_POST);
7680 return;
7681 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7682 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov8h_POST);
7683 return;
7684 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7685 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2s_POST);
7686 return;
7687 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7688 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov4s_POST);
7689 return;
7690 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7691 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov1d_POST);
7692 return;
7693 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7694 SelectPostStore(N: Node, NumVecs: 2, Opc: AArch64::ST1Twov2d_POST);
7695 return;
7696 }
7697 break;
7698 }
7699 case AArch64ISD::ST1x3post: {
7700 VT = Node->getOperand(Num: 1).getValueType();
7701 if (VT == MVT::v8i8) {
7702 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8b_POST);
7703 return;
7704 } else if (VT == MVT::v16i8) {
7705 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev16b_POST);
7706 return;
7707 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7708 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4h_POST);
7709 return;
7710 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7711 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev8h_POST);
7712 return;
7713 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7714 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2s_POST);
7715 return;
7716 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7717 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev4s_POST);
7718 return;
7719 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7720 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev1d_POST);
7721 return;
7722 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7723 SelectPostStore(N: Node, NumVecs: 3, Opc: AArch64::ST1Threev2d_POST);
7724 return;
7725 }
7726 break;
7727 }
7728 case AArch64ISD::ST1x4post: {
7729 VT = Node->getOperand(Num: 1).getValueType();
7730 if (VT == MVT::v8i8) {
7731 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8b_POST);
7732 return;
7733 } else if (VT == MVT::v16i8) {
7734 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv16b_POST);
7735 return;
7736 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7737 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4h_POST);
7738 return;
7739 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7740 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv8h_POST);
7741 return;
7742 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7743 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2s_POST);
7744 return;
7745 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7746 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv4s_POST);
7747 return;
7748 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7749 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv1d_POST);
7750 return;
7751 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7752 SelectPostStore(N: Node, NumVecs: 4, Opc: AArch64::ST1Fourv2d_POST);
7753 return;
7754 }
7755 break;
7756 }
7757 case AArch64ISD::ST2LANEpost: {
7758 VT = Node->getOperand(Num: 1).getValueType();
7759 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7760 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i8_POST);
7761 return;
7762 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7763 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7764 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i16_POST);
7765 return;
7766 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7767 VT == MVT::v2f32) {
7768 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i32_POST);
7769 return;
7770 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7771 VT == MVT::v1f64) {
7772 SelectPostStoreLane(N: Node, NumVecs: 2, Opc: AArch64::ST2i64_POST);
7773 return;
7774 }
7775 break;
7776 }
7777 case AArch64ISD::ST3LANEpost: {
7778 VT = Node->getOperand(Num: 1).getValueType();
7779 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7780 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i8_POST);
7781 return;
7782 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7783 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7784 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i16_POST);
7785 return;
7786 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7787 VT == MVT::v2f32) {
7788 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i32_POST);
7789 return;
7790 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7791 VT == MVT::v1f64) {
7792 SelectPostStoreLane(N: Node, NumVecs: 3, Opc: AArch64::ST3i64_POST);
7793 return;
7794 }
7795 break;
7796 }
7797 case AArch64ISD::ST4LANEpost: {
7798 VT = Node->getOperand(Num: 1).getValueType();
7799 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7800 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i8_POST);
7801 return;
7802 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7803 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7804 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i16_POST);
7805 return;
7806 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7807 VT == MVT::v2f32) {
7808 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i32_POST);
7809 return;
7810 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7811 VT == MVT::v1f64) {
7812 SelectPostStoreLane(N: Node, NumVecs: 4, Opc: AArch64::ST4i64_POST);
7813 return;
7814 }
7815 break;
7816 }
7817 }
7818
7819 // Select the default instruction
7820 SelectCode(N: Node);
7821}
7822
7823/// createAArch64ISelDag - This pass converts a legalized DAG into a
7824/// AArch64-specific DAG, ready for instruction scheduling.
7825FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
7826 CodeGenOptLevel OptLevel) {
7827 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7828}
7829
7830/// When \p PredVT is a scalable vector predicate in the form
7831/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7832/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7833/// structured vectors (NumVec >1), the output data type is
7834/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7835/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7836/// EVT.
7837static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT,
7838 unsigned NumVec) {
7839 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7840 if (!PredVT.isScalableVectorOf(EltVT: MVT::i1))
7841 return EVT();
7842
7843 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7844 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7845 return EVT();
7846
7847 ElementCount EC = PredVT.getVectorElementCount();
7848 EVT ScalarVT =
7849 EVT::getIntegerVT(Context&: Ctx, BitWidth: AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7850 EVT MemVT = EVT::getVectorVT(Context&: Ctx, VT: ScalarVT, EC: EC * NumVec);
7851
7852 return MemVT;
7853}
7854
7855/// Builds an integer vector type large enough to hold \p NumVec instances
7856/// of \p VecVT.
7857static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7858 return EVT::getVectorVT(Context&: Ctx, VT: VecVT.getScalarType().changeTypeToInteger(),
7859 EC: VecVT.getVectorElementCount() * NumVec);
7860}
7861
7862/// Return the EVT of the data associated to a memory operation in \p
7863/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7864static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
7865 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Val: Root))
7866 return MemIntr->getMemoryVT();
7867
7868 if (isa<MemSDNode>(Val: Root)) {
7869 EVT MemVT = cast<MemSDNode>(Val: Root)->getMemoryVT();
7870
7871 EVT DataVT;
7872 if (auto *Load = dyn_cast<LoadSDNode>(Val: Root))
7873 DataVT = Load->getValueType(ResNo: 0);
7874 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Val: Root))
7875 DataVT = Load->getValueType(ResNo: 0);
7876 else if (auto *Store = dyn_cast<StoreSDNode>(Val: Root))
7877 DataVT = Store->getValue().getValueType();
7878 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Val: Root))
7879 DataVT = Store->getValue().getValueType();
7880 else
7881 llvm_unreachable("Unexpected MemSDNode!");
7882
7883 return DataVT.changeVectorElementType(Context&: Ctx, EltVT: MemVT.getVectorElementType());
7884 }
7885
7886 const unsigned Opcode = Root->getOpcode();
7887 // For custom ISD nodes, we have to look at them individually to extract the
7888 // type of the data moved to/from memory.
7889 switch (Opcode) {
7890 case AArch64ISD::LD1_MERGE_ZERO:
7891 case AArch64ISD::LD1S_MERGE_ZERO:
7892 case AArch64ISD::LDNF1_MERGE_ZERO:
7893 case AArch64ISD::LDNF1S_MERGE_ZERO:
7894 return cast<VTSDNode>(Val: Root->getOperand(Num: 3))->getVT();
7895 case AArch64ISD::ST1_PRED:
7896 return cast<VTSDNode>(Val: Root->getOperand(Num: 4))->getVT();
7897 default:
7898 break;
7899 }
7900
7901 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7902 return EVT();
7903
7904 switch (Root->getConstantOperandVal(Num: 1)) {
7905 default:
7906 return EVT();
7907 case Intrinsic::aarch64_sme_ldr:
7908 case Intrinsic::aarch64_sme_str:
7909 return MVT::nxv16i8;
7910 case Intrinsic::aarch64_sve_prf:
7911 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7912 // width of the predicate.
7913 return getPackedVectorTypeFromPredicateType(
7914 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/1);
7915 case Intrinsic::aarch64_sve_ld2_sret:
7916 case Intrinsic::aarch64_sve_ld2q_sret:
7917 return getPackedVectorTypeFromPredicateType(
7918 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/2);
7919 case Intrinsic::aarch64_sve_st2q:
7920 return getPackedVectorTypeFromPredicateType(
7921 Ctx, PredVT: Root->getOperand(Num: 4)->getValueType(ResNo: 0), /*NumVec=*/2);
7922 case Intrinsic::aarch64_sve_ld3_sret:
7923 case Intrinsic::aarch64_sve_ld3q_sret:
7924 return getPackedVectorTypeFromPredicateType(
7925 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/3);
7926 case Intrinsic::aarch64_sve_st3q:
7927 return getPackedVectorTypeFromPredicateType(
7928 Ctx, PredVT: Root->getOperand(Num: 5)->getValueType(ResNo: 0), /*NumVec=*/3);
7929 case Intrinsic::aarch64_sve_ld4_sret:
7930 case Intrinsic::aarch64_sve_ld4q_sret:
7931 return getPackedVectorTypeFromPredicateType(
7932 Ctx, PredVT: Root->getOperand(Num: 2)->getValueType(ResNo: 0), /*NumVec=*/4);
7933 case Intrinsic::aarch64_sve_st4q:
7934 return getPackedVectorTypeFromPredicateType(
7935 Ctx, PredVT: Root->getOperand(Num: 6)->getValueType(ResNo: 0), /*NumVec=*/4);
7936 case Intrinsic::aarch64_sve_ld1_pn_x2:
7937 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7938 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7939 /*NumVec=*/2);
7940 case Intrinsic::aarch64_sve_ld1_pn_x4:
7941 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7942 return getMultipleVectorType(Ctx, VecVT: Root->getValueType(ResNo: 0),
7943 /*NumVec=*/4);
7944 case Intrinsic::aarch64_sve_st1_pn_x2:
7945 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7946 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7947 /*NumVec=*/2);
7948 case Intrinsic::aarch64_sve_st1_pn_x4:
7949 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7950 return getMultipleVectorType(Ctx, VecVT: Root->getOperand(Num: 2).getValueType(),
7951 /*NumVec=*/4);
7952 case Intrinsic::aarch64_sve_ld1udq:
7953 case Intrinsic::aarch64_sve_st1dq:
7954 return EVT(MVT::nxv1i64);
7955 case Intrinsic::aarch64_sve_ld1uwq:
7956 case Intrinsic::aarch64_sve_st1wq:
7957 return EVT(MVT::nxv1i32);
7958 }
7959}
7960
7961/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7962/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7963/// where Root is the memory access using N for its address.
7964template <int64_t Min, int64_t Max>
7965bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7966 SDValue &Base,
7967 SDValue &OffImm) {
7968 const EVT MemVT = getMemVTFromNode(Ctx&: *(CurDAG->getContext()), Root);
7969 const DataLayout &DL = CurDAG->getDataLayout();
7970 const MachineFrameInfo &MFI = MF->getFrameInfo();
7971
7972 if (N.getOpcode() == ISD::FrameIndex) {
7973 int FI = cast<FrameIndexSDNode>(Val&: N)->getIndex();
7974 // We can only encode VL scaled offsets, so only fold in frame indexes
7975 // referencing SVE objects.
7976 if (MFI.hasScalableStackID(ObjectIdx: FI)) {
7977 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
7978 OffImm = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
7979 return true;
7980 }
7981
7982 return false;
7983 }
7984
7985 if (MemVT == EVT())
7986 return false;
7987
7988 if (N.getOpcode() != ISD::ADD)
7989 return false;
7990
7991 SDValue VScale = N.getOperand(i: 1);
7992 int64_t MulImm = std::numeric_limits<int64_t>::max();
7993 if (VScale.getOpcode() == ISD::VSCALE) {
7994 MulImm = cast<ConstantSDNode>(Val: VScale.getOperand(i: 0))->getSExtValue();
7995 } else if (auto C = dyn_cast<ConstantSDNode>(Val&: VScale)) {
7996 int64_t ByteOffset = C->getSExtValue();
7997 const auto KnownVScale =
7998 Subtarget->getSVEVectorSizeInBits() / AArch64::SVEBitsPerBlock;
7999
8000 if (!KnownVScale || ByteOffset % KnownVScale != 0)
8001 return false;
8002
8003 MulImm = ByteOffset / KnownVScale;
8004 } else
8005 return false;
8006
8007 TypeSize TS = MemVT.getSizeInBits();
8008 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
8009
8010 if ((MulImm % MemWidthBytes) != 0)
8011 return false;
8012
8013 int64_t Offset = MulImm / MemWidthBytes;
8014 if (Offset < Min || Offset > Max)
8015 return false;
8016
8017 Base = N.getOperand(i: 0);
8018 if (Base.getOpcode() == ISD::FrameIndex) {
8019 int FI = cast<FrameIndexSDNode>(Val&: Base)->getIndex();
8020 // We can only encode VL scaled offsets, so only fold in frame indexes
8021 // referencing SVE objects.
8022 if (MFI.hasScalableStackID(ObjectIdx: FI))
8023 Base = CurDAG->getTargetFrameIndex(FI, VT: TLI->getPointerTy(DL));
8024 }
8025
8026 OffImm = CurDAG->getTargetConstant(Val: Offset, DL: SDLoc(N), VT: MVT::i64);
8027 return true;
8028}
8029
8030/// Select register plus register addressing mode for SVE, with scaled
8031/// offset.
8032bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
8033 SDValue &Base,
8034 SDValue &Offset) {
8035 if (N.getOpcode() != ISD::ADD)
8036 return false;
8037
8038 // Process an ADD node.
8039 const SDValue LHS = N.getOperand(i: 0);
8040 const SDValue RHS = N.getOperand(i: 1);
8041
8042 // 8 bit data does not come with the SHL node, so it is treated
8043 // separately.
8044 if (Scale == 0) {
8045 Base = LHS;
8046 Offset = RHS;
8047 return true;
8048 }
8049
8050 if (auto C = dyn_cast<ConstantSDNode>(Val: RHS)) {
8051 int64_t ImmOff = C->getSExtValue();
8052 unsigned Size = 1 << Scale;
8053
8054 // To use the reg+reg addressing mode, the immediate must be a multiple of
8055 // the vector element's byte size.
8056 if (ImmOff % Size)
8057 return false;
8058
8059 SDLoc DL(N);
8060 Base = LHS;
8061 Offset = CurDAG->getTargetConstant(Val: ImmOff >> Scale, DL, VT: MVT::i64);
8062 SDValue Ops[] = {Offset};
8063 SDNode *MI = CurDAG->getMachineNode(Opcode: AArch64::MOVi64imm, dl: DL, VT: MVT::i64, Ops);
8064 Offset = SDValue(MI, 0);
8065 return true;
8066 }
8067
8068 // Check if the RHS is a shift node with a constant.
8069 if (RHS.getOpcode() != ISD::SHL)
8070 return false;
8071
8072 const SDValue ShiftRHS = RHS.getOperand(i: 1);
8073 if (auto *C = dyn_cast<ConstantSDNode>(Val: ShiftRHS))
8074 if (C->getZExtValue() == Scale) {
8075 Base = LHS;
8076 Offset = RHS.getOperand(i: 0);
8077 return true;
8078 }
8079
8080 return false;
8081}
8082
8083bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8084 const AArch64TargetLowering *TLI =
8085 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8086
8087 return TLI->isAllActivePredicate(DAG: *CurDAG, N);
8088}
8089
8090bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8091 return N.getValueType().isScalableVectorOf(EltVT: MVT::i1);
8092}
8093
8094bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8095 SDValue &Base, SDValue &Offset,
8096 unsigned Scale) {
8097 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8098 if (auto *C = dyn_cast<ConstantSDNode>(Val&: CN)) {
8099 int64_t ImmOff = C->getSExtValue();
8100 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8101 return CurDAG->getTargetConstant(Val: ImmOff / Scale, DL: SDLoc(N), VT: MVT::i64);
8102 }
8103 return SDValue();
8104 };
8105
8106 if (SDValue C = MatchConstantOffset(N)) {
8107 Base = getZeroRegister(DAG&: *CurDAG, DL: SDLoc(N), VT: MVT::i32);
8108 Offset = C;
8109 return true;
8110 }
8111
8112 // Try to untangle an ADD node into a 'reg + offset'
8113 if (CurDAG->isBaseWithConstantOffset(Op: N)) {
8114 if (SDValue C = MatchConstantOffset(N.getOperand(i: 1))) {
8115 Base = N.getOperand(i: 0);
8116 Offset = C;
8117 return true;
8118 }
8119 }
8120
8121 // By default, just match reg + 0.
8122 Base = N;
8123 Offset = CurDAG->getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i64);
8124 return true;
8125}
8126
8127bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8128 SDValue &Imm) {
8129 AArch64CC::CondCode CC =
8130 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(Num: 1));
8131 if (auto *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
8132 // Check conservatively if the immediate fits the valid range [0, 64).
8133 // Immediate variants for GE and HS definitely need to be decremented
8134 // when lowering the pseudos later, so an immediate of 1 would become 0.
8135 // For the inverse conditions LT and LO we don't know for sure if they
8136 // will need a decrement but should the decision be made to reverse the
8137 // branch condition, we again end up with the need to decrement.
8138 // The same argument holds for LE, LS, GT and HI and possibly
8139 // incremented immediates. This can lead to slightly less optimal
8140 // codegen, e.g. we never codegen the legal case
8141 // cblt w0, #63, A
8142 // because we could end up with the illegal case
8143 // cbge w0, #64, B
8144 // should the decision to reverse the branch direction be made. For the
8145 // lower bound cases this is no problem since we can express comparisons
8146 // against 0 with either tbz/tnbz or using wzr/xzr.
8147 uint64_t LowerBound = 0, UpperBound = 64;
8148 switch (CC) {
8149 case AArch64CC::GE:
8150 case AArch64CC::HS:
8151 case AArch64CC::LT:
8152 case AArch64CC::LO:
8153 LowerBound = 1;
8154 break;
8155 case AArch64CC::LE:
8156 case AArch64CC::LS:
8157 case AArch64CC::GT:
8158 case AArch64CC::HI:
8159 UpperBound = 63;
8160 break;
8161 default:
8162 break;
8163 }
8164
8165 if (CN->getAPIntValue().uge(RHS: LowerBound) &&
8166 CN->getAPIntValue().ult(RHS: UpperBound)) {
8167 SDLoc DL(N);
8168 Imm = CurDAG->getTargetConstant(Val: CN->getZExtValue(), DL, VT: N.getValueType());
8169 return true;
8170 }
8171 }
8172
8173 return false;
8174}
8175
8176template <bool MatchCBB>
8177bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8178 SDValue &ExtType) {
8179
8180 // Use an invalid shift-extend value to indicate we don't need to extend later
8181 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8182 EVT Ty = cast<VTSDNode>(Val: N.getOperand(i: 1))->getVT();
8183 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8184 return false;
8185 Reg = N.getOperand(i: 0);
8186 ExtType = CurDAG->getSignedTargetConstant(Val: AArch64_AM::InvalidShiftExtend,
8187 DL: SDLoc(N), VT: MVT::i32);
8188 return true;
8189 }
8190
8191 AArch64_AM::ShiftExtendType ET = getExtendTypeForNode(N);
8192
8193 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8194 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8195 Reg = N.getOperand(i: 0);
8196 ExtType =
8197 CurDAG->getTargetConstant(Val: getExtendEncoding(ET), DL: SDLoc(N), VT: MVT::i32);
8198 return true;
8199 }
8200
8201 return false;
8202}
8203
8204/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8205///
8206/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8207/// doing this earlier based on the defining operation of X can be invalidated
8208/// by later DAG combines. At this point the DAG is being prepared for
8209/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8210/// final SDValue being selected.
8211/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8212/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8213/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8214/// differ from fcmp+fcsel for signaling NaN inputs.
8215bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *N) {
8216 EVT VT = N->getValueType(ResNo: 0);
8217
8218 // Scalar FP only.
8219 if (!VT.isFloatingPoint() || VT.isVector())
8220 return false;
8221
8222 SDValue TVal = N->getOperand(Num: 0);
8223 SDValue FVal = N->getOperand(Num: 1);
8224 SDValue CCVal = N->getOperand(Num: 2);
8225 SDValue Cmp = N->getOperand(Num: 3);
8226
8227 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8228 return false;
8229
8230 auto *CC = dyn_cast<ConstantSDNode>(Val&: CCVal);
8231 if (!CC)
8232 return false;
8233
8234 SDValue CmpLHS = Cmp.getOperand(i: 0);
8235 SDValue CmpRHS = Cmp.getOperand(i: 1);
8236 unsigned CondCode = CC->getZExtValue();
8237
8238 // Map VT and operation (max/min) to machine opcode.
8239 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8240 if (VT == MVT::f16)
8241 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8242 else if (VT == MVT::f32)
8243 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8244 else if (VT == MVT::f64)
8245 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8246 else
8247 return 0; // unsupported
8248 };
8249
8250 // Determine whether to use max or min based on condition code and operands.
8251 bool isMax;
8252 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8253 if (TVal == CmpLHS && FVal == CmpRHS)
8254 isMax = true;
8255 else
8256 return false;
8257 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8258 if (TVal == CmpLHS && FVal == CmpRHS)
8259 isMax = false;
8260 else
8261 return false;
8262 } else {
8263 return false;
8264 }
8265
8266 // Get the machine opcode for this VT and operation.
8267 unsigned Opc = getOpc(VT, isMax);
8268 if (!Opc)
8269 return false;
8270
8271 // Constant must be non-NaN.
8272 auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: CmpRHS);
8273 if (!CFP || CFP->getValueAPF().isNaN())
8274 return false;
8275
8276 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8277 // fcmp+select's -0. For non-zero constants, semantics are identical.
8278 if (CFP->isZero() && !N->getFlags().hasNoSignedZeros())
8279 return false;
8280
8281 // Only fold if variable operand is never sNaN.
8282 // This runs after DAG combines, so later combines cannot remove a defining
8283 // operation used by isKnownNeverSNaN().
8284 if (!CurDAG->isKnownNeverSNaN(Op: CmpLHS))
8285 return false;
8286
8287 CurDAG->SelectNodeTo(N, MachineOpc: Opc, VT, Op1: CmpLHS, Op2: CmpRHS);
8288 return true;
8289}
8290
8291void AArch64DAGToDAGISel::PreprocessISelDAG() {
8292 bool MadeChange = false;
8293 for (SDNode &N : llvm::make_early_inc_range(Range: CurDAG->allnodes())) {
8294 if (N.use_empty())
8295 continue;
8296
8297 SDValue Result;
8298 switch (N.getOpcode()) {
8299 case ISD::SCALAR_TO_VECTOR: {
8300 EVT ScalarTy = N.getValueType(ResNo: 0).getVectorElementType();
8301 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8302 ScalarTy == N.getOperand(Num: 0).getValueType())
8303 Result = addBitcastHints(DAG&: *CurDAG, N);
8304
8305 break;
8306 }
8307 case AArch64ISD::VSHL: {
8308 // Undo mul(shl(A,C),B) -> shl(mul(A,B),C) canonicalisation when A is an
8309 // extend that can be folded into the shift.
8310 EVT VT = N.getValueType(ResNo: 0);
8311 SDValue A, B, C = N.getOperand(Num: 1);
8312 if (sd_match(N: N.getOperand(Num: 0),
8313 P: m_OneUse(P: m_Mul(L: m_Value(N&: A, P: m_AnyOf(preds: m_ZExt(Op: m_Value()),
8314 preds: m_SExt(Op: m_Value()))),
8315 R: m_Value(N&: B))))) {
8316 // If both mul operands are extended, preserve the smull/umull idiom.
8317 if (B.getOpcode() == A.getOpcode())
8318 break;
8319 SDLoc DL(&N);
8320 SDValue SHL = CurDAG->getNode(Opcode: AArch64ISD::VSHL, DL, VT, N1: A, N2: C);
8321 Result = CurDAG->getNode(Opcode: ISD::MUL, DL, VT, N1: SHL, N2: B);
8322 }
8323 break;
8324 }
8325 default:
8326 break;
8327 }
8328
8329 if (Result) {
8330 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8331 LLVM_DEBUG(N.dump(CurDAG));
8332 LLVM_DEBUG(dbgs() << "\nNew: ");
8333 LLVM_DEBUG(Result.dump(CurDAG));
8334 LLVM_DEBUG(dbgs() << "\n");
8335
8336 CurDAG->ReplaceAllUsesOfValueWith(From: SDValue(&N, 0), To: Result);
8337 MadeChange = true;
8338 }
8339 }
8340
8341 if (MadeChange)
8342 CurDAG->RemoveDeadNodes();
8343
8344 SelectionDAGISel::PreprocessISelDAG();
8345}
8346